Add compatibility for VyOS bare-metal setup

.gitignore

@@ -1,3 +1,4 @@
 /ch-k8s-lbaas-controller
 /ch-k8s-lbaas-agent
 /vendor
+venv

cmd/ch-k8s-lbaas-agent/ch-k8s-lbaas-agent.go

@@ -61,14 +61,23 @@ func main() {
 		},
 	}
 
-	keepalivedConfig := &agent.ConfigManager{
-		Service: fileCfg.Keepalived.Service,
-		Generator: &agent.KeepalivedConfigGenerator{
-			VRIDBase:     fileCfg.Keepalived.VRIDBase,
-			VRRPPassword: fileCfg.Keepalived.VRRPPassword,
-			Interface:    fileCfg.Keepalived.Interface,
-			Priority:     fileCfg.Keepalived.Priority,
-		},
+	var keepalivedConfig *agent.ConfigManager
+
+	if fileCfg.Keepalived.Enabled {
+		keepalivedConfig = &agent.ConfigManager{
+			Service: fileCfg.Keepalived.Service,
+			Generator: &agent.KeepalivedConfigGenerator{
+				VRIDBase:     fileCfg.Keepalived.VRIDBase,
+				VRRPPassword: fileCfg.Keepalived.VRRPPassword,
+				Interface:    fileCfg.Keepalived.Interface,
+				Priority:     fileCfg.Keepalived.Priority,
+			},
+		}
+	}
+
+	// If LiveReload is enabled, reload nftables config directly after start to apply last state
+	if fileCfg.Nftables.LiveReload {
+		nftablesConfig.Reload()
 	}
 
 	http.Handle("/v1/apply", &agent.ApplyHandlerv1{

cmd/ch-k8s-lbaas-controller/ch-k8s-lbaas-controller.go

@@ -20,6 +20,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"github.com/cloudandheat/ch-k8s-lbaas/internal/static"
 	"net"
 	"net/http"
 	"time"
@@ -71,16 +72,30 @@ func main() {
 	}
 	config.FillControllerConfig(&fileCfg)
 
-	osClient, err := openstack.NewClient(&fileCfg.OpenStack.Global)
+	err = config.ValidateControllerConfig(&fileCfg)
 	if err != nil {
-		klog.Fatalf("Failed to connect to OpenStack: %s", err.Error())
+		klog.Fatalf("invalid configuration: %s", err.Error())
 	}
 
-	l3portmanager, err := osClient.NewOpenStackL3PortManager(
-		&fileCfg.OpenStack.Networking,
-	)
-	if err != nil {
-		klog.Fatalf("Failed to create L3 port manager: %s", err.Error())
+	var l3portmanager controller.L3PortManager
+
+	if fileCfg.PortManager == "openstack" {
+		osClient, err := openstack.NewClient(&fileCfg.OpenStack.Global)
+		if err != nil {
+			klog.Fatalf("Failed to connect to OpenStack: %s", err.Error())
+		}
+
+		l3portmanager, err = osClient.NewOpenStackL3PortManager(
+			&fileCfg.OpenStack.Networking,
+		)
+		if err != nil {
+			klog.Fatalf("Failed to create openstack L3 port manager: %s", err.Error())
+		}
+	} else if fileCfg.PortManager == "static" {
+		l3portmanager, err = static.NewStaticL3PortManager(&fileCfg.Static)
+		if err != nil {
+			klog.Fatalf("Failed to create static L3 port manager: %s", err.Error())
+		}
 	}
 
 	agentController, err := controller.NewHTTPAgentController(fileCfg.Agents)

internal/agent/agent.go

@@ -289,21 +289,27 @@ func (h *ApplyHandlerv1) ProcessRequest(lbcfg *model.LoadBalancer) (int, string)
 		return 400, err.Error() // Bad Request
 	}
 
-	changed, err := h.KeepalivedConfig.WriteWithRollback(lbcfg)
-	if err != nil {
-		msg := fmt.Sprintf("Failed to apply keepalived config: %s", err.Error())
-		klog.Error(msg)
-		return 500, msg
+	keepalivedChanged, nftablesChanged := false, false
+
+	if h.KeepalivedConfig != nil {
+		keepalivedChanged, err = h.KeepalivedConfig.WriteWithRollback(lbcfg)
+		if err != nil {
+			msg := fmt.Sprintf("Failed to apply keepalived config: %s", err.Error())
+			klog.Error(msg)
+			return 500, msg
+		}
 	}
 
-	changed, err = h.NftablesConfig.WriteWithRollback(lbcfg)
-	if err != nil {
-		msg := fmt.Sprintf("Failed to apply nftables config: %s", err.Error())
-		klog.Error(msg)
-		return 500, msg
+	if h.NftablesConfig != nil {
+		nftablesChanged, err = h.NftablesConfig.WriteWithRollback(lbcfg)
+		if err != nil {
+			msg := fmt.Sprintf("Failed to apply nftables config: %s", err.Error())
+			klog.Error(msg)
+			return 500, msg
+		}
 	}
 
-	if changed {
+	if keepalivedChanged || nftablesChanged {
 		klog.Infof("Applied configuration update: %#v", lbcfg)
 	}
 

internal/agent/nftables_generator.go

@@ -15,14 +15,19 @@
 package agent
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
+	"os"
+	"os/exec"
 	"sort"
 	"strconv"
 	"strings"
 	"text/template"
 
+	"k8s.io/klog"
+
 	corev1 "k8s.io/api/core/v1"
 
 	"github.com/cloudandheat/ch-k8s-lbaas/internal/config"
@@ -32,43 +37,54 @@ import (
 var (
 	nftablesTemplate = template.Must(template.New("nftables.conf").Parse(`
 {{ $cfg := . }}
+
+flush chain ip {{ .NATTableName }} {{ .NATPreroutingChainName }}
+flush chain ip {{ .NATTableName }} {{ .NATPostroutingChainName }}
+flush chain {{ .FilterTableType }} {{ .FilterTableName }} {{ .FilterForwardChainName }}
+
+# The "add chain" command is a workaround to make sure that the chain exists before deleting it.
+{{- range $chain := $cfg.ExistingPolicyChains }}
+add chain {{ $cfg.FilterTableType }} {{ $cfg.FilterTableName }} {{ $chain }}
+delete chain {{ $cfg.FilterTableType }} {{ $cfg.FilterTableName }} {{ $chain }}
+{{- end }}
+
 table {{ .FilterTableType }} {{ .FilterTableName }} {
 	chain {{ .FilterForwardChainName }} {
 		{{- range $dest := $cfg.PolicyAssignments }}
-		ct mark {{ $cfg.FWMarkBits | printf "0x%x" }} and {{ $cfg.FWMarkMask | printf "0x%x" }} ip daddr {{ $dest.Address }} goto POD-{{ $dest.Address }};
+		ct mark {{ $cfg.FWMarkBits | printf "0x%x" }} and {{ $cfg.FWMarkMask | printf "0x%x" }} ip daddr {{ $dest.Address }} goto {{ $cfg.PolicyPrefix }}POD-{{ $dest.Address }};
 		{{- end }}
 		ct mark {{ $cfg.FWMarkBits | printf "0x%x" }} and {{ $cfg.FWMarkMask | printf "0x%x" }} accept;
 	}
 
 	# Using uppercase POD to prevent collisions with policy names like 'pod-x.x.x.x'
 	{{- range $pod := $cfg.PolicyAssignments }}
-	chain POD-{{ $pod.Address }} {
+	chain {{ $cfg.PolicyPrefix }}POD-{{ $pod.Address }} {
 		{{- range $pol := $pod.NetworkPolicies }}
-		jump {{ $pol }};
+		jump {{ $cfg.PolicyPrefix }}{{ $pol }};
 		{{- end }}
 		drop;
 	}
 	{{- end }}
 
 	# Using uppercase RULE and CIDR to prevent collisions with policy names like 'x-rule-y-cidr-z'
 	{{- range $policy := $cfg.NetworkPolicies }}
-	chain {{ $policy.Name }} {
+	chain {{ $cfg.PolicyPrefix }}{{ $policy.Name }} {
 		{{- range $ruleIndex, $ingressRule := $policy.IngressRuleChains }}
-		jump {{ $policy.Name }}-RULE{{ $ruleIndex }};
+		jump {{ $cfg.PolicyPrefix }}{{ $policy.Name }}-RULE{{ $ruleIndex }};
 		{{- end }}
 	}
 
 	{{- range $ruleIndex, $ingressRule := $policy.IngressRuleChains }}
-	chain {{ $policy.Name }}-RULE{{ $ruleIndex }} {
+	chain {{ $cfg.PolicyPrefix }}{{ $policy.Name }}-RULE{{ $ruleIndex }} {
 		{{- range $entryIndex, $entry := $ingressRule.Entries }}
-		{{ $entry.SaddrMatch.Match }} {{ $entry.PortMatch }} {{- if ne ($entry.SaddrMatch.Except | len) 0 }} jump {{ $policy.Name }}-RULE{{ $ruleIndex }}-CIDR{{ $entryIndex }} {{- else }} accept {{- end }};
+		{{ $entry.SaddrMatch.Match }} {{ $entry.PortMatch }} {{- if ne ($entry.SaddrMatch.Except | len) 0 }} jump {{ $cfg.PolicyPrefix }}{{ $policy.Name }}-RULE{{ $ruleIndex }}-CIDR{{ $entryIndex }} {{- else }} accept {{- end }};
 
 		{{- end }}
 	}
 
 	{{- range $entryIndex, $entry := $ingressRule.Entries }}
 		{{- if ne ($entry.SaddrMatch.Except | len) 0 }}
-	chain {{ $policy.Name }}-RULE{{ $ruleIndex }}-CIDR{{ $entryIndex }} {
+	chain {{ $cfg.PolicyPrefix }}{{ $policy.Name }}-RULE{{ $ruleIndex }}-CIDR{{ $entryIndex }} {
 		{{- range $addr := $entry.SaddrMatch.Except }}
 		ip saddr {{ $addr }} return;
 		{{- end}}
@@ -151,17 +167,33 @@ type nftablesConfig struct {
 	NATTableName            string
 	NATPostroutingChainName string
 	NATPreroutingChainName  string
+	PolicyPrefix            string
 	FWMarkBits              uint32
 	FWMarkMask              uint32
 	Forwards                []nftablesForward
 	NetworkPolicies         map[string]networkPolicy
 	PolicyAssignments       []policyAssignment
+	ExistingPolicyChains    []string
 }
 
 type NftablesGenerator struct {
 	Cfg config.Nftables
 }
 
+type nftablesChainListResultChain struct {
+	Family string `json:"family"`
+	Table  string `json:"table"`
+	Name   string `json:"name"`
+}
+
+type nftablesChainListResultEntry struct {
+	Chain nftablesChainListResultChain `json:"chain,omitempty"`
+}
+
+type nftablesChainListResult struct {
+	Nftables []nftablesChainListResultEntry `json:"nftables"`
+}
+
 // Takes a slice of strings and produces a list ready to be used in an nftables rule
 // eg. []string{"1", "2", "3"} => "{1,2,3}"
 func makeNftablesList(items []string) (string, error) {
@@ -337,6 +369,45 @@ func mapProtocol(k8sproto corev1.Protocol) (string, error) {
 	}
 }
 
+// Return all chain names of type `filterTableType` in table `filterTableName` that start with `policyPrefix`.
+// Uses the `nftCommand`.
+func getExistingPolicyChains(nftCommand []string, filterTableName string, filterTableType string, policyPrefix string) ([]string, error) {
+	// Prepare "list chains" command to get all chains of type filterTableType
+	cmd := append(nftCommand, "-j", "list", "chains", filterTableType)
+
+	klog.V(4).Infof("executing command: %#v", cmd)
+
+	cmdObj := exec.Command(cmd[0], cmd[1:]...)
+	cmdObj.Stderr = os.Stderr
+
+	out, err := cmdObj.Output()
+	if err != nil {
+		return []string{}, fmt.Errorf("failed to get exiting policy chains via %#v: %s", cmd, err.Error())
+	}
+
+	// Parse result from JSON
+	var result nftablesChainListResult
+	err = json.Unmarshal(out, &result)
+
+	if err != nil {
+		return []string{}, fmt.Errorf("could not parse existing policy json: %s", err.Error())
+	}
+
+	var existingChains []string
+
+	// Iterate over all returned chains and check if the conditions are met
+	for _, resultEntry := range result.Nftables {
+		if resultEntry.Chain.Family == filterTableType &&
+			resultEntry.Chain.Table == filterTableName &&
+			strings.HasPrefix(resultEntry.Chain.Name, policyPrefix) {
+			// Append chain name to list
+			existingChains = append(existingChains, resultEntry.Chain.Name)
+		}
+	}
+
+	return existingChains, nil
+}
+
 // Generates a config suitable for nftablesTemplate from a LoadBalancer model
 func (g *NftablesGenerator) GenerateStructuredConfig(m *model.LoadBalancer) (*nftablesConfig, error) {
 	result := &nftablesConfig{
@@ -346,11 +417,13 @@ func (g *NftablesGenerator) GenerateStructuredConfig(m *model.LoadBalancer) (*nf
 		NATTableName:            g.Cfg.NATTableName,
 		NATPostroutingChainName: g.Cfg.NATPostroutingChainName,
 		NATPreroutingChainName:  g.Cfg.NATPreroutingChainName,
+		PolicyPrefix:            g.Cfg.PolicyPrefix,
 		FWMarkBits:              g.Cfg.FWMarkBits,
 		FWMarkMask:              g.Cfg.FWMarkMask,
 		Forwards:                []nftablesForward{},
 		NetworkPolicies:         map[string]networkPolicy{},
 		PolicyAssignments:       []policyAssignment{},
+		ExistingPolicyChains:    []string{},
 	}
 
 	for _, ingress := range m.Ingress {
@@ -396,6 +469,15 @@ func (g *NftablesGenerator) GenerateStructuredConfig(m *model.LoadBalancer) (*nf
 		result.NetworkPolicies[policy.Name] = policy
 	}
 
+	if g.Cfg.LiveReload {
+		// When live reload is enabled, get all existing policy chain names to delete them in the template
+		result.ExistingPolicyChains, err = getExistingPolicyChains(
+			g.Cfg.NftCommand,
+			result.FilterTableName,
+			result.FilterTableType,
+			result.PolicyPrefix)
+	}
+
 	return result, nil
 }