perf: Performance enhancement for adding many rules to Linux IP… (#1644)

* added optional performance strategy for adding chain rule with many rules into linux netfilter (iptables)

Signed-off-by: Filip Gschwandtner <filip.gschwandtner@pantheon.tech>

* moved logic of adding multiple rules into iptables handler

Signed-off-by: Filip Gschwandtner <filip.gschwandtner@pantheon.tech>

plugins/linux/iptablesplugin/descriptor/rulechain.go

@@ -20,7 +20,6 @@ import (
 	"github.com/golang/protobuf/proto"
 	"github.com/pkg/errors"
 	"go.ligato.io/cn-infra/v2/logging"
-
 	kvs "go.ligato.io/vpp-agent/v3/plugins/kvscheduler/api"
 	ifdescriptor "go.ligato.io/vpp-agent/v3/plugins/linux/ifplugin/descriptor"
 	"go.ligato.io/vpp-agent/v3/plugins/linux/iptablesplugin/descriptor/adapter"
@@ -69,19 +68,22 @@ type RuleChainDescriptor struct {
 
 	// parallelization of the Retrieve operation
 	goRoutinesCnt int
+	// performance solution threshold
+	minRuleCountForPerfRuleAddition int
 }
 
 // NewRuleChainDescriptor creates a new instance of the iptables RuleChain descriptor.
 func NewRuleChainDescriptor(
 	scheduler kvs.KVScheduler, ipTablesHandler linuxcalls.IPTablesAPI, nsPlugin nsplugin.API,
-	log logging.PluginLogger, goRoutinesCnt int) *kvs.KVDescriptor {
+	log logging.PluginLogger, goRoutinesCnt int, minRuleCountForPerfRuleAddition int) *kvs.KVDescriptor {
 
 	descrCtx := &RuleChainDescriptor{
-		scheduler:       scheduler,
-		ipTablesHandler: ipTablesHandler,
-		nsPlugin:        nsPlugin,
-		goRoutinesCnt:   goRoutinesCnt,
-		log:             log.NewLogger("ipt-rulechain-descriptor"),
+		scheduler:                       scheduler,
+		ipTablesHandler:                 ipTablesHandler,
+		nsPlugin:                        nsPlugin,
+		goRoutinesCnt:                   goRoutinesCnt,
+		minRuleCountForPerfRuleAddition: minRuleCountForPerfRuleAddition,
+		log:                             log.NewLogger("ipt-rulechain-descriptor"),
 	}
 
 	typedDescr := &adapter.RuleChainDescriptor{
@@ -204,16 +206,13 @@ func (d *RuleChainDescriptor) Create(key string, rch *linux_iptables.RuleChain)
 	// wipe all rules in the chain that may have existed before
 	err = d.ipTablesHandler.DeleteAllRules(protocolType(rch), tableNameStr(rch), chainNameStr(rch))
 	if err != nil {
-		d.log.Warnf("Error by wiping iptables rules: %v", err)
+		return nil, errors.Errorf("Error by wiping iptables rules: %v", err)
 	}
 
 	// append all rules
-	for _, rule := range rch.Rules {
-		err := d.ipTablesHandler.AppendRule(protocolType(rch), tableNameStr(rch), chainNameStr(rch), rule)
-		if err != nil {
-			d.log.Errorf("Error by appending iptables rule: %v", err)
-			break
-		}
+	err = d.ipTablesHandler.AppendRules(protocolType(rch), tableNameStr(rch), chainNameStr(rch), rch.Rules...)
+	if err != nil {
+		return nil, errors.Errorf("Error by adding rules: %v", err)
 	}
 
 	return nil, err

plugins/linux/iptablesplugin/iptablesplugin.go

@@ -17,10 +17,10 @@
 package iptablesplugin
 
 import (
-	"go.ligato.io/cn-infra/v2/infra"
+	"math"
 
+	"go.ligato.io/cn-infra/v2/infra"
 	kvs "go.ligato.io/vpp-agent/v3/plugins/kvscheduler/api"
-
 	"go.ligato.io/vpp-agent/v3/plugins/linux/iptablesplugin/descriptor"
 	"go.ligato.io/vpp-agent/v3/plugins/linux/iptablesplugin/linuxcalls"
 	"go.ligato.io/vpp-agent/v3/plugins/linux/nsplugin"
@@ -30,6 +30,13 @@ const (
 	// by default, at most 10 go routines will split the configured rule chains
 	// to execute the Retrieve operation in parallel.
 	defaultGoRoutinesCnt = 10
+
+	// by default, no rules will be added by alternative performance strategy using
+	// iptables-save/modify data/iptables-store technique
+	// If this performance technique is needed, then the minimum rule limit should be lowered
+	// by configuration to some lower value (0 means that the permance strategy is
+	// always used)
+	defaultMinRuleCountForPerfRuleAddition = math.MaxInt32
 )
 
 // IPTablesPlugin configures Linux iptables rules.
@@ -56,6 +63,8 @@ type Deps struct {
 
 // Config holds the plugin configuration.
 type Config struct {
+	linuxcalls.HandlerConfig `json:"handler"`
+
 	Disabled      bool `json:"disabled"`
 	GoRoutinesCnt int  `json:"go-routines-count"`
 }
@@ -76,7 +85,7 @@ func (p *IPTablesPlugin) Init() error {
 
 	// init iptables handler
 	p.iptHandler = linuxcalls.NewIPTablesHandler()
-	err = p.iptHandler.Init()
+	err = p.iptHandler.Init(&config.HandlerConfig)
 	if err != nil && p.configFound {
 		// just warn here, iptables / ip6tables just may not be installed - will return
 		// an error by attempt to configure it
@@ -85,7 +94,7 @@ func (p *IPTablesPlugin) Init() error {
 
 	// init & register the descriptor
 	ruleChainDescriptor := descriptor.NewRuleChainDescriptor(
-		p.KVScheduler, p.iptHandler, p.NsPlugin, p.Log, config.GoRoutinesCnt)
+		p.KVScheduler, p.iptHandler, p.NsPlugin, p.Log, config.GoRoutinesCnt, config.MinRuleCountForPerfRuleAddition)
 
 	err = p.Deps.KVScheduler.RegisterKVDescriptor(ruleChainDescriptor)
 	if err != nil {
@@ -105,6 +114,9 @@ func (p *IPTablesPlugin) retrieveConfig() (*Config, error) {
 	config := &Config{
 		// default configuration
 		GoRoutinesCnt: defaultGoRoutinesCnt,
+		HandlerConfig: linuxcalls.HandlerConfig{
+			MinRuleCountForPerfRuleAddition: defaultMinRuleCountForPerfRuleAddition,
+		},
 	}
 	found, err := p.Cfg.LoadValue(config)
 	if !found {

plugins/linux/iptablesplugin/linux-iptablesplugin.conf

@@ -4,3 +4,12 @@ disabled: false
 # How many go routines (at most) will split configured network namespaces to execute
 # the Retrieve operation in parallel.
 go-routines-count: 10
+
+handler:
+    # Minimal rule count needed to perform alternative rule additions by creating RuleChain.
+    # Alternative method is based on exporting iptables data(iptables-save), adding rules
+    # into that exported data and import them back (iptables-restore). This can very efficient
+    # in case of filling many rules at once.
+    # By default off (rule count to activate alternative method is super high and therefore
+    # practically turned off)
+    min-rule-count-for-performance-rule-addition: 2147483647

plugins/linux/iptablesplugin/linuxcalls/iptables_api.go

@@ -26,7 +26,7 @@ const (
 // to manage linux iptables rules.
 type IPTablesAPI interface {
 	// Init initializes an iptables handler.
-	Init() error
+	Init(config *HandlerConfig) error
 
 	IPTablesAPIWrite
 	IPTablesAPIRead
@@ -47,6 +47,9 @@ type IPTablesAPIWrite interface {
 	// AppendRule appends a rule into the specified chain.
 	AppendRule(protocol L3Protocol, table, chain string, rule string) error
 
+	// AppendRules appends rules into the specified chain.
+	AppendRules(protocol L3Protocol, table, chain string, rules ...string) error
+
 	// DeleteRule deletes a rule from the specified chain.
 	DeleteRule(protocol L3Protocol, table, chain string, rule string) error
 
@@ -61,6 +64,11 @@ type IPTablesAPIRead interface {
 	ListRules(protocol L3Protocol, table, chain string) (rules []string, err error)
 }
 
+// HandlerConfig holds the IPTablesHandler related configuration.
+type HandlerConfig struct {
+	MinRuleCountForPerfRuleAddition int `json:"min-rule-count-for-performance-rule-addition"`
+}
+
 // NewIPTablesHandler creates new instance of iptables handler.
 func NewIPTablesHandler() *IPTablesHandler {
 	return &IPTablesHandler{}

plugins/linux/iptablesplugin/linuxcalls/iptables_linuxcalls.go

@@ -15,10 +15,13 @@
 package linuxcalls
 
 import (
+	"bytes"
 	"fmt"
+	"os/exec"
 	"strings"
 
 	"github.com/coreos/go-iptables/iptables"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -27,16 +30,23 @@ const (
 
 	// prefix of a "new chain" rule
 	newChainRulePrefix = "-N"
+
+	// command names
+	IPv4SaveCmd    string = "iptables-save"
+	IPv4RestoreCmd string = "iptables-restore"
+	IPv6RestoreCmd string = "ip6tables-restore"
+	IPv6SaveCmd    string = "ip6tables-save"
 )
 
 // IPTablesHandler is a handler for all operations on Linux iptables / ip6tables.
 type IPTablesHandler struct {
-	v4Handler *iptables.IPTables
-	v6Handler *iptables.IPTables
+	v4Handler                       *iptables.IPTables
+	v6Handler                       *iptables.IPTables
+	minRuleCountForPerfRuleAddition int
 }
 
 // Init initializes an iptables handler.
-func (h *IPTablesHandler) Init() error {
+func (h *IPTablesHandler) Init(config *HandlerConfig) error {
 	var err error
 
 	h.v4Handler, err = iptables.NewWithProtocol(iptables.ProtocolIPv4)
@@ -51,6 +61,8 @@ func (h *IPTablesHandler) Init() error {
 		// continue, ip6tables just may not be installed
 	}
 
+	h.minRuleCountForPerfRuleAddition = config.MinRuleCountForPerfRuleAddition
+
 	return err
 }
 
@@ -92,6 +104,51 @@ func (h *IPTablesHandler) AppendRule(protocol L3Protocol, table, chain string, r
 	return handler.Append(table, chain, ruleSlice[:]...)
 }
 
+// AppendRules appends rules into the specified chain.
+func (h *IPTablesHandler) AppendRules(protocol L3Protocol, table, chain string, rules ...string) error {
+	if len(rules) == 0 {
+		return nil // nothing to do
+	}
+
+	if len(rules) < h.minRuleCountForPerfRuleAddition { // use normal method of addition
+		for _, rule := range rules {
+			err := h.AppendRule(protocol, table, chain, rule)
+			if err != nil {
+				return errors.Errorf("Error by appending iptables rule: %v", err)
+			}
+		}
+	} else { // use performance solution (this makes performance difference with higher count of appended rules)
+		// export existing iptables data
+		data, err := h.saveTable(protocol, table, true)
+		if err != nil {
+			return errors.Errorf(": Can't export all rules due to: %v", err)
+		}
+
+		// add rules to exported data
+		insertPoint := bytes.Index(data, []byte("COMMIT"))
+		if insertPoint == -1 {
+			return errors.Errorf("Error by adding rules: Can't find COMMIT statement in iptables-save data")
+		}
+		var rulesSB strings.Builder
+		for _, rule := range rules {
+			rulesSB.WriteString(fmt.Sprintf("[0:0] -A %s %s\n", chain, rule))
+		}
+		insertData := []byte(rulesSB.String())
+		updatedData := make([]byte, len(data)+len(insertData))
+		copy(updatedData[:insertPoint], data[:insertPoint])
+		copy(updatedData[insertPoint:insertPoint+len(insertData)], insertData)
+		copy(updatedData[insertPoint+len(insertData):], data[insertPoint:])
+
+		// import modified data to linux
+		err = h.restoreTable(protocol, table, updatedData, true, true)
+		if err != nil {
+			return errors.Errorf("Error by adding rules: Can't restore modified iptables data due to: %v", err)
+		}
+	}
+
+	return nil
+}
+
 // DeleteRule deletes a rule from the specified chain.
 func (h *IPTablesHandler) DeleteRule(protocol L3Protocol, table, chain string, rule string) error {
 	handler, err := h.getHandler(protocol)
@@ -136,6 +193,55 @@ func (h *IPTablesHandler) ListRules(protocol L3Protocol, table, chain string) (r
 	return
 }
 
+// saveTable exports all data for given table in IPTable-save output format
+func (h *IPTablesHandler) saveTable(protocol L3Protocol, table string, exportCounters bool) ([]byte, error) {
+	// create command with arguments
+	saveCmd := IPv4SaveCmd
+	if protocol == ProtocolIPv6 {
+		saveCmd = IPv6SaveCmd
+	}
+	args := []string{"-t", table}
+	if exportCounters {
+		args = append(args, "-c")
+	}
+	cmd := exec.Command(saveCmd, args...)
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	// run command and extract result
+	err := cmd.Run()
+	if err != nil {
+		return nil, errors.Errorf("%s failed due to: %v (%s)", saveCmd, err, stderr.String())
+	}
+	return stdout.Bytes(), nil
+}
+
+// restoreTable import all data (in IPTable-save output format) for given table
+func (h *IPTablesHandler) restoreTable(protocol L3Protocol, table string, data []byte, flush bool, importCounters bool) error {
+	// create command with arguments
+	restoreCmd := IPv4RestoreCmd
+	if protocol == ProtocolIPv6 {
+		restoreCmd = IPv6RestoreCmd
+	}
+	args := []string{"-T", table}
+	if importCounters {
+		args = append(args, "-c")
+	}
+	if !flush {
+		args = append(args, "-n")
+	}
+	cmd := exec.Command(restoreCmd, args...)
+	cmd.Stdin = bytes.NewReader(data)
+
+	// run command and extract result
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return errors.Errorf("%s failed due to: %v (%s)", restoreCmd, err, string(output))
+	}
+	return nil
+}
+
 // getHandler returns the iptables handler for the given protocol.
 // returns an error if the requested handler is not initialized.
 func (h *IPTablesHandler) getHandler(protocol L3Protocol) (*iptables.IPTables, error) {