From 753bc8b312441a067bd369f4ba018e9547fb1881 Mon Sep 17 00:00:00 2001 From: fgschwan Date: Tue, 7 Apr 2020 15:32:04 +0200 Subject: [PATCH] =?UTF-8?q?perf:=20Performance=20enhancement=20for=20addin?= =?UTF-8?q?g=20many=20rules=20to=20Linux=20IP=E2=80=A6=20(#1644)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * added optional performance strategy for adding chain rule with many rules into linux netfilter (iptables) Signed-off-by: Filip Gschwandtner * moved logic of adding multiple rules into iptables handler Signed-off-by: Filip Gschwandtner --- .../iptablesplugin/descriptor/rulechain.go | 27 ++--- .../linux/iptablesplugin/iptablesplugin.go | 20 +++- .../iptablesplugin/linux-iptablesplugin.conf | 9 ++ .../iptablesplugin/linuxcalls/iptables_api.go | 10 +- .../linuxcalls/iptables_linuxcalls.go | 112 +++++++++++++++++- 5 files changed, 156 insertions(+), 22 deletions(-) diff --git a/plugins/linux/iptablesplugin/descriptor/rulechain.go b/plugins/linux/iptablesplugin/descriptor/rulechain.go index c50ece3d0a..63d321ed37 100644 --- a/plugins/linux/iptablesplugin/descriptor/rulechain.go +++ b/plugins/linux/iptablesplugin/descriptor/rulechain.go @@ -20,7 +20,6 @@ import ( "github.com/golang/protobuf/proto" "github.com/pkg/errors" "go.ligato.io/cn-infra/v2/logging" - kvs "go.ligato.io/vpp-agent/v3/plugins/kvscheduler/api" ifdescriptor "go.ligato.io/vpp-agent/v3/plugins/linux/ifplugin/descriptor" "go.ligato.io/vpp-agent/v3/plugins/linux/iptablesplugin/descriptor/adapter" @@ -69,19 +68,22 @@ type RuleChainDescriptor struct { // parallelization of the Retrieve operation goRoutinesCnt int + // performance solution threshold + minRuleCountForPerfRuleAddition int } // NewRuleChainDescriptor creates a new instance of the iptables RuleChain descriptor. func NewRuleChainDescriptor( scheduler kvs.KVScheduler, ipTablesHandler linuxcalls.IPTablesAPI, nsPlugin nsplugin.API, - log logging.PluginLogger, goRoutinesCnt int) *kvs.KVDescriptor { + log logging.PluginLogger, goRoutinesCnt int, minRuleCountForPerfRuleAddition int) *kvs.KVDescriptor { descrCtx := &RuleChainDescriptor{ - scheduler: scheduler, - ipTablesHandler: ipTablesHandler, - nsPlugin: nsPlugin, - goRoutinesCnt: goRoutinesCnt, - log: log.NewLogger("ipt-rulechain-descriptor"), + scheduler: scheduler, + ipTablesHandler: ipTablesHandler, + nsPlugin: nsPlugin, + goRoutinesCnt: goRoutinesCnt, + minRuleCountForPerfRuleAddition: minRuleCountForPerfRuleAddition, + log: log.NewLogger("ipt-rulechain-descriptor"), } typedDescr := &adapter.RuleChainDescriptor{ @@ -204,16 +206,13 @@ func (d *RuleChainDescriptor) Create(key string, rch *linux_iptables.RuleChain) // wipe all rules in the chain that may have existed before err = d.ipTablesHandler.DeleteAllRules(protocolType(rch), tableNameStr(rch), chainNameStr(rch)) if err != nil { - d.log.Warnf("Error by wiping iptables rules: %v", err) + return nil, errors.Errorf("Error by wiping iptables rules: %v", err) } // append all rules - for _, rule := range rch.Rules { - err := d.ipTablesHandler.AppendRule(protocolType(rch), tableNameStr(rch), chainNameStr(rch), rule) - if err != nil { - d.log.Errorf("Error by appending iptables rule: %v", err) - break - } + err = d.ipTablesHandler.AppendRules(protocolType(rch), tableNameStr(rch), chainNameStr(rch), rch.Rules...) + if err != nil { + return nil, errors.Errorf("Error by adding rules: %v", err) } return nil, err diff --git a/plugins/linux/iptablesplugin/iptablesplugin.go b/plugins/linux/iptablesplugin/iptablesplugin.go index 26624d817d..197f6f2859 100644 --- a/plugins/linux/iptablesplugin/iptablesplugin.go +++ b/plugins/linux/iptablesplugin/iptablesplugin.go @@ -17,10 +17,10 @@ package iptablesplugin import ( - "go.ligato.io/cn-infra/v2/infra" + "math" + "go.ligato.io/cn-infra/v2/infra" kvs "go.ligato.io/vpp-agent/v3/plugins/kvscheduler/api" - "go.ligato.io/vpp-agent/v3/plugins/linux/iptablesplugin/descriptor" "go.ligato.io/vpp-agent/v3/plugins/linux/iptablesplugin/linuxcalls" "go.ligato.io/vpp-agent/v3/plugins/linux/nsplugin" @@ -30,6 +30,13 @@ const ( // by default, at most 10 go routines will split the configured rule chains // to execute the Retrieve operation in parallel. defaultGoRoutinesCnt = 10 + + // by default, no rules will be added by alternative performance strategy using + // iptables-save/modify data/iptables-store technique + // If this performance technique is needed, then the minimum rule limit should be lowered + // by configuration to some lower value (0 means that the permance strategy is + // always used) + defaultMinRuleCountForPerfRuleAddition = math.MaxInt32 ) // IPTablesPlugin configures Linux iptables rules. @@ -56,6 +63,8 @@ type Deps struct { // Config holds the plugin configuration. type Config struct { + linuxcalls.HandlerConfig `json:"handler"` + Disabled bool `json:"disabled"` GoRoutinesCnt int `json:"go-routines-count"` } @@ -76,7 +85,7 @@ func (p *IPTablesPlugin) Init() error { // init iptables handler p.iptHandler = linuxcalls.NewIPTablesHandler() - err = p.iptHandler.Init() + err = p.iptHandler.Init(&config.HandlerConfig) if err != nil && p.configFound { // just warn here, iptables / ip6tables just may not be installed - will return // an error by attempt to configure it @@ -85,7 +94,7 @@ func (p *IPTablesPlugin) Init() error { // init & register the descriptor ruleChainDescriptor := descriptor.NewRuleChainDescriptor( - p.KVScheduler, p.iptHandler, p.NsPlugin, p.Log, config.GoRoutinesCnt) + p.KVScheduler, p.iptHandler, p.NsPlugin, p.Log, config.GoRoutinesCnt, config.MinRuleCountForPerfRuleAddition) err = p.Deps.KVScheduler.RegisterKVDescriptor(ruleChainDescriptor) if err != nil { @@ -105,6 +114,9 @@ func (p *IPTablesPlugin) retrieveConfig() (*Config, error) { config := &Config{ // default configuration GoRoutinesCnt: defaultGoRoutinesCnt, + HandlerConfig: linuxcalls.HandlerConfig{ + MinRuleCountForPerfRuleAddition: defaultMinRuleCountForPerfRuleAddition, + }, } found, err := p.Cfg.LoadValue(config) if !found { diff --git a/plugins/linux/iptablesplugin/linux-iptablesplugin.conf b/plugins/linux/iptablesplugin/linux-iptablesplugin.conf index 40712e3dc4..6c28b64bc4 100644 --- a/plugins/linux/iptablesplugin/linux-iptablesplugin.conf +++ b/plugins/linux/iptablesplugin/linux-iptablesplugin.conf @@ -4,3 +4,12 @@ disabled: false # How many go routines (at most) will split configured network namespaces to execute # the Retrieve operation in parallel. go-routines-count: 10 + +handler: + # Minimal rule count needed to perform alternative rule additions by creating RuleChain. + # Alternative method is based on exporting iptables data(iptables-save), adding rules + # into that exported data and import them back (iptables-restore). This can very efficient + # in case of filling many rules at once. + # By default off (rule count to activate alternative method is super high and therefore + # practically turned off) + min-rule-count-for-performance-rule-addition: 2147483647 diff --git a/plugins/linux/iptablesplugin/linuxcalls/iptables_api.go b/plugins/linux/iptablesplugin/linuxcalls/iptables_api.go index 8206556ed3..2432aa1b15 100644 --- a/plugins/linux/iptablesplugin/linuxcalls/iptables_api.go +++ b/plugins/linux/iptablesplugin/linuxcalls/iptables_api.go @@ -26,7 +26,7 @@ const ( // to manage linux iptables rules. type IPTablesAPI interface { // Init initializes an iptables handler. - Init() error + Init(config *HandlerConfig) error IPTablesAPIWrite IPTablesAPIRead @@ -47,6 +47,9 @@ type IPTablesAPIWrite interface { // AppendRule appends a rule into the specified chain. AppendRule(protocol L3Protocol, table, chain string, rule string) error + // AppendRules appends rules into the specified chain. + AppendRules(protocol L3Protocol, table, chain string, rules ...string) error + // DeleteRule deletes a rule from the specified chain. DeleteRule(protocol L3Protocol, table, chain string, rule string) error @@ -61,6 +64,11 @@ type IPTablesAPIRead interface { ListRules(protocol L3Protocol, table, chain string) (rules []string, err error) } +// HandlerConfig holds the IPTablesHandler related configuration. +type HandlerConfig struct { + MinRuleCountForPerfRuleAddition int `json:"min-rule-count-for-performance-rule-addition"` +} + // NewIPTablesHandler creates new instance of iptables handler. func NewIPTablesHandler() *IPTablesHandler { return &IPTablesHandler{} diff --git a/plugins/linux/iptablesplugin/linuxcalls/iptables_linuxcalls.go b/plugins/linux/iptablesplugin/linuxcalls/iptables_linuxcalls.go index 83a975026c..0fefb3efc1 100644 --- a/plugins/linux/iptablesplugin/linuxcalls/iptables_linuxcalls.go +++ b/plugins/linux/iptablesplugin/linuxcalls/iptables_linuxcalls.go @@ -15,10 +15,13 @@ package linuxcalls import ( + "bytes" "fmt" + "os/exec" "strings" "github.com/coreos/go-iptables/iptables" + "github.com/pkg/errors" ) const ( @@ -27,16 +30,23 @@ const ( // prefix of a "new chain" rule newChainRulePrefix = "-N" + + // command names + IPv4SaveCmd string = "iptables-save" + IPv4RestoreCmd string = "iptables-restore" + IPv6RestoreCmd string = "ip6tables-restore" + IPv6SaveCmd string = "ip6tables-save" ) // IPTablesHandler is a handler for all operations on Linux iptables / ip6tables. type IPTablesHandler struct { - v4Handler *iptables.IPTables - v6Handler *iptables.IPTables + v4Handler *iptables.IPTables + v6Handler *iptables.IPTables + minRuleCountForPerfRuleAddition int } // Init initializes an iptables handler. -func (h *IPTablesHandler) Init() error { +func (h *IPTablesHandler) Init(config *HandlerConfig) error { var err error h.v4Handler, err = iptables.NewWithProtocol(iptables.ProtocolIPv4) @@ -51,6 +61,8 @@ func (h *IPTablesHandler) Init() error { // continue, ip6tables just may not be installed } + h.minRuleCountForPerfRuleAddition = config.MinRuleCountForPerfRuleAddition + return err } @@ -92,6 +104,51 @@ func (h *IPTablesHandler) AppendRule(protocol L3Protocol, table, chain string, r return handler.Append(table, chain, ruleSlice[:]...) } +// AppendRules appends rules into the specified chain. +func (h *IPTablesHandler) AppendRules(protocol L3Protocol, table, chain string, rules ...string) error { + if len(rules) == 0 { + return nil // nothing to do + } + + if len(rules) < h.minRuleCountForPerfRuleAddition { // use normal method of addition + for _, rule := range rules { + err := h.AppendRule(protocol, table, chain, rule) + if err != nil { + return errors.Errorf("Error by appending iptables rule: %v", err) + } + } + } else { // use performance solution (this makes performance difference with higher count of appended rules) + // export existing iptables data + data, err := h.saveTable(protocol, table, true) + if err != nil { + return errors.Errorf(": Can't export all rules due to: %v", err) + } + + // add rules to exported data + insertPoint := bytes.Index(data, []byte("COMMIT")) + if insertPoint == -1 { + return errors.Errorf("Error by adding rules: Can't find COMMIT statement in iptables-save data") + } + var rulesSB strings.Builder + for _, rule := range rules { + rulesSB.WriteString(fmt.Sprintf("[0:0] -A %s %s\n", chain, rule)) + } + insertData := []byte(rulesSB.String()) + updatedData := make([]byte, len(data)+len(insertData)) + copy(updatedData[:insertPoint], data[:insertPoint]) + copy(updatedData[insertPoint:insertPoint+len(insertData)], insertData) + copy(updatedData[insertPoint+len(insertData):], data[insertPoint:]) + + // import modified data to linux + err = h.restoreTable(protocol, table, updatedData, true, true) + if err != nil { + return errors.Errorf("Error by adding rules: Can't restore modified iptables data due to: %v", err) + } + } + + return nil +} + // DeleteRule deletes a rule from the specified chain. func (h *IPTablesHandler) DeleteRule(protocol L3Protocol, table, chain string, rule string) error { handler, err := h.getHandler(protocol) @@ -136,6 +193,55 @@ func (h *IPTablesHandler) ListRules(protocol L3Protocol, table, chain string) (r return } +// saveTable exports all data for given table in IPTable-save output format +func (h *IPTablesHandler) saveTable(protocol L3Protocol, table string, exportCounters bool) ([]byte, error) { + // create command with arguments + saveCmd := IPv4SaveCmd + if protocol == ProtocolIPv6 { + saveCmd = IPv6SaveCmd + } + args := []string{"-t", table} + if exportCounters { + args = append(args, "-c") + } + cmd := exec.Command(saveCmd, args...) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + // run command and extract result + err := cmd.Run() + if err != nil { + return nil, errors.Errorf("%s failed due to: %v (%s)", saveCmd, err, stderr.String()) + } + return stdout.Bytes(), nil +} + +// restoreTable import all data (in IPTable-save output format) for given table +func (h *IPTablesHandler) restoreTable(protocol L3Protocol, table string, data []byte, flush bool, importCounters bool) error { + // create command with arguments + restoreCmd := IPv4RestoreCmd + if protocol == ProtocolIPv6 { + restoreCmd = IPv6RestoreCmd + } + args := []string{"-T", table} + if importCounters { + args = append(args, "-c") + } + if !flush { + args = append(args, "-n") + } + cmd := exec.Command(restoreCmd, args...) + cmd.Stdin = bytes.NewReader(data) + + // run command and extract result + output, err := cmd.CombinedOutput() + if err != nil { + return errors.Errorf("%s failed due to: %v (%s)", restoreCmd, err, string(output)) + } + return nil +} + // getHandler returns the iptables handler for the given protocol. // returns an error if the requested handler is not initialized. func (h *IPTablesHandler) getHandler(protocol L3Protocol) (*iptables.IPTables, error) {