perf: Performance enhancement for adding many rules to Linux IPtables chain

plugins/linux/iptablesplugin/descriptor/rulechain.go

@@ -15,12 +15,13 @@
 package descriptor
 
 import (
+	"bytes"
+	"fmt"
 	"strings"
 
 	"github.com/golang/protobuf/proto"
 	"github.com/pkg/errors"
 	"go.ligato.io/cn-infra/v2/logging"
-
 	kvs "go.ligato.io/vpp-agent/v3/plugins/kvscheduler/api"
 	ifdescriptor "go.ligato.io/vpp-agent/v3/plugins/linux/ifplugin/descriptor"
 	"go.ligato.io/vpp-agent/v3/plugins/linux/iptablesplugin/descriptor/adapter"
@@ -69,19 +70,22 @@ type RuleChainDescriptor struct {
 
 	// parallelization of the Retrieve operation
 	goRoutinesCnt int
+	// performance solution threshold
+	minRuleCountForPerfRuleAddition int
 }
 
 // NewRuleChainDescriptor creates a new instance of the iptables RuleChain descriptor.
 func NewRuleChainDescriptor(
 	scheduler kvs.KVScheduler, ipTablesHandler linuxcalls.IPTablesAPI, nsPlugin nsplugin.API,
-	log logging.PluginLogger, goRoutinesCnt int) *kvs.KVDescriptor {
+	log logging.PluginLogger, goRoutinesCnt int, minRuleCountForPerfRuleAddition int) *kvs.KVDescriptor {
 
 	descrCtx := &RuleChainDescriptor{
-		scheduler:       scheduler,
-		ipTablesHandler: ipTablesHandler,
-		nsPlugin:        nsPlugin,
-		goRoutinesCnt:   goRoutinesCnt,
-		log:             log.NewLogger("ipt-rulechain-descriptor"),
+		scheduler:                       scheduler,
+		ipTablesHandler:                 ipTablesHandler,
+		nsPlugin:                        nsPlugin,
+		goRoutinesCnt:                   goRoutinesCnt,
+		minRuleCountForPerfRuleAddition: minRuleCountForPerfRuleAddition,
+		log:                             log.NewLogger("ipt-rulechain-descriptor"),
 	}
 
 	typedDescr := &adapter.RuleChainDescriptor{
@@ -208,11 +212,43 @@ func (d *RuleChainDescriptor) Create(key string, rch *linux_iptables.RuleChain)
 	}
 
 	// append all rules
-	for _, rule := range rch.Rules {
-		err := d.ipTablesHandler.AppendRule(protocolType(rch), tableNameStr(rch), chainNameStr(rch), rule)
-		if err != nil {
-			d.log.Errorf("Error by appending iptables rule: %v", err)
-			break
+	if len(rch.Rules) > 0 {
+		if len(rch.Rules) < d.minRuleCountForPerfRuleAddition { // use normal method of addition
+			for _, rule := range rch.Rules {
+				err := d.ipTablesHandler.AppendRule(protocolType(rch), tableNameStr(rch), chainNameStr(rch), rule)
+				if err != nil {
+					d.log.Errorf("Error by appending iptables rule: %v", err)
+					break
+				}
+			}
+		} else { // use performance solution (this makes performance difference with higher count of appended rules)
+			// export existing iptables data
+			data, err := d.ipTablesHandler.SaveTable(protocolType(rch), tableNameStr(rch), true)
+			if err != nil {
+				return nil, errors.Errorf("Error by adding rules: Can't export all rules due to: %v", err)
+			}
+
+			// add rules to exported data
+			insertPoint := bytes.Index(data, []byte("COMMIT"))
+			if insertPoint == -1 {
+				return nil, errors.Errorf("Error by adding rules: Can't find COMMIT statement in iptables-save data")
+			}
+			var rules strings.Builder
+			chain := chainNameStr(rch)
+			for _, rule := range rch.Rules {
+				rules.WriteString(fmt.Sprintf("[0:0] -A %s %s\n", chain, rule))
+			}
+			insertData := []byte(rules.String())
+			updatedData := make([]byte, len(data)+len(insertData))
+			copy(updatedData[:insertPoint], data[:insertPoint])
+			copy(updatedData[insertPoint:insertPoint+len(insertData)], insertData)
+			copy(updatedData[insertPoint+len(insertData):], data[insertPoint:])
+
+			// import modified data to linux
+			err = d.ipTablesHandler.RestoreTable(protocolType(rch), tableNameStr(rch), updatedData, true, true)
+			if err != nil {
+				return nil, errors.Errorf("Error by adding rules: Can't restore modified iptables data due to: %v", err)
+			}
 		}
 	}
 

plugins/linux/iptablesplugin/iptablesplugin.go

@@ -17,10 +17,10 @@
 package iptablesplugin
 
 import (
-	"go.ligato.io/cn-infra/v2/infra"
+	"math"
 
+	"go.ligato.io/cn-infra/v2/infra"
 	kvs "go.ligato.io/vpp-agent/v3/plugins/kvscheduler/api"
-
 	"go.ligato.io/vpp-agent/v3/plugins/linux/iptablesplugin/descriptor"
 	"go.ligato.io/vpp-agent/v3/plugins/linux/iptablesplugin/linuxcalls"
 	"go.ligato.io/vpp-agent/v3/plugins/linux/nsplugin"
@@ -30,6 +30,13 @@ const (
 	// by default, at most 10 go routines will split the configured rule chains
 	// to execute the Retrieve operation in parallel.
 	defaultGoRoutinesCnt = 10
+
+	// by default, no rules will be added by alternative performance strategy using
+	// iptables-save/modify data/iptables-store technique
+	// If this performance technique is needed, then the minimum rule limit should be lowered
+	// by configuration to some lower value (0 means that the permance strategy is
+	// always used)
+	defaultMinRuleCountForPerfRuleAddition = math.MaxInt32
 )
 
 // IPTablesPlugin configures Linux iptables rules.
@@ -56,8 +63,9 @@ type Deps struct {
 
 // Config holds the plugin configuration.
 type Config struct {
-	Disabled      bool `json:"disabled"`
-	GoRoutinesCnt int  `json:"go-routines-count"`
+	Disabled                        bool `json:"disabled"`
+	GoRoutinesCnt                   int  `json:"go-routines-count"`
+	MinRuleCountForPerfRuleAddition int  `json:"min-rule-count-for-performance-rule-addition"`
 }
 
 // Init initializes and registers descriptors and handlers for Linux iptables rules.
@@ -85,7 +93,7 @@ func (p *IPTablesPlugin) Init() error {
 
 	// init & register the descriptor
 	ruleChainDescriptor := descriptor.NewRuleChainDescriptor(
-		p.KVScheduler, p.iptHandler, p.NsPlugin, p.Log, config.GoRoutinesCnt)
+		p.KVScheduler, p.iptHandler, p.NsPlugin, p.Log, config.GoRoutinesCnt, config.MinRuleCountForPerfRuleAddition)
 
 	err = p.Deps.KVScheduler.RegisterKVDescriptor(ruleChainDescriptor)
 	if err != nil {
@@ -104,7 +112,8 @@ func (p *IPTablesPlugin) Close() error {
 func (p *IPTablesPlugin) retrieveConfig() (*Config, error) {
 	config := &Config{
 		// default configuration
-		GoRoutinesCnt: defaultGoRoutinesCnt,
+		GoRoutinesCnt:                   defaultGoRoutinesCnt,
+		MinRuleCountForPerfRuleAddition: defaultMinRuleCountForPerfRuleAddition,
 	}
 	found, err := p.Cfg.LoadValue(config)
 	if !found {

plugins/linux/iptablesplugin/linux-iptablesplugin.conf

@@ -4,3 +4,11 @@ disabled: false
 # How many go routines (at most) will split configured network namespaces to execute
 # the Retrieve operation in parallel.
 go-routines-count: 10
+
+# Minimal rule count needed to perform alternative rule additions by creating RuleChain.
+# Alternative method is based on exporting iptables data(iptables-save), adding rules
+# into that exported data and import them back (iptables-restore). This can very efficient
+# in case of filling many rules at once.
+# By default off (rule count to activate alternative method is super high and therefore
+# practically turned off)
+min-rule-count-for-performance-rule-addition: 2147483647

plugins/linux/iptablesplugin/linuxcalls/iptables_api.go

@@ -52,13 +52,19 @@ type IPTablesAPIWrite interface {
 
 	// DeleteAllRules deletes all rules within the specified chain.
 	DeleteAllRules(protocol L3Protocol, table, chain string) error
+
+	// RestoreTable import all data (in IPTable-save output format) for given table
+	RestoreTable(protocol L3Protocol, table string, data []byte, flush bool, importCounters bool) error
 }
 
 // IPTablesAPIRead interface covers read methods inside linux calls package
 // needed to manage linux iptables rules.
 type IPTablesAPIRead interface {
 	// ListRules lists all rules within the specified chain.
 	ListRules(protocol L3Protocol, table, chain string) (rules []string, err error)
+
+	// SaveTable exports all data for given table in IPTable-save output format
+	SaveTable(protocol L3Protocol, table string, exportCounters bool) ([]byte, error)
 }
 
 // NewIPTablesHandler creates new instance of iptables handler.

plugins/linux/iptablesplugin/linuxcalls/iptables_linuxcalls.go

@@ -15,10 +15,13 @@
 package linuxcalls
 
 import (
+	"bytes"
 	"fmt"
+	"os/exec"
 	"strings"
 
 	"github.com/coreos/go-iptables/iptables"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -27,6 +30,12 @@ const (
 
 	// prefix of a "new chain" rule
 	newChainRulePrefix = "-N"
+
+	// command names
+	IPv4SaveCmd    string = "iptables-save"
+	IPv4RestoreCmd string = "iptables-restore"
+	IPv6RestoreCmd string = "ip6tables-restore"
+	IPv6SaveCmd    string = "ip6tables-save"
 )
 
 // IPTablesHandler is a handler for all operations on Linux iptables / ip6tables.
@@ -136,6 +145,55 @@ func (h *IPTablesHandler) ListRules(protocol L3Protocol, table, chain string) (r
 	return
 }
 
+// SaveTable exports all data for given table in IPTable-save output format
+func (h *IPTablesHandler) SaveTable(protocol L3Protocol, table string, exportCounters bool) ([]byte, error) {
+	// create command with arguments
+	saveCmd := IPv4SaveCmd
+	if protocol == ProtocolIPv6 {
+		saveCmd = IPv6SaveCmd
+	}
+	args := []string{"-t", table}
+	if exportCounters {
+		args = append(args, "-c")
+	}
+	cmd := exec.Command(saveCmd, args...)
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	// run command and extract result
+	err := cmd.Run()
+	if err != nil {
+		return nil, errors.Errorf("%s failed due to: %v (%s)", saveCmd, err, stderr.String())
+	}
+	return stdout.Bytes(), nil
+}
+
+// RestoreTable import all data (in IPTable-save output format) for given table
+func (h *IPTablesHandler) RestoreTable(protocol L3Protocol, table string, data []byte, flush bool, importCounters bool) error {
+	// create command with arguments
+	restoreCmd := IPv4RestoreCmd
+	if protocol == ProtocolIPv6 {
+		restoreCmd = IPv6RestoreCmd
+	}
+	args := []string{"-T", table}
+	if importCounters {
+		args = append(args, "-c")
+	}
+	if !flush {
+		args = append(args, "-n")
+	}
+	cmd := exec.Command(restoreCmd, args...)
+	cmd.Stdin = bytes.NewReader(data)
+
+	// run command and extract result
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return errors.Errorf("%s failed due to: %v (%s)", restoreCmd, err, string(output))
+	}
+	return nil
+}
+
 // getHandler returns the iptables handler for the given protocol.
 // returns an error if the requested handler is not initialized.
 func (h *IPTablesHandler) getHandler(protocol L3Protocol) (*iptables.IPTables, error) {