Add IPv6/dual-stack support

.github/workflows/cni-plugin-integration.yml

@@ -12,33 +12,20 @@ on:
       - justfile*
 
 jobs:
-  cni-flannel-test:
-    continue-on-error: true
-    timeout-minutes: 15
-    runs-on: ubuntu-latest
-    steps:
-      - uses: linkerd/dev/actions/setup-tools@v43
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
-      - name: Run CNI integration tests
-        run: just cni-plugin-test-integration-flannel
-  cni-calico-test:
-    continue-on-error: true
-    timeout-minutes: 15
-    runs-on: ubuntu-latest
-    steps:
-      - uses: linkerd/dev/actions/setup-tools@v43
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
-      - name: Run CNI integration tests
-        run: just cni-plugin-test-integration-calico
-  cni-cilium-test:
-    continue-on-error: true
+  cni-test:
+    strategy:
+      matrix:
+        cni: [flannel, calico, cilium]
+        iptables-mode: [legacy, nft]
     timeout-minutes: 15
     runs-on: ubuntu-latest
     steps:
       - uses: linkerd/dev/actions/setup-tools@v43
       - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
       - name: Run CNI integration tests
-        run: just cni-plugin-test-integration-cilium
+        env:
+          IPTABLES_MODE: ${{ matrix.iptables-mode }}
+        run: just cni-plugin-test-integration-${{ matrix.cni }}
   ordering-test:
     continue-on-error: true
     timeout-minutes: 15

cni-plugin/integration/manifests/calico/linkerd-cni.yaml

@@ -80,7 +80,9 @@ data:
         "ports-to-redirect": [],
         "inbound-ports-to-ignore": ["4191","4190"],
         "simulate": false,
-        "use-wait-flag": false
+        "use-wait-flag": false,
+        "iptables-mode": "$IPTABLES_MODE",
+        "ipv6": true
       }
     }
 ---

cni-plugin/integration/manifests/cilium/linkerd-cni.yaml

@@ -80,7 +80,9 @@ data:
         "ports-to-redirect": [],
         "inbound-ports-to-ignore": ["4191","4190"],
         "simulate": false,
-        "use-wait-flag": false
+        "use-wait-flag": false,
+        "iptables-mode": "$IPTABLES_MODE",
+        "ipv6": true
       }
     }
 ---

cni-plugin/integration/manifests/flannel/linkerd-cni.yaml

@@ -84,7 +84,9 @@ data:
         "ports-to-redirect": [],
         "inbound-ports-to-ignore": ["4191","4190"],
         "simulate": false,
-        "use-wait-flag": false
+        "use-wait-flag": false,
+        "iptables-mode": "$IPTABLES_MODE",
+        "ipv6": true
       }
     }
 ---

cni-plugin/integration/run.sh

@@ -4,9 +4,8 @@ set -euxo pipefail
 
 cd "${BASH_SOURCE[0]%/*}"
 
-# Integration tests to run. Scenario is passed in as an environment variable.
-# Default is 'flannel'
 SCENARIO=${CNI_TEST_SCENARIO:-flannel}
+IPTABLES_MODE=${IPTABLES_MODE:-legacy}
 
 # Run kubectl with the correct context.
 function k() {
@@ -25,7 +24,10 @@ function create_test_lab() {
     # can enable a testing matrix?
     # Apply all files in scenario directory. For non-flannel CNIs, this will
     # include the CNI manifest itself.
-    k apply -f "manifests/$SCENARIO/"
+    for f in ./manifests/"$SCENARIO"/*.yaml
+    do
+      envsubst < "$f" | k apply -f -
+    done
 }
 
 function cleanup() {

cni-plugin/integration/testutil/test_util.go

@@ -49,8 +49,11 @@ type ProxyInit struct {
 	PortsToRedirect       []int    `json:"ports-to-redirect"`
 	InboundPortsToIgnore  []string `json:"inbound-ports-to-ignore"`
 	OutboundPortsToIgnore []string `json:"outbound-ports-to-ignore"`
+	SubnetsToIgnore       []string `json:"subnets-to-ignore"`
 	Simulate              bool     `json:"simulate"`
 	UseWaitFlag           bool     `json:"use-wait-flag"`
+	IPTablesMode          string   `json:"iptables-mode"`
+	IPv6                  bool     `json:"ipv6"`
 }
 
 // LinkerdPlugin is what we use for CNI configuration in the plugins section

cni-plugin/main.go

@@ -52,6 +52,8 @@ type ProxyInit struct {
 	SubnetsToIgnore       []string `json:"subnets-to-ignore"`
 	Simulate              bool     `json:"simulate"`
 	UseWaitFlag           bool     `json:"use-wait-flag"`
+	IPTablesMode          string   `json:"iptables-mode"`
+	IPv6                  bool     `json:"ipv6"`
 }
 
 // Kubernetes a K8s specific struct to hold config
@@ -219,8 +221,8 @@ func cmdAdd(args *skel.CmdArgs) error {
 				SimulateOnly:          conf.ProxyInit.Simulate,
 				NetNs:                 args.Netns,
 				UseWaitFlag:           conf.ProxyInit.UseWaitFlag,
-				FirewallBinPath:       "iptables-legacy",
-				FirewallSaveBinPath:   "iptables-legacy-save",
+				IPTablesMode:          conf.ProxyInit.IPTablesMode,
+				IPv6:                  conf.ProxyInit.IPv6,
 			}
 
 			// Check if there are any overridden ports to be skipped
@@ -292,17 +294,24 @@ func cmdAdd(args *skel.CmdArgs) error {
 				options.OutboundPortsToIgnore = append(options.OutboundPortsToIgnore, skippedPorts...)
 			}
 
-			firewallConfiguration, err := cmd.BuildFirewallConfiguration(&options)
-			if err != nil {
-				logEntry.Errorf("linkerd-cni: could not create a Firewall Configuration from the options: %v", options)
-				return err
+			// This ensures BC against linkerd2-cni older versions not yet passing this flag
+			if options.IPTablesMode == "" {
+				options.IPTablesMode = cmd.IPTablesModeLegacy
 			}
 
-			err = iptables.ConfigureFirewall(*firewallConfiguration)
-			if err != nil {
-				logEntry.Errorf("linkerd-cni: could not configure firewall: %s", err)
+			// always trigger the IPv4 rules
+			optIPv4 := options
+			optIPv4.IPv6 = false
+			if err := buildAndConfigure(logEntry, &optIPv4); err != nil {
 				return err
 			}
+
+			// trigger the IPv6 rules
+			if options.IPv6 {
+				if err := buildAndConfigure(logEntry, &options); err != nil {
+					return err
+				}
+			}
 		} else {
 			if containsInitContainer {
 				logEntry.Debug("linkerd-cni: linkerd-init initContainer is present, skipping.")
@@ -353,6 +362,24 @@ func getAPIServerPorts(ctx context.Context, api *kubernetes.Clientset) ([]string
 	return ports, nil
 }
 
+func buildAndConfigure(logEntry *logrus.Entry, options *cmd.RootOptions) error {
+	firewallConfiguration, err := cmd.BuildFirewallConfiguration(options)
+	if err != nil {
+		logEntry.Errorf("linkerd-cni: could not create a Firewall Configuration from the options: %v", options)
+		return err
+	}
+
+	err = iptables.ConfigureFirewall(*firewallConfiguration)
+	// We couldn't find a robust way of checking IPv6 support besides trying to just call ip6tables-save.
+	// If IPv4 rules worked but not IPv6, let's not fail the container (the actual problem will get logged).
+	if !options.IPv6 && err != nil {
+		logEntry.Errorf("linkerd-cni: could not configure firewall: %s", err)
+		return err
+	}
+
+	return nil
+}
+
 func getAnnotationOverride(ctx context.Context, api *kubernetes.Clientset, pod *v1.Pod, key string) (string, error) {
 	// Check if the annotation is present on the pod
 	if override := pod.GetObjectMeta().GetAnnotations()[key]; override != "" {

justfile

@@ -224,6 +224,7 @@ _cni-plugin-setup-cilium:
     echo "Mounted /sys/fs/bpf to cilium-test-server cluster"
     helm repo add cilium https://helm.cilium.io/
     helm install cilium cilium/cilium --version 1.13.0 \
+        --kube-context k3d-l5d-cilium-test \
         --namespace kube-system \
         --set kubeProxyReplacement=partial \
         --set hostServices.enabled=false \

proxy-init/cmd/root.go

@@ -13,6 +13,22 @@ import (
 	"github.com/linkerd/linkerd2-proxy-init/internal/util"
 )
 
+const (
+	// IPTablesModeLegacy signals the usage of the iptables-legacy commands
+	IPTablesModeLegacy = "legacy"
+	// IPTablesModeNFT signals the usage of the iptables-nft commands
+	IPTablesModeNFT = "nft"
+
+	cmdLegacy         = "iptables-legacy"
+	cmdLegacySave     = "iptables-legacy-save"
+	cmdLegacyIPv6     = "ip6tables-legacy"
+	cmdLegacyIPv6Save = "ip6tables-legacy-save"
+	cmdNFT            = "iptables-nft"
+	cmdNFTSave        = "iptables-nft-save"
+	cmdNFTIPv6        = "ip6tables-nft"
+	cmdNFTIPv6Save    = "ip6tables-nft-save"
+)
+
 // RootOptions provides the information that will be used to build a firewall configuration.
 type RootOptions struct {
 	IncomingProxyPort     int
@@ -30,6 +46,8 @@ type RootOptions struct {
 	LogLevel              string
 	FirewallBinPath       string
 	FirewallSaveBinPath   string
+	IPTablesMode          string
+	IPv6                  bool
 }
 
 func newRootOptions() *RootOptions {
@@ -47,8 +65,10 @@ func newRootOptions() *RootOptions {
 		TimeoutCloseWaitSecs:  0,
 		LogFormat:             "plain",
 		LogLevel:              "info",
-		FirewallBinPath:       "iptables-legacy",
-		FirewallSaveBinPath:   "iptables-legacy-save",
+		FirewallBinPath:       "",
+		FirewallSaveBinPath:   "",
+		IPTablesMode:          "",
+		IPv6:                  true,
 	}
 }
 
@@ -61,7 +81,7 @@ func NewRootCmd() *cobra.Command {
 		Use:   "proxy-init",
 		Short: "proxy-init adds a Kubernetes pod to the Linkerd service mesh",
 		Long:  "proxy-init adds a Kubernetes pod to the Linkerd service mesh.",
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(_ *cobra.Command, _ []string) error {
 
 			if options.TimeoutCloseWaitSecs != 0 {
 				sysctl := exec.Command("sysctl", "-w",
@@ -75,16 +95,39 @@ func NewRootCmd() *cobra.Command {
 				log.Info(string(out))
 			}
 
-			config, err := BuildFirewallConfiguration(options)
+			log.SetFormatter(getFormatter(options.LogFormat))
+			err := setLogLevel(options.LogLevel)
 			if err != nil {
 				return err
 			}
-			log.SetFormatter(getFormatter(options.LogFormat))
-			err = setLogLevel(options.LogLevel)
+
+			// always trigger the IPv4 rules
+			optIPv4 := *options
+			optIPv4.IPv6 = false
+			config, err := BuildFirewallConfiguration(&optIPv4)
 			if err != nil {
 				return err
 			}
-			return iptables.ConfigureFirewall(*config)
+
+			if err = iptables.ConfigureFirewall(*config); err != nil {
+				return err
+			}
+
+			if !options.IPv6 {
+				return nil
+			}
+
+			// trigger the IPv6 rules
+			config, err = BuildFirewallConfiguration(options)
+			if err != nil {
+				return err
+			}
+
+			// We couldn't find a robust way of checking IPv6 support besides trying to just call ip6tables-save.
+			// If IPv4 rules worked but not IPv6, let's not fail the container (the actual problem will get logged).
+			_ = iptables.ConfigureFirewall(*config)
+
+			return nil
 		},
 	}
 
@@ -101,13 +144,32 @@ func NewRootCmd() *cobra.Command {
 	cmd.PersistentFlags().IntVar(&options.TimeoutCloseWaitSecs, "timeout-close-wait-secs", options.TimeoutCloseWaitSecs, "Sets nf_conntrack_tcp_timeout_close_wait")
 	cmd.PersistentFlags().StringVar(&options.LogFormat, "log-format", options.LogFormat, "Configure log format ('plain' or 'json')")
 	cmd.PersistentFlags().StringVar(&options.LogLevel, "log-level", options.LogLevel, "Configure log level")
+	cmd.PersistentFlags().StringVar(&options.IPTablesMode, "iptables-mode", options.IPTablesMode, "Variant of iptables command to use (\"legacy\" or \"nft\"); overrides --firewall-bin-path and --firewall-save-bin-path")
+	cmd.PersistentFlags().BoolVar(&options.IPv6, "ipv6", options.IPv6, "Set rules both via iptables and ip6tables to support dual-stack networking")
+
+	// these two flags are kept for backwards-compatibility, but --iptables-mode is preferred
 	cmd.PersistentFlags().StringVar(&options.FirewallBinPath, "firewall-bin-path", options.FirewallBinPath, "Path to iptables binary")
 	cmd.PersistentFlags().StringVar(&options.FirewallSaveBinPath, "firewall-save-bin-path", options.FirewallSaveBinPath, "Path to iptables-save binary")
 	return cmd
 }
 
 // BuildFirewallConfiguration returns an iptables FirewallConfiguration suitable to use to configure iptables.
 func BuildFirewallConfiguration(options *RootOptions) (*iptables.FirewallConfiguration, error) {
+	if options.IPTablesMode != "" && options.IPTablesMode != IPTablesModeLegacy && options.IPTablesMode != IPTablesModeNFT {
+		return nil, fmt.Errorf("--iptables-mode valid values are only \"%s\" and \"%s\"", IPTablesModeLegacy, IPTablesModeNFT)
+	}
+
+	if options.IPTablesMode == "" {
+		switch options.FirewallBinPath {
+		case "", cmdLegacy:
+			options.IPTablesMode = IPTablesModeLegacy
+		case cmdNFT:
+			options.IPTablesMode = IPTablesModeNFT
+		default:
+			return nil, fmt.Errorf("--firewall-bin-path valid values are only \"%s\" and \"%s\"", cmdLegacy, cmdNFT)
+		}
+	}
+
 	if !util.IsValidPort(options.IncomingProxyPort) {
 		return nil, fmt.Errorf("--incoming-proxy-port must be a valid TCP port number")
 	}
@@ -116,6 +178,8 @@ func BuildFirewallConfiguration(options *RootOptions) (*iptables.FirewallConfigu
 		return nil, fmt.Errorf("--outgoing-proxy-port must be a valid TCP port number")
 	}
 
+	cmd, cmdSave := getCommands(options)
+
 	sanitizedSubnets := []string{}
 	for _, subnet := range options.SubnetsToIgnore {
 		subnet := strings.TrimSpace(subnet)
@@ -138,8 +202,8 @@ func BuildFirewallConfiguration(options *RootOptions) (*iptables.FirewallConfigu
 		SimulateOnly:           options.SimulateOnly,
 		NetNs:                  options.NetNs,
 		UseWaitFlag:            options.UseWaitFlag,
-		BinPath:                options.FirewallBinPath,
-		SaveBinPath:            options.FirewallSaveBinPath,
+		BinPath:                cmd,
+		SaveBinPath:            cmdSave,
 	}
 
 	if len(options.PortsToRedirect) > 0 {
@@ -160,6 +224,21 @@ func getFormatter(format string) log.Formatter {
 	}
 }
 
+func getCommands(options *RootOptions) (string, string) {
+	if options.IPTablesMode == IPTablesModeLegacy {
+		if options.IPv6 {
+			return cmdLegacyIPv6, cmdLegacyIPv6Save
+		}
+		return cmdLegacy, cmdLegacySave
+	}
+
+	if options.IPv6 {
+		return cmdNFTIPv6, cmdNFTIPv6Save
+	}
+
+	return cmdNFT, cmdNFTSave
+}
+
 func setLogLevel(logLevel string) error {
 	level, err := log.ParseLevel(logLevel)
 	if err != nil {