Feature: use ipset for policy route

.github/workflows/build.yml

@@ -40,7 +40,11 @@ jobs:
             ${{ runner.os }}-buildx-
 
       - name: Unit test
-        run: sudo go test -v -short ./pkg/... ./cmd/... -coverprofile cover.out
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install -y ipset
+          sudo apt-get install -y iptables
+          sudo go test -v -short ./pkg/... ./cmd/... -coverprofile cover.out
 
       - name: Publish Unit Test Coverage
         uses: codecov/codecov-action@v3
@@ -100,10 +104,14 @@ jobs:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ github.sha }}
           restore-keys: |
-            ${{ runner.os }}-buildx-
+            ${{ runner.os }}-buildx- 
 
       - name: Unit test
-        run: sudo go test -v -short ./pkg/... ./cmd/...
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install -y ipset
+          sudo apt-get install -y iptables
+          sudo go test -v -short ./pkg/... ./cmd/...
 
       - name: Docker meta
         id: meta

Dockerfile

@@ -22,7 +22,7 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GO111MODULE=on go build
 
 FROM alpine:3.17
 RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories \
-    && apk add openrc libreswan libreswan-openrc iptables python3 bash --no-cache \
+    && apk add openrc libreswan libreswan-openrc ipset iptables python3 bash --no-cache \
     && sed -i 's/runscript/openrc-run/g' /etc/init.d/ipsec \
     && sed -i 's/#logfile=/logfile=/g' /etc/ipsec.conf \
     && mkdir -p /run/openrc \

go.mod

@@ -4,13 +4,16 @@ go 1.18
 
 require (
 	github.com/EvilSuperstars/go-cidrman v0.0.0-20190607145828-28e79e32899a
+	github.com/coreos/go-iptables v0.6.0
+	github.com/gonetx/ipset v0.1.0
 	github.com/openyurtio/openyurt v1.2.1-0.20230320014349-7cc573e1d097
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.2
 	github.com/vdobler/ht v5.3.0+incompatible
 	github.com/vishvananda/netlink v1.2.1-beta.2
+	golang.org/x/sys v0.6.0
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20220504211119-3d4a969bb56b
 	k8s.io/apimachinery v0.23.2
 	k8s.io/apiserver v0.23.2
@@ -87,7 +90,6 @@ require (
 	golang.org/x/net v0.7.0 // indirect
 	golang.org/x/oauth2 v0.5.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
 	golang.org/x/term v0.5.0 // indirect
 	golang.org/x/text v0.7.0 // indirect
 	golang.org/x/time v0.3.0 // indirect

go.sum

@@ -105,6 +105,8 @@ github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoC
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk=
+github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
 github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
 github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@@ -221,6 +223,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
 github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/gonetx/ipset v0.1.0 h1:LFkRdTbedg2UYXFN/2mOtgbvdWyo+OERrwVbtrPVuYY=
+github.com/gonetx/ipset v0.1.0/go.mod h1:AwNAf1Vtqg0cJ4bha4w1ROX5cO/8T50UYoegxM20AH8=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=

pkg/networkengine/routedriver/vxlan/utils.go

@@ -24,11 +24,16 @@ import (
 	"net"
 
 	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
 	"k8s.io/klog/v2"
 
 	netlinkutil "github.com/openyurtio/raven/pkg/networkengine/util/netlink"
 )
 
+const (
+	resetMark = 0x0
+)
+
 func ensureVxlanLink(vxlan netlink.Vxlan, vtepIP net.IP) (netlink.Link, error) {
 	linkExist := func() netlink.Link {
 		link, err := netlink.LinkByName(vxlanLinkName)
@@ -74,9 +79,154 @@ func ensureVxlanLink(vxlan netlink.Vxlan, vtepIP net.IP) (netlink.Link, error) {
 		return nil, fmt.Errorf("error add vxlan addr: %v", err)
 	}
 
+	// tc qdisc add dev raven0 clsact
+	err = ensureClsActQdsic(vxLink)
+	if err != nil {
+		return nil, fmt.Errorf("error ensure qdisc: %v", err)
+	}
+
+	// tc filter add dev raven0 egress protocol ip prio 1 matchall action skbedit mark 0x0
+	err = ensureSkbEditFilter(vxLink)
+	if err != nil {
+		return nil, fmt.Errorf("error ensure filter: %v", err)
+	}
+
 	return vxLink, nil
 }
 
+func ensureClsActQdsic(link netlink.Link) error {
+	qds, err := netlink.QdiscList(link)
+	if err != nil {
+		return fmt.Errorf("list qdisc for dev %s error, %w", link.Attrs().Name, err)
+	}
+	for _, q := range qds {
+		if q.Type() == "clsact" {
+			return nil
+		}
+	}
+	qdisc := &netlink.GenericQdisc{
+		QdiscAttrs: netlink.QdiscAttrs{
+			LinkIndex: link.Attrs().Index,
+			Parent:    netlink.HANDLE_CLSACT,
+			Handle:    netlink.HANDLE_CLSACT & 0xffff0000,
+		},
+		QdiscType: "clsact",
+	}
+	if err := netlink.QdiscReplace(qdisc); err != nil {
+		return fmt.Errorf("replace clsact qdisc for dev %s error, %w", link.Attrs().Name, err)
+	}
+	return nil
+}
+
+func deleteClsActQdsic(link netlink.Link) error {
+	qds, err := netlink.QdiscList(link)
+	if err != nil {
+		return fmt.Errorf("list qdisc for dev %s error, %w", link.Attrs().Name, err)
+	}
+	var qdisc netlink.Qdisc
+	for _, q := range qds {
+		if q.Type() == "clsact" {
+			qdisc = q
+			break
+		}
+	}
+	if qdisc != nil {
+		err = netlink.QdiscDel(qdisc)
+		if err != nil {
+			return fmt.Errorf("error delete qdisc: %s", err)
+		}
+	}
+	return nil
+}
+
+func ensureSkbEditFilter(link netlink.Link) error {
+	filters, err := netlink.FilterList(link, netlink.HANDLE_MIN_EGRESS)
+	if err != nil {
+		return fmt.Errorf("list egress filter for %s error, %w", link.Attrs().Name, err)
+	}
+
+	for _, f := range filters {
+		if isMatch(f) {
+			return nil
+		}
+	}
+
+	skbedit := netlink.NewSkbEditAction()
+	mark := uint32(resetMark)
+	skbedit.Mark = &mark
+	match := &netlink.MatchAll{
+		FilterAttrs: netlink.FilterAttrs{
+			LinkIndex: link.Attrs().Index,
+			Parent:    netlink.HANDLE_MIN_EGRESS,
+			Priority:  20000,
+			Protocol:  unix.ETH_P_IP,
+		},
+		Actions: []netlink.Action{
+			skbedit,
+		},
+	}
+
+	return netlink.FilterReplace(match)
+}
+
+func deleteSkbEditFilter(link netlink.Link) error {
+	filters, err := netlink.FilterList(link, netlink.HANDLE_MIN_EGRESS)
+	if err != nil {
+		return fmt.Errorf("list egress filter for %s error, %w", link.Attrs().Name, err)
+	}
+	for _, f := range filters {
+		_ = netlink.FilterDel(f)
+	}
+	return nil
+}
+
+func isMatch(filter netlink.Filter) bool {
+	match, ok := filter.(*netlink.MatchAll)
+	if !ok {
+		return false
+	}
+	if match.Parent != netlink.HANDLE_MIN_EGRESS || match.Protocol != unix.ETH_P_IP {
+		return false
+	}
+	if len(match.Actions) != 1 {
+		return false
+	}
+	action, ok := match.Actions[0].(*netlink.SkbEditAction)
+	if !ok {
+		return false
+	}
+	if *action.Mark != resetMark {
+		return false
+	}
+	return true
+}
+
+func deleteVxlanLink(linkName string) error {
+	vxLink, err := netlink.LinkByName(linkName)
+	if err != nil {
+		if _, ok := err.(netlink.LinkNotFoundError); ok {
+			return nil
+		}
+		return fmt.Errorf("error finding vxlan link: %s", err)
+	}
+
+	err = deleteSkbEditFilter(vxLink)
+	if err != nil {
+		return fmt.Errorf("error deleting skbedit filter: %s", err)
+	}
+	err = deleteClsActQdsic(vxLink)
+	if err != nil {
+		return fmt.Errorf("error deleting clsact qdsic: %s", err)
+	}
+	err = netlink.LinkDel(vxLink)
+	if err != nil {
+		if _, ok := err.(netlink.LinkNotFoundError); !ok {
+			return fmt.Errorf("error deleting vxlan link: %s", err)
+		}
+	}
+	return nil
+}
+
 func isVxlanConfigChanged(newLink, currentLink netlink.Link) bool {
 	required := newLink.(*netlink.Vxlan)
 	existing := currentLink.(*netlink.Vxlan)