From bfc22fcc4b5ffea656ea49a9a241258911a727ec Mon Sep 17 00:00:00 2001 From: rambohe Date: Thu, 6 May 2021 15:47:17 +0800 Subject: [PATCH] [feature]: add dummy network interface for yurthub (#289) --- Makefile | 2 +- cmd/yurthub/app/config/config.go | 65 ++++++----- cmd/yurthub/app/options/options.go | 55 +++++++++ cmd/yurthub/app/start.go | 20 +++- go.mod | 1 + go.sum | 4 +- pkg/yurthub/network/dummyif_linux.go | 123 +++++++++++++++++++++ pkg/yurthub/network/dummyif_test.go | 98 ++++++++++++++++ pkg/yurthub/network/dummyif_unsupported.go | 51 +++++++++ pkg/yurthub/network/iptables.go | 88 +++++++++++++++ pkg/yurthub/network/network.go | 99 +++++++++++++++++ pkg/yurthub/server/server.go | 37 ++++++- 12 files changed, 605 insertions(+), 38 deletions(-) create mode 100644 pkg/yurthub/network/dummyif_linux.go create mode 100644 pkg/yurthub/network/dummyif_test.go create mode 100644 pkg/yurthub/network/dummyif_unsupported.go create mode 100644 pkg/yurthub/network/iptables.go create mode 100644 pkg/yurthub/network/network.go diff --git a/Makefile b/Makefile index c2281f8fec8..cd4256f7920 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ gen-yaml: # Run test test: fmt vet - go test -v ./pkg/... ./cmd/... -coverprofile cover.out + go test -v -short ./pkg/... ./cmd/... -coverprofile cover.out go test -v -coverpkg=./pkg/yurttunnel/... -coverprofile=yurttunnel-cover.out ./test/integration/yurttunnel_test.go # Run go fmt against code diff --git a/cmd/yurthub/app/config/config.go b/cmd/yurthub/app/config/config.go index 16038084bc2..cfdd03de0b9 100644 --- a/cmd/yurthub/app/config/config.go +++ b/cmd/yurthub/app/config/config.go @@ -30,20 +30,24 @@ import ( // YurtHubConfiguration represents configuration of yurthub type YurtHubConfiguration struct { - LBMode string - RemoteServers []*url.URL - YurtHubServerAddr string - YurtHubProxyServerAddr string - GCFrequency int - CertMgrMode string - NodeName string - HeartbeatFailedRetry int - HeartbeatHealthyThreshold int - HeartbeatTimeoutSeconds int - MaxRequestInFlight int - JoinToken string - RootDir string - EnableProfiling bool + LBMode string + RemoteServers []*url.URL + YurtHubServerAddr string + YurtHubProxyServerAddr string + YurtHubProxyServerDummyAddr string + GCFrequency int + CertMgrMode string + NodeName string + HeartbeatFailedRetry int + HeartbeatHealthyThreshold int + HeartbeatTimeoutSeconds int + MaxRequestInFlight int + JoinToken string + RootDir string + EnableProfiling bool + EnableDummyIf bool + EnableIptables bool + HubAgentDummyIfName string } // Complete converts *options.YurtHubOptions to *YurtHubConfiguration @@ -55,21 +59,26 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) { hubServerAddr := net.JoinHostPort(options.YurtHubHost, options.YurtHubPort) proxyServerAddr := net.JoinHostPort(options.YurtHubHost, options.YurtHubProxyPort) + proxyServerDummyAddr := net.JoinHostPort(options.HubAgentDummyIfIP, options.YurtHubProxyPort) cfg := &YurtHubConfiguration{ - LBMode: options.LBMode, - RemoteServers: us, - YurtHubServerAddr: hubServerAddr, - YurtHubProxyServerAddr: proxyServerAddr, - GCFrequency: options.GCFrequency, - CertMgrMode: options.CertMgrMode, - NodeName: options.NodeName, - HeartbeatFailedRetry: options.HeartbeatFailedRetry, - HeartbeatHealthyThreshold: options.HeartbeatHealthyThreshold, - HeartbeatTimeoutSeconds: options.HeartbeatTimeoutSeconds, - MaxRequestInFlight: options.MaxRequestInFlight, - JoinToken: options.JoinToken, - RootDir: options.RootDir, - EnableProfiling: options.EnableProfiling, + LBMode: options.LBMode, + RemoteServers: us, + YurtHubServerAddr: hubServerAddr, + YurtHubProxyServerAddr: proxyServerAddr, + YurtHubProxyServerDummyAddr: proxyServerDummyAddr, + GCFrequency: options.GCFrequency, + CertMgrMode: options.CertMgrMode, + NodeName: options.NodeName, + HeartbeatFailedRetry: options.HeartbeatFailedRetry, + HeartbeatHealthyThreshold: options.HeartbeatHealthyThreshold, + HeartbeatTimeoutSeconds: options.HeartbeatTimeoutSeconds, + MaxRequestInFlight: options.MaxRequestInFlight, + JoinToken: options.JoinToken, + RootDir: options.RootDir, + EnableProfiling: options.EnableProfiling, + EnableDummyIf: options.EnableDummyIf, + EnableIptables: options.EnableIptables, + HubAgentDummyIfName: options.HubAgentDummyIfName, } return cfg, nil diff --git a/cmd/yurthub/app/options/options.go b/cmd/yurthub/app/options/options.go index d264b57b7ee..f4bb0aa3287 100644 --- a/cmd/yurthub/app/options/options.go +++ b/cmd/yurthub/app/options/options.go @@ -18,6 +18,7 @@ package options import ( "fmt" + "net" "path/filepath" "github.com/openyurtio/openyurt/pkg/projectinfo" @@ -25,6 +26,11 @@ import ( "github.com/spf13/pflag" ) +const ( + DummyIfCIDR = "169.254.0.0/16" + ExclusiveCIDR = "169.254.31.0/24" +) + // YurtHubOptions is the main settings for the yurthub type YurtHubOptions struct { ServerAddr string @@ -43,6 +49,10 @@ type YurtHubOptions struct { RootDir string Version bool EnableProfiling bool + EnableDummyIf bool + EnableIptables bool + HubAgentDummyIfIP string + HubAgentDummyIfName string } // NewYurtHubOptions creates a new YurtHubOptions with a default config. @@ -60,6 +70,10 @@ func NewYurtHubOptions() *YurtHubOptions { MaxRequestInFlight: 250, RootDir: filepath.Join("/var/lib/", projectinfo.GetHubName()), EnableProfiling: true, + EnableDummyIf: true, + EnableIptables: true, + HubAgentDummyIfIP: "169.254.2.1", + HubAgentDummyIfName: fmt.Sprintf("%s-dummy0", projectinfo.GetHubName()), } return o @@ -83,6 +97,10 @@ func ValidateOptions(options *YurtHubOptions) error { return fmt.Errorf("cert manage mode %s is not supported", options.CertMgrMode) } + if err := verifyDummyIP(options.HubAgentDummyIfIP); err != nil { + return fmt.Errorf("dummy ip %s is not invalid, %v", options.HubAgentDummyIfIP, err) + } + return nil } @@ -104,4 +122,41 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&o.RootDir, "root-dir", o.RootDir, "directory path for managing hub agent files(pki, cache etc).") fs.BoolVar(&o.Version, "version", o.Version, "print the version information.") fs.BoolVar(&o.EnableProfiling, "profiling", o.EnableProfiling, "Enable profiling via web interface host:port/debug/pprof/") + fs.BoolVar(&o.EnableDummyIf, "enable-dummy-if", o.EnableDummyIf, "enable dummy interface or not") + fs.BoolVar(&o.EnableIptables, "enable-iptables", o.EnableIptables, "enable iptables manager to setup rules for accessing hub agent") + fs.StringVar(&o.HubAgentDummyIfIP, "dummy-if-ip", o.HubAgentDummyIfIP, "the ip address of dummy interface that used for container connect hub agent(exclusive ips: 169.254.31.0/24, 169.254.1.1/32)") + fs.StringVar(&o.HubAgentDummyIfName, "dummy-if-name", o.HubAgentDummyIfName, "the name of dummy interface that is used for hub agent") +} + +// verifyDummyIP verify the specified ip is valid or not +func verifyDummyIP(dummyIP string) error { + //169.254.2.1/32 + dip := net.ParseIP(dummyIP) + if dip == nil { + return fmt.Errorf("dummy ip %s is invalid", dummyIP) + } + + _, dummyIfIPNet, err := net.ParseCIDR(DummyIfCIDR) + if err != nil { + return fmt.Errorf("cidr(%s) is invalid, %v", DummyIfCIDR, err) + } + + if !dummyIfIPNet.Contains(dip) { + return fmt.Errorf("dummy ip %s is not in cidr(%s)", dummyIP, DummyIfCIDR) + } + + _, exclusiveIPNet, err := net.ParseCIDR(ExclusiveCIDR) + if err != nil { + return fmt.Errorf("cidr(%s) is invalid, %v", ExclusiveCIDR, err) + } + + if exclusiveIPNet.Contains(dip) { + return fmt.Errorf("dummy ip %s is in reserved cidr(%s)", dummyIP, ExclusiveCIDR) + } + + if dummyIP == "169.254.1.1" { + return fmt.Errorf("dummy ip is a reserved ip(%s)", dummyIP) + } + + return nil } diff --git a/cmd/yurthub/app/start.go b/cmd/yurthub/app/start.go index 47e834baf24..47e29f387fb 100644 --- a/cmd/yurthub/app/start.go +++ b/cmd/yurthub/app/start.go @@ -29,6 +29,7 @@ import ( "github.com/openyurtio/openyurt/pkg/yurthub/gc" "github.com/openyurtio/openyurt/pkg/yurthub/healthchecker" "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/serializer" + "github.com/openyurtio/openyurt/pkg/yurthub/network" "github.com/openyurtio/openyurt/pkg/yurthub/proxy" "github.com/openyurtio/openyurt/pkg/yurthub/server" "github.com/openyurtio/openyurt/pkg/yurthub/storage/factory" @@ -157,10 +158,27 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error { } trace++ + if cfg.EnableDummyIf { + klog.Infof("%d. create dummy network interface %s and init iptables manager", trace, cfg.HubAgentDummyIfName) + networkMgr, err := network.NewNetworkManager(cfg) + if err != nil { + klog.Errorf("could not create network manager, %v", err) + return err + } + networkMgr.Run(stopCh) + trace++ + klog.Infof("%d. new %s server and begin to serve, dummy proxy server: %s", trace, projectinfo.GetHubName(), cfg.YurtHubProxyServerDummyAddr) + } + klog.Infof("%d. new %s server and begin to serve, proxy server: %s, hub server: %s", trace, projectinfo.GetHubName(), cfg.YurtHubProxyServerAddr, cfg.YurtHubServerAddr) - s := server.NewYurtHubServer(cfg, certManager, yurtProxyHandler) + s, err := server.NewYurtHubServer(cfg, certManager, yurtProxyHandler) + if err != nil { + klog.Errorf("could not create hub server, %v", err) + return err + } s.Run() + klog.Infof("hub agent exited") <-stopCh return nil } diff --git a/go.mod b/go.mod index 9b5b52ab28c..d8e24393138 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/prometheus/procfs v0.0.11 // indirect github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 + github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect golang.org/x/text v0.3.3 // indirect golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect diff --git a/go.sum b/go.sum index 585849cf097..abf203c1a84 100644 --- a/go.sum +++ b/go.sum @@ -567,7 +567,9 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e h1:f1yevOHP+Suqk0rVc13fIkzcLULJbyQcXDba2klljD0= github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= +github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936 h1:J9gO8RJCAFlln1jsvRba/CWVUnMHwObklfxxjErl1uk= github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/vmware/govmomi v0.20.1/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= @@ -575,8 +577,6 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/zyjhtangtang/client-go v0.0.0-20210415100044-40a00cf7d1cc h1:InzXkH/XEWcc7vO5IV1ldmBlbwGUgutlF4QylFNeNOg= -github.com/zyjhtangtang/client-go v0.0.0-20210415100044-40a00cf7d1cc/go.mod h1:ThjPlh7Kx+XoBFOCt775vx5J7atwY7F/zaFzTco5gL0= go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= diff --git a/pkg/yurthub/network/dummyif_linux.go b/pkg/yurthub/network/dummyif_linux.go new file mode 100644 index 00000000000..4a2868249e4 --- /dev/null +++ b/pkg/yurthub/network/dummyif_linux.go @@ -0,0 +1,123 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package network + +import ( + "fmt" + "net" + "strings" + + "k8s.io/klog" + + "github.com/vishvananda/netlink" +) + +type DummyInterfaceController interface { + EnsureDummyInterface(ifName string, ifIP net.IP) error + DeleteDummyInterface(ifName string) error + ListDummyInterface(ifName string) ([]net.IP, error) +} + +type dummyInterfaceController struct { + netlink.Handle +} + +// NewDummyInterfaceManager returns an instance for create/delete dummy net interface +func NewDummyInterfaceController() DummyInterfaceController { + return &dummyInterfaceController{ + Handle: netlink.Handle{}, + } +} + +// EnsureDummyInterface make sure the dummy net interface with specified name and ip exist +func (dic *dummyInterfaceController) EnsureDummyInterface(ifName string, ifIP net.IP) error { + link, err := dic.LinkByName(ifName) + if err == nil { + addrs, err := dic.AddrList(link, 0) + if err != nil { + return err + } + + for _, addr := range addrs { + if addr.IP != nil && addr.IP.Equal(ifIP) { + return nil + } + } + + klog.Infof("ip address for %s interface changed to %s", ifName, ifIP.String()) + return dic.AddrReplace(link, &netlink.Addr{IPNet: netlink.NewIPNet(ifIP)}) + } + + if strings.Contains(err.Error(), "Link not found") && link == nil { + return dic.addDummyInterface(ifName, ifIP) + } + + return err +} + +// addDummyInterface creates a dummy net interface with the specified name and ip +func (dic *dummyInterfaceController) addDummyInterface(ifName string, ifIP net.IP) error { + _, err := dic.LinkByName(ifName) + if err == nil { + return fmt.Errorf("Link %s exists", ifName) + } + + dummy := &netlink.Dummy{ + LinkAttrs: netlink.LinkAttrs{Name: ifName}, + } + err = dic.LinkAdd(dummy) + if err != nil { + return err + } + + link, err := dic.LinkByName(ifName) + if err != nil { + return err + } + return dic.AddrAdd(link, &netlink.Addr{IPNet: netlink.NewIPNet(ifIP)}) +} + +// DeleteDummyInterface delete the dummy net interface with specified name +func (dic *dummyInterfaceController) DeleteDummyInterface(ifName string) error { + link, err := dic.LinkByName(ifName) + if err != nil { + return err + } + return dic.LinkDel(link) +} + +// ListDummyInterface list all ips for network interface specified by ifName +func (dic *dummyInterfaceController) ListDummyInterface(ifName string) ([]net.IP, error) { + ips := make([]net.IP, 0) + link, err := dic.LinkByName(ifName) + if err != nil { + return ips, err + } + + addrs, err := dic.AddrList(link, 0) + if err != nil { + return ips, err + } + + for _, addr := range addrs { + if addr.IP != nil { + ips = append(ips, addr.IP) + } + } + + return ips, nil +} diff --git a/pkg/yurthub/network/dummyif_test.go b/pkg/yurthub/network/dummyif_test.go new file mode 100644 index 00000000000..7e10fb34564 --- /dev/null +++ b/pkg/yurthub/network/dummyif_test.go @@ -0,0 +1,98 @@ +// +build linux + +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package network + +import ( + "net" + "testing" +) + +const ( + testDummyIfName = "test-dummyif" +) + +func TestEnsureDummyInterface(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode") + } + testcases := map[string]struct { + preparedIP string + testIP string + resultIP string + }{ + "init ensure dummy interface": { + testIP: "169.254.2.1", + resultIP: "169.254.2.1", + }, + "ensure dummy interface after prepared": { + preparedIP: "169.254.2.2", + testIP: "169.254.2.2", + resultIP: "169.254.2.2", + }, + "ensure dummy interface with new ip": { + preparedIP: "169.254.2.3", + testIP: "169.254.2.4", + resultIP: "169.254.2.4", + }, + } + + mgr := NewDummyInterfaceController() + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + if len(tc.preparedIP) != 0 { + err := mgr.EnsureDummyInterface(testDummyIfName, net.ParseIP(tc.preparedIP)) + if err != nil { + t.Errorf("failed to prepare dummy interface with ip(%s), %v", tc.preparedIP, err) + } + ips, err := mgr.ListDummyInterface(testDummyIfName) + if err != nil || len(ips) == 0 { + t.Errorf("failed to prepare dummy interface(%s: %s), %v", testDummyIfName, tc.preparedIP, err) + } + } + + err := mgr.EnsureDummyInterface(testDummyIfName, net.ParseIP(tc.testIP)) + if err != nil { + t.Errorf("failed to ensure dummy interface with ip(%s), %v", tc.testIP, err) + } + + ips2, err := mgr.ListDummyInterface(testDummyIfName) + if err != nil || len(ips2) == 0 { + t.Errorf("failed to list dummy interface(%s), %v", testDummyIfName, err) + } + + sameIP := false + for _, ip := range ips2 { + if ip.String() == tc.resultIP { + sameIP = true + break + } + } + + if !sameIP { + t.Errorf("dummy if with ip(%s) is not ensured, addrs: %s", tc.resultIP, ips2[0].String()) + } + + // delete dummy interface + err = mgr.DeleteDummyInterface(testDummyIfName) + if err != nil { + t.Errorf("failed to delte dummy interface, %v", err) + } + }) + } +} diff --git a/pkg/yurthub/network/dummyif_unsupported.go b/pkg/yurthub/network/dummyif_unsupported.go new file mode 100644 index 00000000000..f972412e49b --- /dev/null +++ b/pkg/yurthub/network/dummyif_unsupported.go @@ -0,0 +1,51 @@ +// +build !linux + +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package network + +import ( + "net" +) + +type DummyInterfaceController interface { + EnsureDummyInterface(ifName string, ifIP net.IP) error + DeleteDummyInterface(ifName string) error + ListDummyInterface(ifName string) ([]net.IP, error) +} + +type unsupportedInterfaceController struct { +} + +func NewDummyInterfaceController() DummyInterfaceController { + return &unsupportedInterfaceController{} +} + +// EnsureDummyInterface unimplemented +func (uic *unsupportedInterfaceController) EnsureDummyInterface(ifName string, ifIP net.IP) error { + return nil +} + +// DeleteDummyInterface unimplemented +func (uic *unsupportedInterfaceController) DeleteDummyInterface(ifName string) error { + return nil +} + +// ListDummyInterface unimplemented +func (dic *unsupportedInterfaceController) ListDummyInterface(ifName string) ([]net.IP, error) { + return []net.IP{}, nil +} diff --git a/pkg/yurthub/network/iptables.go b/pkg/yurthub/network/iptables.go new file mode 100644 index 00000000000..1be90ded5f4 --- /dev/null +++ b/pkg/yurthub/network/iptables.go @@ -0,0 +1,88 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package network + +import ( + "strings" + + "k8s.io/klog" + utildbus "k8s.io/kubernetes/pkg/util/dbus" + "k8s.io/kubernetes/pkg/util/iptables" + "k8s.io/utils/exec" +) + +type iptablesRule struct { + pos iptables.RulePosition + table iptables.Table + chain iptables.Chain + args []string +} + +type IptablesManager struct { + iptables iptables.Interface + rules []iptablesRule +} + +func NewIptablesManager(dummyIfIP, dummyIfPort string) *IptablesManager { + protocol := iptables.ProtocolIpv4 + execer := exec.New() + dbus := utildbus.New() + iptInterface := iptables.New(execer, dbus, protocol) + + im := &IptablesManager{ + iptables: iptInterface, + rules: makeupIptablesRules(dummyIfIP, dummyIfPort), + } + + return im +} + +func makeupIptablesRules(ifIP, ifPort string) []iptablesRule { + return []iptablesRule{ + // skip connection track for traffic from container to 169.254.2.1:10261 + {iptables.Prepend, iptables.Table("raw"), iptables.ChainPrerouting, []string{"-p", "tcp", "--dport", ifPort, "--destination", ifIP, "-j", "NOTRACK"}}, + // skip connection track for traffic from host network to 169.254.2.1:10261 + {iptables.Prepend, iptables.Table("raw"), iptables.ChainOutput, []string{"-p", "tcp", "--dport", ifPort, "--destination", ifIP, "-j", "NOTRACK"}}, + // accept traffic to 169.254.2.1:10261 + {iptables.Prepend, iptables.TableFilter, iptables.ChainInput, []string{"-p", "tcp", "-m", "comment", "--comment", "for container access hub agent", "--dport", ifPort, "--destination", ifIP, "-j", "ACCEPT"}}, + // skip connection track for traffic from 169.254.2.1:10261 + {iptables.Prepend, iptables.Table("raw"), iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", ifIP, "-j", "NOTRACK"}}, + // accept traffic from 169.254.2.1:10261 + {iptables.Prepend, iptables.TableFilter, iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", ifIP, "-j", "ACCEPT"}}, + // skip connection track for traffic from container to 127.0.0.1:10261 + {iptables.Prepend, iptables.Table("raw"), iptables.ChainPrerouting, []string{"-p", "tcp", "--dport", ifPort, "--destination", "127.0.0.1", "-j", "NOTRACK"}}, + // skip connection track for traffic from host network to 127.0.0.1:10261 + {iptables.Prepend, iptables.Table("raw"), iptables.ChainOutput, []string{"-p", "tcp", "--dport", ifPort, "--destination", "127.0.0.1", "-j", "NOTRACK"}}, + // accept traffic to 127.0.0.1:10261 + {iptables.Prepend, iptables.TableFilter, iptables.ChainInput, []string{"-p", "tcp", "--dport", ifPort, "--destination", "127.0.0.1", "-j", "ACCEPT"}}, + // skip connection track for traffic from 127.0.0.1:10261 + {iptables.Prepend, iptables.Table("raw"), iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", "127.0.0.1", "-j", "NOTRACK"}}, + // accept traffic from 127.0.0.1:10261 + {iptables.Prepend, iptables.TableFilter, iptables.ChainOutput, []string{"-p", "tcp", "--sport", ifPort, "-s", "127.0.0.1", "-j", "ACCEPT"}}, + } +} + +func (im *IptablesManager) EnsureIptablesRules() error { + for _, rule := range im.rules { + _, err := im.iptables.EnsureRule(rule.pos, rule.table, rule.chain, rule.args...) + if err != nil { + klog.Errorf("could not ensure iptables rule(%s -t %s %s %s), %v", rule.pos, rule.table, rule.chain, strings.Join(rule.args, ","), err) + return err + } + } + return nil +} diff --git a/pkg/yurthub/network/network.go b/pkg/yurthub/network/network.go new file mode 100644 index 00000000000..55f3aed6dc0 --- /dev/null +++ b/pkg/yurthub/network/network.go @@ -0,0 +1,99 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package network + +import ( + "fmt" + "net" + "time" + + "github.com/openyurtio/openyurt/cmd/yurthub/app/config" + + "k8s.io/klog" +) + +const ( + SyncNetworkPeriod = 60 +) + +type NetworkManager struct { + ifController DummyInterfaceController + iptablesManager *IptablesManager + dummyIfIP net.IP + dummyIfName string + enableIptables bool +} + +func NewNetworkManager(cfg *config.YurtHubConfiguration) (*NetworkManager, error) { + if cfg == nil { + return nil, fmt.Errorf("configuration for hub agent is nil") + } + + ip, port, err := net.SplitHostPort(cfg.YurtHubProxyServerDummyAddr) + if err != nil { + return nil, err + } + m := &NetworkManager{ + ifController: NewDummyInterfaceController(), + iptablesManager: NewIptablesManager(ip, port), + dummyIfIP: net.ParseIP(ip), + dummyIfName: cfg.HubAgentDummyIfName, + enableIptables: cfg.EnableIptables, + } + if err = m.configureNetwork(); err != nil { + return nil, err + } + + return m, nil +} + +func (m *NetworkManager) Run(stopCh <-chan struct{}) { + go func() { + ticker := time.NewTicker(SyncNetworkPeriod * time.Second) + defer ticker.Stop() + + for { + select { + case <-stopCh: + klog.Infof("exit network manager run goroutine normally") + return + case <-ticker.C: + if err := m.configureNetwork(); err != nil { + // do nothing here + } + } + } + }() +} + +func (m *NetworkManager) configureNetwork() error { + err := m.ifController.EnsureDummyInterface(m.dummyIfName, m.dummyIfIP) + if err != nil { + klog.Errorf("ensure dummy interface failed, %v", err) + return err + } + + if m.enableIptables { + err := m.iptablesManager.EnsureIptablesRules() + if err != nil { + klog.Errorf("ensure iptables for dummy interface failed, %v", err) + return err + } + } + + return nil +} diff --git a/pkg/yurthub/server/server.go b/pkg/yurthub/server/server.go index 4d221f8ea07..4753966062f 100644 --- a/pkg/yurthub/server/server.go +++ b/pkg/yurthub/server/server.go @@ -18,6 +18,7 @@ package server import ( "fmt" + "net" "net/http" "github.com/openyurtio/openyurt/cmd/yurthub/app/config" @@ -37,14 +38,15 @@ type Server interface { // and hubServer handles requests by hub agent itself, like profiling, metrics, healthz // and proxyServer does not handle requests locally and proxy requests to kube-apiserver type yurtHubServer struct { - hubServer *http.Server - proxyServer *http.Server + hubServer *http.Server + proxyServer *http.Server + dummyProxyServer *http.Server } // NewYurtHubServer creates a Server object func NewYurtHubServer(cfg *config.YurtHubConfiguration, certificateMgr interfaces.YurtCertificateManager, - proxyHandler http.Handler) Server { + proxyHandler http.Handler) (Server, error) { hubMux := mux.NewRouter() registerHandlers(hubMux, cfg, certificateMgr) hubServer := &http.Server{ @@ -59,10 +61,24 @@ func NewYurtHubServer(cfg *config.YurtHubConfiguration, MaxHeaderBytes: 1 << 20, } - return &yurtHubServer{ - hubServer: hubServer, - proxyServer: proxyServer, + var dummyProxyServer *http.Server + if cfg.EnableDummyIf { + if _, err := net.InterfaceByName(cfg.HubAgentDummyIfName); err != nil { + return nil, err + } + + dummyProxyServer = &http.Server{ + Addr: cfg.YurtHubProxyServerDummyAddr, + Handler: proxyHandler, + MaxHeaderBytes: 1 << 20, + } } + + return &yurtHubServer{ + hubServer: hubServer, + proxyServer: proxyServer, + dummyProxyServer: dummyProxyServer, + }, nil } // Run will start hub server and proxy server @@ -74,6 +90,15 @@ func (s *yurtHubServer) Run() { } }() + if s.dummyProxyServer != nil { + go func() { + err := s.dummyProxyServer.ListenAndServe() + if err != nil { + panic(err) + } + }() + } + err := s.proxyServer.ListenAndServe() if err != nil { panic(err)