Skip to content

Commit

Permalink
[feature]: add dummy network interface for yurthub
Browse files Browse the repository at this point in the history
  • Loading branch information
rambohe-ch committed May 6, 2021
1 parent 267643e commit e1466a6
Show file tree
Hide file tree
Showing 12 changed files with 605 additions and 39 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ gen-yaml:

# Run test
test: fmt vet
go test -v ./pkg/... ./cmd/... -coverprofile cover.out
go test -v -short ./pkg/... ./cmd/... -coverprofile cover.out
go test -v -coverpkg=./pkg/yurttunnel/... -coverprofile=yurttunnel-cover.out ./test/integration/yurttunnel_test.go

# Run go fmt against code
Expand Down
65 changes: 37 additions & 28 deletions cmd/yurthub/app/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,20 +30,24 @@ import (

// YurtHubConfiguration represents configuration of yurthub
type YurtHubConfiguration struct {
LBMode string
RemoteServers []*url.URL
YurtHubServerAddr string
YurtHubProxyServerAddr string
GCFrequency int
CertMgrMode string
NodeName string
HeartbeatFailedRetry int
HeartbeatHealthyThreshold int
HeartbeatTimeoutSeconds int
MaxRequestInFlight int
JoinToken string
RootDir string
EnableProfiling bool
LBMode string
RemoteServers []*url.URL
YurtHubServerAddr string
YurtHubProxyServerAddr string
YurtHubProxyServerDummyAddr string
GCFrequency int
CertMgrMode string
NodeName string
HeartbeatFailedRetry int
HeartbeatHealthyThreshold int
HeartbeatTimeoutSeconds int
MaxRequestInFlight int
JoinToken string
RootDir string
EnableProfiling bool
EnableDummyIf bool
EnableIptables bool
HubAgentDummyIfName string
}

// Complete converts *options.YurtHubOptions to *YurtHubConfiguration
Expand All @@ -55,21 +59,26 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) {

hubServerAddr := net.JoinHostPort(options.YurtHubHost, options.YurtHubPort)
proxyServerAddr := net.JoinHostPort(options.YurtHubHost, options.YurtHubProxyPort)
proxyServerDummyAddr := net.JoinHostPort(options.HubAgentDummyIfIP, options.YurtHubProxyPort)
cfg := &YurtHubConfiguration{
LBMode: options.LBMode,
RemoteServers: us,
YurtHubServerAddr: hubServerAddr,
YurtHubProxyServerAddr: proxyServerAddr,
GCFrequency: options.GCFrequency,
CertMgrMode: options.CertMgrMode,
NodeName: options.NodeName,
HeartbeatFailedRetry: options.HeartbeatFailedRetry,
HeartbeatHealthyThreshold: options.HeartbeatHealthyThreshold,
HeartbeatTimeoutSeconds: options.HeartbeatTimeoutSeconds,
MaxRequestInFlight: options.MaxRequestInFlight,
JoinToken: options.JoinToken,
RootDir: options.RootDir,
EnableProfiling: options.EnableProfiling,
LBMode: options.LBMode,
RemoteServers: us,
YurtHubServerAddr: hubServerAddr,
YurtHubProxyServerAddr: proxyServerAddr,
YurtHubProxyServerDummyAddr: proxyServerDummyAddr,
GCFrequency: options.GCFrequency,
CertMgrMode: options.CertMgrMode,
NodeName: options.NodeName,
HeartbeatFailedRetry: options.HeartbeatFailedRetry,
HeartbeatHealthyThreshold: options.HeartbeatHealthyThreshold,
HeartbeatTimeoutSeconds: options.HeartbeatTimeoutSeconds,
MaxRequestInFlight: options.MaxRequestInFlight,
JoinToken: options.JoinToken,
RootDir: options.RootDir,
EnableProfiling: options.EnableProfiling,
EnableDummyIf: options.EnableDummyIf,
EnableIptables: options.EnableIptables,
HubAgentDummyIfName: options.HubAgentDummyIfName,
}

return cfg, nil
Expand Down
55 changes: 55 additions & 0 deletions cmd/yurthub/app/options/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,19 @@ package options

import (
"fmt"
"net"
"path/filepath"

"github.com/openyurtio/openyurt/pkg/projectinfo"
"github.com/openyurtio/openyurt/pkg/yurthub/util"
"github.com/spf13/pflag"
)

const (
DummyIfCIDR = "169.254.0.0/16"
ExclusiveCIDR = "169.254.31.0/24"
)

// YurtHubOptions is the main settings for the yurthub
type YurtHubOptions struct {
ServerAddr string
Expand All @@ -43,6 +49,10 @@ type YurtHubOptions struct {
RootDir string
Version bool
EnableProfiling bool
EnableDummyIf bool
EnableIptables bool
HubAgentDummyIfIP string
HubAgentDummyIfName string
}

// NewYurtHubOptions creates a new YurtHubOptions with a default config.
Expand All @@ -60,6 +70,10 @@ func NewYurtHubOptions() *YurtHubOptions {
MaxRequestInFlight: 250,
RootDir: filepath.Join("/var/lib/", projectinfo.GetHubName()),
EnableProfiling: true,
EnableDummyIf: true,
EnableIptables: true,
HubAgentDummyIfIP: "169.254.2.1",
HubAgentDummyIfName: fmt.Sprintf("%s-dummy0", projectinfo.GetHubName()),
}

return o
Expand All @@ -83,6 +97,10 @@ func ValidateOptions(options *YurtHubOptions) error {
return fmt.Errorf("cert manage mode %s is not supported", options.CertMgrMode)
}

if err := verifyDummyIP(options.HubAgentDummyIfIP); err != nil {
return fmt.Errorf("dummy ip %s is not invalid, %v", options.HubAgentDummyIfIP, err)
}

return nil
}

Expand All @@ -104,4 +122,41 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&o.RootDir, "root-dir", o.RootDir, "directory path for managing hub agent files(pki, cache etc).")
fs.BoolVar(&o.Version, "version", o.Version, "print the version information.")
fs.BoolVar(&o.EnableProfiling, "profiling", o.EnableProfiling, "Enable profiling via web interface host:port/debug/pprof/")
fs.BoolVar(&o.EnableDummyIf, "enable-dummy-if", o.EnableDummyIf, "enable dummy interface or not")
fs.BoolVar(&o.EnableIptables, "enable-iptables", o.EnableIptables, "enable iptables manager to setup rules for accessing hub agent")
fs.StringVar(&o.HubAgentDummyIfIP, "dummy-if-ip", o.HubAgentDummyIfIP, "the ip address of dummy interface that used for container connect hub agent(exclusive ips: 169.254.31.0/24, 169.254.1.1/32)")
fs.StringVar(&o.HubAgentDummyIfName, "dummy-if-name", o.HubAgentDummyIfName, "the name of dummy interface that is used for hub agent")
}

// verifyDummyIP verify the specified ip is valid or not
func verifyDummyIP(dummyIP string) error {
//169.254.2.1/32
dip := net.ParseIP(dummyIP)
if dip == nil {
return fmt.Errorf("dummy ip %s is invalid", dummyIP)
}

_, dummyIfIPNet, err := net.ParseCIDR(DummyIfCIDR)
if err != nil {
return fmt.Errorf("cidr(%s) is invalid, %v", DummyIfCIDR, err)
}

if !dummyIfIPNet.Contains(dip) {
return fmt.Errorf("dummy ip %s is not in cidr(%s)", dummyIP, DummyIfCIDR)
}

_, exclusiveIPNet, err := net.ParseCIDR(ExclusiveCIDR)
if err != nil {
return fmt.Errorf("cidr(%s) is invalid, %v", ExclusiveCIDR, err)
}

if exclusiveIPNet.Contains(dip) {
return fmt.Errorf("dummy ip %s is in reserved cidr(%s)", dummyIP, ExclusiveCIDR)
}

if dummyIP == "169.254.1.1" {
return fmt.Errorf("dummy ip is a reserved ip(%s)", dummyIP)
}

return nil
}
23 changes: 21 additions & 2 deletions cmd/yurthub/app/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/openyurtio/openyurt/pkg/yurthub/gc"
"github.com/openyurtio/openyurt/pkg/yurthub/healthchecker"
"github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/serializer"
"github.com/openyurtio/openyurt/pkg/yurthub/network"
"github.com/openyurtio/openyurt/pkg/yurthub/proxy"
"github.com/openyurtio/openyurt/pkg/yurthub/server"
"github.com/openyurtio/openyurt/pkg/yurthub/storage/factory"
Expand Down Expand Up @@ -157,10 +158,28 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error {
}
trace++

klog.Infof("%d. new %s server and begin to serve, proxy server: %s, hub server: %s", trace, projectinfo.GetHubName(), cfg.YurtHubProxyServerAddr, cfg.YurtHubServerAddr)
s := server.NewYurtHubServer(cfg, certManager, yurtProxyHandler)
klog.Infof("%d. create dummy network interface %s and init iptables manager", trace, cfg.HubAgentDummyIfName)
networkMgr, err := network.NewNetworkManager(cfg)
if err != nil {
klog.Errorf("could not create network manager, %v", err)
return err
}
networkMgr.Run(stopCh)
trace++

klog.Infof("%d. new %s server and begin to serve, proxy server: %s;%s, hub server: %s", trace,
projectinfo.GetHubName(),
cfg.YurtHubProxyServerAddr,
cfg.YurtHubProxyServerDummyAddr,
cfg.YurtHubServerAddr)
s, err := server.NewYurtHubServer(cfg, certManager, yurtProxyHandler)
if err != nil {
klog.Errorf("could not create hub server, %v", err)
return err
}
s.Run()

klog.Infof("hub agent exited")
<-stopCh
return nil
}
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ require (
github.com/prometheus/procfs v0.0.11 // indirect
github.com/spf13/cobra v1.0.0
github.com/spf13/pflag v1.0.5
github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
golang.org/x/text v0.3.3 // indirect
golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -567,16 +567,16 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e h1:f1yevOHP+Suqk0rVc13fIkzcLULJbyQcXDba2klljD0=
github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936 h1:J9gO8RJCAFlln1jsvRba/CWVUnMHwObklfxxjErl1uk=
github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
github.com/vmware/govmomi v0.20.1/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/zyjhtangtang/client-go v0.0.0-20210415100044-40a00cf7d1cc h1:InzXkH/XEWcc7vO5IV1ldmBlbwGUgutlF4QylFNeNOg=
github.com/zyjhtangtang/client-go v0.0.0-20210415100044-40a00cf7d1cc/go.mod h1:ThjPlh7Kx+XoBFOCt775vx5J7atwY7F/zaFzTco5gL0=
go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
Expand Down
117 changes: 117 additions & 0 deletions pkg/yurthub/network/dummyif_linux.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
/*
Copyright 2021 The OpenYurt Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package network

import (
"fmt"
"net"

"k8s.io/klog"

"github.com/vishvananda/netlink"
)

type DummyInterfaceController interface {
EnsureDummyInterface(ifName string, ifIP net.IP) error
DeleteDummyInterface(ifName string) error
ListDummyInterface(ifName string) ([]net.IP, error)
}

type dummyInterfaceController struct {
netlink.Handle
}

// NewDummyInterfaceManager returns an instance for create/delete dummy net interface
func NewDummyInterfaceController() DummyInterfaceController {
return &dummyInterfaceController{
Handle: netlink.Handle{},
}
}

// EnsureDummyInterface make sure the dummy net interface with specified name and ip exist
func (dic *dummyInterfaceController) EnsureDummyInterface(ifName string, ifIP net.IP) error {
link, err := dic.LinkByName(ifName)
if err == nil {
addrs, err := dic.AddrList(link, 0)
if err != nil {
return err
}

for _, addr := range addrs {
if addr.IP != nil && addr.IP.Equal(ifIP) {
return nil
}
}

klog.Infof("ip address for %s interface changed to %s", ifName, ifIP.String())
return dic.AddrReplace(link, &netlink.Addr{IPNet: netlink.NewIPNet(ifIP)})
}
return dic.addDummyInterface(ifName, ifIP)
}

// addDummyInterface creates a dummy net interface with the specified name and ip
func (dic *dummyInterfaceController) addDummyInterface(ifName string, ifIP net.IP) error {
_, err := dic.LinkByName(ifName)
if err == nil {
return fmt.Errorf("Link %s exists", ifName)
}

dummy := &netlink.Dummy{
LinkAttrs: netlink.LinkAttrs{Name: ifName},
}
err = dic.LinkAdd(dummy)
if err != nil {
return err
}

link, err := dic.LinkByName(ifName)
if err != nil {
return err
}
return dic.AddrAdd(link, &netlink.Addr{IPNet: netlink.NewIPNet(ifIP)})
}

// DeleteDummyInterface delete the dummy net interface with specified name
func (dic *dummyInterfaceController) DeleteDummyInterface(ifName string) error {
link, err := dic.LinkByName(ifName)
if err != nil {
return err
}
return dic.LinkDel(link)
}

// ListDummyInterface list all ips for network interface specified by ifName
func (dic *dummyInterfaceController) ListDummyInterface(ifName string) ([]net.IP, error) {
ips := make([]net.IP, 0)
link, err := dic.LinkByName(ifName)
if err != nil {
return ips, err
}

addrs, err := dic.AddrList(link, 0)
if err != nil {
return ips, err
}

for _, addr := range addrs {
if addr.IP != nil {
ips = append(ips, addr.IP)
}
}

return ips, nil
}
Loading

0 comments on commit e1466a6

Please sign in to comment.