From 80027f4ffc6277698ec791667f9a2614f2a1c3bb Mon Sep 17 00:00:00 2001
From: Wei Fu <fuweid89@gmail.com>
Date: Thu, 27 Feb 2020 15:50:00 +0800
Subject: [PATCH] Status: async reload latest CNI network conf

Signed-off-by: Wei Fu <fuweid89@gmail.com>
---
 pkg/config/config.go         |  3 +++
 pkg/config/config_unix.go    |  9 +++++----
 pkg/config/config_windows.go |  9 +++++----
 pkg/server/service.go        | 27 +++++++++++++++++++++++++++
 pkg/server/status.go         |  4 ----
 5 files changed, 40 insertions(+), 12 deletions(-)

diff --git a/pkg/config/config.go b/pkg/config/config.go
index 7a5ccc2af..d35e43c97 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -97,6 +97,9 @@ type CniConfig struct {
 	// a temporary backward-compatible solution for them.
 	// TODO(random-liu): Deprecate this option when kubenet is deprecated.
 	NetworkPluginConfTemplate string `toml:"conf_template" json:"confTemplate"`
+	// NetworkPluginConfReloadPeriod is the period (in seconds) of reloading
+	// CNI conf. The default value is 10 seconds.
+	NetworkPluginConfReloadPeriod int `toml:"conf_reload_period" json:"confReloadPeriod"`
 }
 
 // Mirror contains the config related to the registry mirror
diff --git a/pkg/config/config_unix.go b/pkg/config/config_unix.go
index 5501914d7..755225a7a 100644
--- a/pkg/config/config_unix.go
+++ b/pkg/config/config_unix.go
@@ -27,10 +27,11 @@ import (
 func DefaultConfig() PluginConfig {
 	return PluginConfig{
 		CniConfig: CniConfig{
-			NetworkPluginBinDir:       "/opt/cni/bin",
-			NetworkPluginConfDir:      "/etc/cni/net.d",
-			NetworkPluginMaxConfNum:   1, // only one CNI plugin config file will be loaded
-			NetworkPluginConfTemplate: "",
+			NetworkPluginBinDir:           "/opt/cni/bin",
+			NetworkPluginConfDir:          "/etc/cni/net.d",
+			NetworkPluginMaxConfNum:       1, // only one CNI plugin config file will be loaded
+			NetworkPluginConfTemplate:     "",
+			NetworkPluginConfReloadPeriod: 10,
 		},
 		ContainerdConfig: ContainerdConfig{
 			Snapshotter:        containerd.DefaultSnapshotter,
diff --git a/pkg/config/config_windows.go b/pkg/config/config_windows.go
index 7a73b985f..ce419c514 100644
--- a/pkg/config/config_windows.go
+++ b/pkg/config/config_windows.go
@@ -30,10 +30,11 @@ import (
 func DefaultConfig() PluginConfig {
 	return PluginConfig{
 		CniConfig: CniConfig{
-			NetworkPluginBinDir:       filepath.Join(os.Getenv("ProgramFiles"), "containerd", "cni", "bin"),
-			NetworkPluginConfDir:      filepath.Join(os.Getenv("ProgramFiles"), "containerd", "cni", "conf"),
-			NetworkPluginMaxConfNum:   1,
-			NetworkPluginConfTemplate: "",
+			NetworkPluginBinDir:           filepath.Join(os.Getenv("ProgramFiles"), "containerd", "cni", "bin"),
+			NetworkPluginConfDir:          filepath.Join(os.Getenv("ProgramFiles"), "containerd", "cni", "conf"),
+			NetworkPluginMaxConfNum:       1,
+			NetworkPluginConfTemplate:     "",
+			NetworkPluginConfReloadPeriod: 10,
 		},
 		ContainerdConfig: ContainerdConfig{
 			Snapshotter:        containerd.DefaultSnapshotter,
diff --git a/pkg/server/service.go b/pkg/server/service.go
index b7fd1c36a..d237dbc65 100644
--- a/pkg/server/service.go
+++ b/pkg/server/service.go
@@ -169,6 +169,17 @@ func (c *criService) Run() error {
 	)
 	snapshotsSyncer.start()
 
+	// Start CNI network conf syncer
+	//
+	// NOTE(fuweid): When there are a lot of requests to create or stop
+	// sandbox pod, the CNI plugin might need more time to allocate/release
+	// IP resources because of throttling. In this case, CNI plugin will
+	// hold lock for a while, which impacts that Status service can not
+	// reply to kubelet in time. Kubelet will be not ready. To prevent this,
+	// load latest CNI network conf async.
+	logrus.Info("Start CNI network conf syncer")
+	c.cniNetworkConfSyncer(time.Duration(c.config.NetworkPluginConfReloadPeriod) * time.Second)
+
 	// Start streaming server.
 	logrus.Info("Start streaming server")
 	streamServerErrCh := make(chan error)
@@ -243,6 +254,22 @@ func (c *criService) register(s *grpc.Server) error {
 	return nil
 }
 
+// cniNetworkConfSyncer loads latest CNI network conf.
+func (c *criService) cniNetworkConfSyncer(du time.Duration) {
+	var tick = time.NewTicker(du)
+
+	go func() {
+		defer tick.Stop()
+
+		for {
+			if err := c.netPlugin.Load(c.cniLoadOptions()...); err != nil {
+				logrus.WithError(err).Errorf("Failed to load cni configuration")
+			}
+			<-tick.C
+		}
+	}()
+}
+
 // imageFSPath returns containerd image filesystem path.
 // Note that if containerd changes directory layout, we also needs to change this.
 func imageFSPath(rootDir, snapshotter string) string {
diff --git a/pkg/server/status.go b/pkg/server/status.go
index af2521781..eabeb7d39 100644
--- a/pkg/server/status.go
+++ b/pkg/server/status.go
@@ -41,10 +41,6 @@ func (c *criService) Status(ctx context.Context, r *runtime.StatusRequest) (*run
 		Type:   runtime.NetworkReady,
 		Status: true,
 	}
-	// Load the latest cni configuration to be in sync with the latest network configuration
-	if err := c.netPlugin.Load(c.cniLoadOptions()...); err != nil {
-		log.G(ctx).WithError(err).Errorf("Failed to load cni configuration")
-	}
 	// Check the status of the cni initialization
 	if err := c.netPlugin.Status(); err != nil {
 		networkCondition.Status = false