From 429953e850685ef5b262b5531fcbfbc1f9de5f5f Mon Sep 17 00:00:00 2001 From: honpey Date: Wed, 27 Apr 2022 19:56:32 +0800 Subject: [PATCH] Introduce HookServer config loading from /etc/runtime/hookserver.d/ (#100) Signed-off-by: pengyang.hpy --- examples/runtime-hook-server/koordlet.json | 7 + pkg/runtime-manager/config/config.go | 116 ++++++++ pkg/runtime-manager/config/config_manager.go | 262 +++++++++++++++++++ 3 files changed, 385 insertions(+) create mode 100644 examples/runtime-hook-server/koordlet.json create mode 100644 pkg/runtime-manager/config/config.go create mode 100644 pkg/runtime-manager/config/config_manager.go diff --git a/examples/runtime-hook-server/koordlet.json b/examples/runtime-hook-server/koordlet.json new file mode 100644 index 000000000..f9d838213 --- /dev/null +++ b/examples/runtime-hook-server/koordlet.json @@ -0,0 +1,7 @@ +{ + "remote-endpoint": "/var/run/koordlet-runtimehookserver.sock", + "failure-policy": "Fail", + "runtime-hooks": [ + "PreRunPodSandbox" + ] +} diff --git a/pkg/runtime-manager/config/config.go b/pkg/runtime-manager/config/config.go new file mode 100644 index 000000000..98c6b0379 --- /dev/null +++ b/pkg/runtime-manager/config/config.go @@ -0,0 +1,116 @@ +/* + * Copyright 2022 The Koordinator Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package config + +import "strings" + +type FailurePolicyType string + +const ( + // PolicyFail returns error to caller when got an error cri hook server + PolicyFail FailurePolicyType = "Fail" + // PolicyIgnore transfer cri request to containerd/dockerd when got an error to cri serer + PolicyIgnore FailurePolicyType = "Ignore" +) + +type RuntimeHookType string + +const ( + defaultRuntimeHookConfigPath string = "/etc/runtime/hookserver.d" +) + +const ( + PreRunPodSandbox RuntimeHookType = "PreRunPodSandbox" + PreStartContainer RuntimeHookType = "PreStartContainer" + PostStartContainer RuntimeHookType = "PostStartContainer" + PreUpdateContainerResources RuntimeHookType = "PreUpdateContainerResources" + PostStopContainer RuntimeHookType = "PostStopContainer" + NoneRuntimeHookType RuntimeHookType = "NoneRuntimeHookType" +) + +type RuntimeHookConfig struct { + RemoteEndpoint string `json:"remote-endpoint,omitempty"` + FailurePolicy FailurePolicyType `json:"failure-policy,omitempty"` + RuntimeHooks []RuntimeHookType `json:"runtime-hooks,omitempty"` +} + +type RuntimeRequestPath string + +const ( + RunPodSandbox RuntimeRequestPath = "RunPodSandbox" + StartContainer RuntimeRequestPath = "StartContainer" + UpdateContainerResources RuntimeRequestPath = "UpdateContainerResources" + StopContainer RuntimeRequestPath = "StopContainer" + NoneRuntimeHookPath RuntimeRequestPath = "NoneRuntimeHookPath" +) + +func (ht RuntimeHookType) OccursOn(path RuntimeRequestPath) bool { + switch ht { + case PreRunPodSandbox: + if path == RunPodSandbox { + return true + } + case PreStartContainer: + if path == StartContainer { + return true + } + case PostStartContainer: + if path == StartContainer { + return true + } + case PreUpdateContainerResources: + if path == UpdateContainerResources { + return true + } + case PostStopContainer: + if path == StopContainer { + return true + } + } + return false +} + +func (hp RuntimeRequestPath) PreHookType() RuntimeHookType { + if hp == RunPodSandbox { + return PreRunPodSandbox + } + return NoneRuntimeHookType +} + +func (hp RuntimeRequestPath) PostHookType() RuntimeHookType { + if hp == RunPodSandbox { + return NoneRuntimeHookType + } + return NoneRuntimeHookType +} + +type RuntimeHookStage string + +const ( + PreHook RuntimeHookStage = "PreHook" + PostHook RuntimeHookStage = "PostHook" + UnknownHook RuntimeHookStage = "UnknownHook" +) + +func (ht RuntimeHookType) HookStage() RuntimeHookStage { + if strings.HasPrefix(string(ht), "Pre") { + return PreHook + } else if strings.HasPrefix(string(ht), "Post") { + return PostHook + } + return UnknownHook +} diff --git a/pkg/runtime-manager/config/config_manager.go b/pkg/runtime-manager/config/config_manager.go new file mode 100644 index 000000000..a274f1d6d --- /dev/null +++ b/pkg/runtime-manager/config/config_manager.go @@ -0,0 +1,262 @@ +/* + * Copyright 2022 The Koordinator Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package config + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "github.com/fsnotify/fsnotify" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog/v2" +) + +const ( + defaultConfigFileNums = 2 +) + +type Manager struct { + sync.Mutex + configs map[string]*RuntimeHookConfigItem + watcher *fsnotify.Watcher +} + +type RuntimeHookConfigItem struct { + filePath string + fileIno uint64 + updateTime time.Time + *RuntimeHookConfig +} + +func (m *Manager) GetAllHook() []*RuntimeHookConfig { + var runtimeConfigs []*RuntimeHookConfig + m.Lock() + defer m.Unlock() + for _, config := range m.configs { + runtimeConfigs = append(runtimeConfigs, config.RuntimeHookConfig) + } + return runtimeConfigs +} + +func (m *Manager) getAllRegisteredFiles() []string { + var files []string + m.Lock() + defer m.Unlock() + for filepath := range m.configs { + files = append(files, filepath) + } + return files +} + +func NewConfigManager() *Manager { + return &Manager{ + configs: make(map[string]*RuntimeHookConfigItem, defaultConfigFileNums), + } +} + +func (m *Manager) registerFileToWatchIfNeed(file string) error { + fileInfo, err := os.Stat(file) + if err != nil { + return err + } + stat, ok := fileInfo.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("fail to get file ino: %v", file) + } + m.Lock() + defer m.Unlock() + config, exist := m.configs[file] + if exist && config.fileIno == stat.Ino { + return nil + } + if exist && config.fileIno != stat.Ino { + m.watcher.Remove(file) + klog.Infof("remove previous file %v with inode number %v", file, config.fileIno) + } + m.watcher.Add(file) + m.configs[file] = &RuntimeHookConfigItem{ + filePath: file, + fileIno: stat.Ino, + } + klog.Infof("add new watching file %v with inode number %v", file, stat.Ino) + return nil +} + +func (m *Manager) removeFileToWatch(filepath string) { + m.Lock() + defer m.Unlock() + if _, exist := m.configs[filepath]; !exist { + return + } + err := m.watcher.Remove(filepath) + if err != nil { + klog.Errorf("fail to remove %s to watch", filepath) + } + delete(m.configs, filepath) + klog.Infof("remove watching file %v", filepath) +} + +func (m *Manager) needRefreshConfig(filepath string) bool { + fileStat, err := os.Stat(filepath) + if err != nil { + klog.Errorf("fail to stat %v", err) + return false + } + fileModTime := fileStat.ModTime() + lastUpdateTimestamp := func(filepath string) time.Time { + m.Lock() + defer m.Unlock() + if config, exist := m.configs[filepath]; !exist { + return time.Time{} + } else { + return config.updateTime + } + }(filepath) + + return lastUpdateTimestamp.Before(fileModTime) +} + +// updateHookConfig loads config file, and register file to fsnotify watcher to watch +// config file content changed +// the filepath should be absolute path +func (m *Manager) updateHookConfig(filepath string) error { + if !strings.HasSuffix(filepath, "json") { + return nil + } + + if err := m.registerFileToWatchIfNeed(filepath); err != nil { + klog.Errorf("fail to registry file %v", filepath) + return err + } + + if !m.needRefreshConfig(filepath) { + return nil + } + + updateTime := time.Now() + data, err := ioutil.ReadFile(filepath) + if err != nil { + return err + } + config := &RuntimeHookConfig{} + if err := json.Unmarshal(data, config); err != nil { + return err + } + + m.Lock() + defer m.Unlock() + + configItem, exist := m.configs[filepath] + if !exist { + return fmt.Errorf("no found config file %v", filepath) + } + configItem.RuntimeHookConfig = config + configItem.updateTime = updateTime + klog.Infof("update config for %v %v", filepath, config) + return nil +} + +func (m *Manager) Setup() error { + if _, err := os.Stat(defaultRuntimeHookConfigPath); os.IsNotExist(err) { + klog.Infof("create %v", defaultRuntimeHookConfigPath) + if err := os.MkdirAll(defaultRuntimeHookConfigPath, 0755); err != nil { + klog.Errorf("fail to create %v %v", defaultRuntimeHookConfigPath, err) + return err + } + } + // watch the newly generated config file + watcher, err := fsnotify.NewWatcher() + if err != nil { + return err + } + m.watcher = watcher + + if err := m.watcher.Add(defaultRuntimeHookConfigPath); err != nil { + return err + } + go m.syncLoop() + + // collect the existing config + m.collectAllConfigs() + go m.healthCheck() + + return nil +} + +func (m *Manager) collectAllConfigs() error { + items, err := ioutil.ReadDir(defaultRuntimeHookConfigPath) + if err != nil { + return err + } + for _, item := range items { + if item.IsDir() { + continue + } + if err := m.updateHookConfig(filepath.Join(defaultRuntimeHookConfigPath, item.Name())); err != nil { + continue + } + } + return nil +} + +func (m *Manager) syncLoop() error { + for { + select { + case event, ok := <-m.watcher.Events: + if !ok { + klog.Infof("config manager channel is closed") + return nil + } + // only reload config when write/rename/remove events + if event.Op&(fsnotify.Chmod) > 0 { + klog.V(5).Infof("ignore event from runtime hook config dir %v", event) + continue + } + // should add the config file to watcher if event.Op is fsnotify.Create + klog.V(5).Infof("receive change event from runtime hook config dir %v", event) + m.updateHookConfig(event.Name) + case err := <-m.watcher.Errors: + if err != nil { + klog.Errorf("failed to continue to sync %v", defaultRuntimeHookConfigPath) + } + } + } +} + +func (m *Manager) removeUnusedConfigs() { + for _, file := range m.getAllRegisteredFiles() { + if _, err := os.Stat(file); os.IsNotExist(err) { + m.removeFileToWatch(file) + } + } +} + +func (m *Manager) healthCheck() { + wait.Until(func() { + m.removeUnusedConfigs() + m.collectAllConfigs() + allFiles := m.getAllRegisteredFiles() + klog.V(6).Infof("current runtime hook config infos %v(%v)", allFiles, len(allFiles)) + }, time.Minute, wait.NeverStop) +}