From 4efa1bf02c689f2746a88547dfdff32a04b3a561 Mon Sep 17 00:00:00 2001 From: rambohe Date: Mon, 24 Jul 2023 11:01:19 +0800 Subject: [PATCH 01/21] add kubelet certificate mode in yurthub (#1625) --- cmd/yurthub/app/config/config.go | 57 ++----- cmd/yurthub/app/config/config_test.go | 4 +- cmd/yurthub/app/options/options.go | 3 + cmd/yurthub/app/options/options_test.go | 1 + pkg/yurthub/certificate/interfaces.go | 15 +- .../kubeletcertificate/kubelet_certificate.go | 119 ++++++++++++++ .../kubelet_certificate_test.go | 59 +++++++ pkg/yurthub/certificate/manager/manager.go | 137 ++++++++++++++++ .../certificate/manager/manager_test.go | 136 +++++++++++++++ pkg/yurthub/certificate/server/server.go | 102 ++++++++++++ pkg/yurthub/certificate/server/server_test.go | 55 +++++++ .../certificate/{token => }/testdata/ca.crt | 0 .../certificate/{token => }/testdata/ca.key | 0 .../{token => }/testdata/fake_client.go | 0 pkg/yurthub/certificate/testdata/kubelet.conf | 14 ++ pkg/yurthub/certificate/testdata/kubelet.pem | 22 +++ pkg/yurthub/certificate/token/token.go | 132 +++------------ pkg/yurthub/certificate/token/token_test.go | 155 +++--------------- pkg/yurthub/kubernetes/rest/config_test.go | 18 +- pkg/yurthub/server/certificate_test.go | 19 +-- 20 files changed, 733 insertions(+), 315 deletions(-) create mode 100644 pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate.go create mode 100644 pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate_test.go create mode 100644 pkg/yurthub/certificate/manager/manager.go create mode 100644 pkg/yurthub/certificate/manager/manager_test.go create mode 100644 pkg/yurthub/certificate/server/server.go create mode 100644 pkg/yurthub/certificate/server/server_test.go rename pkg/yurthub/certificate/{token => }/testdata/ca.crt (100%) rename pkg/yurthub/certificate/{token => }/testdata/ca.key (100%) rename pkg/yurthub/certificate/{token => }/testdata/fake_client.go (100%) create mode 100644 pkg/yurthub/certificate/testdata/kubelet.conf create mode 100644 pkg/yurthub/certificate/testdata/kubelet.pem diff --git a/cmd/yurthub/app/config/config.go b/cmd/yurthub/app/config/config.go index 85a64ab435a..06529a69fc5 100644 --- a/cmd/yurthub/app/config/config.go +++ b/cmd/yurthub/app/config/config.go @@ -44,10 +44,9 @@ import ( "github.com/openyurtio/openyurt/cmd/yurthub/app/options" "github.com/openyurtio/openyurt/pkg/projectinfo" - ipUtils "github.com/openyurtio/openyurt/pkg/util/ip" "github.com/openyurtio/openyurt/pkg/yurthub/cachemanager" "github.com/openyurtio/openyurt/pkg/yurthub/certificate" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token" + certificatemgr "github.com/openyurtio/openyurt/pkg/yurthub/certificate/manager" "github.com/openyurtio/openyurt/pkg/yurthub/filter" "github.com/openyurtio/openyurt/pkg/yurthub/filter/manager" "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/meta" @@ -178,10 +177,21 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) { LeaderElection: options.LeaderElection, } - certMgr, err := createCertManager(options, us) + certMgr, err := certificatemgr.NewYurtHubCertManager(options, us) if err != nil { return nil, err } + certMgr.Start() + err = wait.PollImmediate(5*time.Second, 4*time.Minute, func() (bool, error) { + isReady := certMgr.Ready() + if isReady { + return true, nil + } + return false, nil + }) + if err != nil { + return nil, fmt.Errorf("hub certificates preparation failed, %v", err) + } cfg.CertManager = certMgr if options.EnableDummyIf { @@ -230,7 +240,7 @@ func parseRemoteServers(serverAddr string) ([]*url.URL, error) { return us, nil } -// createSharedInformers create sharedInformers from the given proxyAddr. +// createClientAndSharedInformers create kubeclient and sharedInformers from the given proxyAddr. func createClientAndSharedInformers(proxyAddr string, enableNodePool bool) (kubernetes.Interface, informers.SharedInformerFactory, yurtinformers.SharedInformerFactory, error) { var kubeConfig *rest.Config var yurtClient yurtclientset.Interface @@ -341,45 +351,6 @@ func isServiceTopologyFilterEnabled(options *options.YurtHubOptions) bool { return true } -func createCertManager(options *options.YurtHubOptions, remoteServers []*url.URL) (certificate.YurtCertificateManager, error) { - // use dummy ip and bind ip as cert IP SANs - certIPs := ipUtils.RemoveDupIPs([]net.IP{ - net.ParseIP(options.HubAgentDummyIfIP), - net.ParseIP(options.YurtHubHost), - net.ParseIP(options.YurtHubProxyHost), - }) - - cfg := &token.CertificateManagerConfiguration{ - RootDir: options.RootDir, - NodeName: options.NodeName, - JoinToken: options.JoinToken, - BootstrapFile: options.BootstrapFile, - CaCertHashes: options.CACertHashes, - YurtHubCertOrganizations: options.YurtHubCertOrganizations, - CertIPs: certIPs, - RemoteServers: remoteServers, - Client: options.ClientForTest, - } - certManager, err := token.NewYurtHubCertManager(cfg) - if err != nil { - return nil, fmt.Errorf("failed to create cert manager for yurthub, %v", err) - } - - certManager.Start() - err = wait.PollImmediate(5*time.Second, 4*time.Minute, func() (bool, error) { - isReady := certManager.Ready() - if isReady { - return true, nil - } - return false, nil - }) - if err != nil { - return nil, fmt.Errorf("hub certificates preparation failed, %v", err) - } - - return certManager, nil -} - func prepareServerServing(options *options.YurtHubOptions, certMgr certificate.YurtCertificateManager, cfg *YurtHubConfiguration) error { if err := (&apiserveroptions.DeprecatedInsecureServingOptions{ BindAddress: net.ParseIP(options.YurtHubHost), diff --git a/cmd/yurthub/app/config/config_test.go b/cmd/yurthub/app/config/config_test.go index cd657071e51..3fd46d2108e 100644 --- a/cmd/yurthub/app/config/config_test.go +++ b/cmd/yurthub/app/config/config_test.go @@ -20,12 +20,12 @@ import ( "testing" "github.com/openyurtio/openyurt/cmd/yurthub/app/options" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token/testdata" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/testdata" ) func TestComplete(t *testing.T) { options := options.NewYurtHubOptions() - client, err := testdata.CreateCertFakeClient("../../../../pkg/yurthub/certificate/token/testdata") + client, err := testdata.CreateCertFakeClient("../../../../pkg/yurthub/certificate/testdata") if err != nil { t.Errorf("failed to create cert fake client, %v", err) return diff --git a/cmd/yurthub/app/options/options.go b/cmd/yurthub/app/options/options.go index ccabdbdd097..a6f0deadbaf 100644 --- a/cmd/yurthub/app/options/options.go +++ b/cmd/yurthub/app/options/options.go @@ -62,6 +62,7 @@ type YurtHubOptions struct { HeartbeatIntervalSeconds int MaxRequestInFlight int JoinToken string + BootstrapMode string BootstrapFile string RootDir string Version bool @@ -105,6 +106,7 @@ func NewYurtHubOptions() *YurtHubOptions { HeartbeatTimeoutSeconds: 2, HeartbeatIntervalSeconds: 10, MaxRequestInFlight: 250, + BootstrapMode: "token", RootDir: filepath.Join("/var/lib/", projectinfo.GetHubName()), EnableProfiling: true, EnableDummyIf: true, @@ -189,6 +191,7 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) { fs.IntVar(&o.MaxRequestInFlight, "max-requests-in-flight", o.MaxRequestInFlight, "the maximum number of parallel requests.") fs.StringVar(&o.JoinToken, "join-token", o.JoinToken, "the Join token for bootstrapping hub agent.") fs.MarkDeprecated("join-token", "It is planned to be removed from OpenYurt in the version v1.5. Please use --bootstrap-file to bootstrap hub agent.") + fs.StringVar(&o.BootstrapMode, "bootstrap-mode", o.BootstrapMode, "the mode for bootstrapping hub agent(token, kubeletcertificate).") fs.StringVar(&o.BootstrapFile, "bootstrap-file", o.BootstrapFile, "the bootstrap file for bootstrapping hub agent.") fs.StringVar(&o.RootDir, "root-dir", o.RootDir, "directory path for managing hub agent files(pki, cache etc).") fs.BoolVar(&o.Version, "version", o.Version, "print the version information.") diff --git a/cmd/yurthub/app/options/options_test.go b/cmd/yurthub/app/options/options_test.go index d9d2379b600..a4370df98cc 100644 --- a/cmd/yurthub/app/options/options_test.go +++ b/cmd/yurthub/app/options/options_test.go @@ -49,6 +49,7 @@ func TestNewYurtHubOptions(t *testing.T) { HeartbeatTimeoutSeconds: 2, HeartbeatIntervalSeconds: 10, MaxRequestInFlight: 250, + BootstrapMode: "token", RootDir: filepath.Join("/var/lib/", projectinfo.GetHubName()), EnableProfiling: true, EnableDummyIf: true, diff --git a/pkg/yurthub/certificate/interfaces.go b/pkg/yurthub/certificate/interfaces.go index 6ac9d8a9bae..ecbaedc8220 100644 --- a/pkg/yurthub/certificate/interfaces.go +++ b/pkg/yurthub/certificate/interfaces.go @@ -22,14 +22,25 @@ import ( // YurtCertificateManager is responsible for managing node certificate for yurthub type YurtCertificateManager interface { - Start() - Stop() + YurtClientCertificateManager + YurtServerCertificateManager // Ready should be called after yurt certificate manager started by Start. Ready() bool +} + +// YurtClientCertificateManager is responsible for managing node client certificates for yurthub +type YurtClientCertificateManager interface { + Start() + Stop() UpdateBootstrapConf(joinToken string) error GetHubConfFile() string GetCaFile() string GetAPIServerClientCert() *tls.Certificate +} + +type YurtServerCertificateManager interface { + Start() + Stop() GetHubServerCert() *tls.Certificate GetHubServerCertFile() string } diff --git a/pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate.go b/pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate.go new file mode 100644 index 00000000000..00d1e7e911d --- /dev/null +++ b/pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate.go @@ -0,0 +1,119 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeletcertificate + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "time" + + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/pkg/yurthub/certificate" + "github.com/openyurtio/openyurt/pkg/yurthub/util" +) + +var ( + KubeConfNotExistErr = errors.New("/etc/kubernetes/kubelet.conf file doesn't exist") + KubeletCANotExistErr = errors.New("/etc/kubernetes/pki/ca.crt file doesn't exist") + KubeletPemNotExistErr = errors.New("/var/lib/kubelet/pki/kubelet-current.pem file doesn't exist") +) + +type kubeletCertManager struct { + kubeConfFile string + kubeletCAFile string + kubeletPemFile string + cert *tls.Certificate +} + +func NewKubeletCertManager(kubeConfFile, kubeletCAFile, kubeletPemFile string) (certificate.YurtClientCertificateManager, error) { + if exist, _ := util.FileExists(kubeConfFile); !exist { + return nil, KubeConfNotExistErr + } + + if exist, _ := util.FileExists(kubeletCAFile); !exist { + return nil, KubeletCANotExistErr + } + + if exist, _ := util.FileExists(kubeletPemFile); !exist { + return nil, KubeletPemNotExistErr + } + + cert, err := loadFile(kubeletPemFile) + if err != nil { + return nil, err + } + + return &kubeletCertManager{ + kubeConfFile: kubeConfFile, + kubeletCAFile: kubeletCAFile, + kubeletPemFile: kubeletPemFile, + cert: cert, + }, nil +} + +func (kcm *kubeletCertManager) Start() { + // do nothing +} + +func (kcm *kubeletCertManager) Stop() { + // do nothing +} + +func (kcm *kubeletCertManager) UpdateBootstrapConf(_ string) error { + return nil +} + +func (kcm *kubeletCertManager) GetHubConfFile() string { + return kcm.kubeConfFile +} + +func (kcm *kubeletCertManager) GetCaFile() string { + return kcm.kubeletCAFile +} + +func (kcm *kubeletCertManager) GetAPIServerClientCert() *tls.Certificate { + if kcm.cert != nil && kcm.cert.Leaf != nil && !time.Now().After(kcm.cert.Leaf.NotAfter) { + return kcm.cert + } + + klog.Warningf("current certificate: %s is expired, reload it", kcm.kubeletPemFile) + cert, err := loadFile(kcm.kubeletPemFile) + if err != nil { + klog.Errorf("failed to load client certificate(%s), %v", kcm.kubeletPemFile, err) + return nil + } + kcm.cert = cert + return kcm.cert +} + +func loadFile(pairFile string) (*tls.Certificate, error) { + // LoadX509KeyPair knows how to parse combined cert and private key from + // the same file. + cert, err := tls.LoadX509KeyPair(pairFile, pairFile) + if err != nil { + return nil, fmt.Errorf("could not convert data from %q into cert/key pair: %v", pairFile, err) + } + certs, err := x509.ParseCertificates(cert.Certificate[0]) + if err != nil { + return nil, fmt.Errorf("unable to parse certificate data: %v", err) + } + cert.Leaf = certs[0] + return &cert, nil +} diff --git a/pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate_test.go b/pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate_test.go new file mode 100644 index 00000000000..c9e46a72506 --- /dev/null +++ b/pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate_test.go @@ -0,0 +1,59 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeletcertificate + +import "testing" + +func TestNewKubeletCertManager(t *testing.T) { + testcases := map[string]struct { + kubeConfFile string + kubeletCAFile string + kubeletPemFile string + err error + }{ + "kubelet.conf doesn't exist": { + kubeConfFile: "invalid file", + err: KubeConfNotExistErr, + }, + "ca.crt file doesn't exist": { + kubeConfFile: "../testdata/kubelet.conf", + kubeletCAFile: "invalid file", + err: KubeletCANotExistErr, + }, + "kubelet.pem doesn't exist": { + kubeConfFile: "../testdata/kubelet.conf", + kubeletCAFile: "../testdata/ca.crt", + kubeletPemFile: "invalid file", + err: KubeletPemNotExistErr, + }, + "normal kubelet cert manager": { + kubeConfFile: "../testdata/kubelet.conf", + kubeletCAFile: "../testdata/ca.crt", + kubeletPemFile: "../testdata/kubelet.pem", + err: nil, + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + _, err := NewKubeletCertManager(tc.kubeConfFile, tc.kubeletCAFile, tc.kubeletPemFile) + if err != tc.err { + t.Errorf("expect error is %v, but got %v", tc.err, err) + } + }) + } +} diff --git a/pkg/yurthub/certificate/manager/manager.go b/pkg/yurthub/certificate/manager/manager.go new file mode 100644 index 00000000000..9d1f8cfff40 --- /dev/null +++ b/pkg/yurthub/certificate/manager/manager.go @@ -0,0 +1,137 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manager + +import ( + "errors" + "net" + "net/url" + "path/filepath" + + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/cmd/yurthub/app/options" + "github.com/openyurtio/openyurt/pkg/projectinfo" + ipUtils "github.com/openyurtio/openyurt/pkg/util/ip" + hubCert "github.com/openyurtio/openyurt/pkg/yurthub/certificate" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/kubeletcertificate" + hubServerCert "github.com/openyurtio/openyurt/pkg/yurthub/certificate/server" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token" + "github.com/openyurtio/openyurt/pkg/yurthub/util" +) + +const ( + KubeConfFile = "/etc/kubernetes/kubelet.conf" + KubeletCAFile = "/etc/kubernetes/pki/ca.crt" + KubeletPemFile = "/var/lib/kubelet/pki/current-kubelet.pem" +) + +var ( + serverCertNotReadyError = errors.New("hub server certificate") + apiServerClientCertNotReadyError = errors.New("APIServer client certificate") + caCertIsNotReadyError = errors.New("ca.crt file") + + DefaultRootDir = "/var/lib" +) + +type yurtHubCertManager struct { + hubCert.YurtClientCertificateManager + hubCert.YurtServerCertificateManager +} + +// NewYurtHubCertManager new a YurtCertificateManager instance +func NewYurtHubCertManager(options *options.YurtHubOptions, remoteServers []*url.URL) (hubCert.YurtCertificateManager, error) { + var clientCertManager hubCert.YurtClientCertificateManager + var err error + + workDir := filepath.Join(options.RootDir, projectinfo.GetHubName()) + if len(options.RootDir) == 0 { + workDir = filepath.Join(DefaultRootDir, projectinfo.GetHubName()) + } + + if options.BootstrapMode == "kubeletcertificate" { + clientCertManager, err = kubeletcertificate.NewKubeletCertManager(KubeConfFile, KubeletCAFile, KubeletPemFile) + if err != nil { + return nil, err + } + } else { + cfg := &token.ClientCertificateManagerConfiguration{ + WorkDir: workDir, + NodeName: options.NodeName, + JoinToken: options.JoinToken, + BootstrapFile: options.BootstrapFile, + CaCertHashes: options.CACertHashes, + YurtHubCertOrganizations: options.YurtHubCertOrganizations, + RemoteServers: remoteServers, + Client: options.ClientForTest, + } + clientCertManager, err = token.NewYurtHubClientCertManager(cfg) + if err != nil { + return nil, err + } + } + + // use dummy ip and bind ip as cert IP SANs + certIPs := ipUtils.RemoveDupIPs([]net.IP{ + net.ParseIP(options.HubAgentDummyIfIP), + net.ParseIP(options.YurtHubHost), + net.ParseIP(options.YurtHubProxyHost), + }) + serverCertManager, err := hubServerCert.NewHubServerCertificateManager(options.ClientForTest, clientCertManager, options.NodeName, filepath.Join(workDir, "pki"), certIPs) + if err != nil { + return nil, err + } + + hubCertManager := &yurtHubCertManager{ + YurtClientCertificateManager: clientCertManager, + YurtServerCertificateManager: serverCertManager, + } + + return hubCertManager, nil +} + +func (hcm *yurtHubCertManager) Start() { + hcm.YurtClientCertificateManager.Start() + hcm.YurtServerCertificateManager.Start() +} + +func (hcm *yurtHubCertManager) Stop() { + hcm.YurtClientCertificateManager.Stop() + hcm.YurtServerCertificateManager.Stop() +} + +func (hcm *yurtHubCertManager) Ready() bool { + var errs []error + if hcm.GetAPIServerClientCert() == nil { + errs = append(errs, apiServerClientCertNotReadyError) + } + + if exist, _ := util.FileExists(hcm.YurtClientCertificateManager.GetCaFile()); !exist { + errs = append(errs, caCertIsNotReadyError) + } + + if hcm.GetHubServerCert() == nil { + errs = append(errs, serverCertNotReadyError) + } + + if len(errs) != 0 { + klog.Errorf("hub certificates are not ready: %s", utilerrors.NewAggregate(errs).Error()) + return false + } + return true +} diff --git a/pkg/yurthub/certificate/manager/manager_test.go b/pkg/yurthub/certificate/manager/manager_test.go new file mode 100644 index 00000000000..5a462c2fce4 --- /dev/null +++ b/pkg/yurthub/certificate/manager/manager_test.go @@ -0,0 +1,136 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manager + +import ( + "fmt" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/openyurtio/openyurt/cmd/yurthub/app/options" + "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/testdata" +) + +func TestGetHubServerCertFile(t *testing.T) { + nodeName := "foo" + u, _ := url.Parse("http://127.0.0.1") + remoteServers := []*url.URL{u} + testcases := map[string]struct { + rootDir string + path string + }{ + "use default root dir": { + rootDir: "", + path: filepath.Join("/var/lib", projectinfo.GetHubName(), "pki", fmt.Sprintf("%s-server-current.pem", projectinfo.GetHubName())), + }, + "define root dir": { + rootDir: "/tmp", + path: filepath.Join("/tmp", projectinfo.GetHubName(), "pki", fmt.Sprintf("%s-server-current.pem", projectinfo.GetHubName())), + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + opt := &options.YurtHubOptions{ + NodeName: nodeName, + YurtHubHost: "127.0.0.1", + RootDir: tc.rootDir, + } + + mgr, err := NewYurtHubCertManager(opt, remoteServers) + if err != nil { + t.Errorf("failed to new cert manager, %v", err) + } + + if mgr.GetHubServerCertFile() != tc.path { + t.Errorf("expect hub server cert file %s, but got %s", tc.path, mgr.GetHubServerCertFile()) + } + }) + } +} + +var ( + joinToken = "123456.abcdef1234567890" + rootDir = "/tmp/token/cert" +) + +func TestReady(t *testing.T) { + nodeName := "foo" + u, _ := url.Parse("http://127.0.0.1") + remoteServers := []*url.URL{u} + + client, err := testdata.CreateCertFakeClient("../testdata") + if err != nil { + t.Errorf("failed to create cert fake client, %v", err) + return + } + + mgr, err := NewYurtHubCertManager(&options.YurtHubOptions{ + NodeName: nodeName, + YurtHubHost: "127.0.0.1", + RootDir: rootDir, + JoinToken: joinToken, + YurtHubCertOrganizations: []string{"yurthub:tenant:foo"}, + ClientForTest: client, + }, remoteServers) + if err != nil { + t.Errorf("failed to new yurt cert manager, %v", err) + return + } + mgr.Start() + + err = wait.PollImmediate(2*time.Second, 1*time.Minute, func() (done bool, err error) { + if mgr.Ready() { + return true, nil + } + return false, nil + }) + + if err != nil { + t.Errorf("certificates are not ready, %v", err) + } + + mgr.Stop() + + // reuse the config and ca file + t.Logf("go to check the reuse of config and ca file") + newMgr, err := NewYurtHubCertManager(&options.YurtHubOptions{ + NodeName: nodeName, + YurtHubHost: "127.0.0.1", + RootDir: rootDir, + JoinToken: joinToken, + YurtHubCertOrganizations: []string{"yurthub:tenant:foo"}, + ClientForTest: client, + }, remoteServers) + if err != nil { + t.Errorf("failed to new another yurt cert manager, %v", err) + return + } + newMgr.Start() + if !newMgr.Ready() { + t.Errorf("certificates can not be reused") + } + newMgr.Stop() + + os.RemoveAll(rootDir) +} diff --git a/pkg/yurthub/certificate/server/server.go b/pkg/yurthub/certificate/server/server.go new file mode 100644 index 00000000000..eae2f136e9a --- /dev/null +++ b/pkg/yurthub/certificate/server/server.go @@ -0,0 +1,102 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "crypto/tls" + "fmt" + "net" + "time" + + "github.com/pkg/errors" + certificatesv1 "k8s.io/api/certificates/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/authentication/user" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/certificate" + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/pkg/projectinfo" + yurtutil "github.com/openyurtio/openyurt/pkg/util" + certfactory "github.com/openyurtio/openyurt/pkg/util/certmanager/factory" + "github.com/openyurtio/openyurt/pkg/util/certmanager/store" + kubeconfigutil "github.com/openyurtio/openyurt/pkg/util/kubeconfig" + hubCert "github.com/openyurtio/openyurt/pkg/yurthub/certificate" +) + +type hubServerCertificateManager struct { + hubServerCertManager certificate.Manager + hubServerCertStore certificate.FileStore +} + +func NewHubServerCertificateManager(client clientset.Interface, clientCertManager hubCert.YurtClientCertificateManager, nodeName, pkiDir string, certIPs []net.IP) (hubCert.YurtServerCertificateManager, error) { + hubServerCertStore, err := store.NewFileStoreWrapper(fmt.Sprintf("%s-server", projectinfo.GetHubName()), pkiDir, pkiDir, "", "") + if err != nil { + return nil, errors.Wrap(err, "couldn't new hub server cert store") + } + + kubeClientFn := func(current *tls.Certificate) (clientset.Interface, error) { + // waiting for the certificate is generated + _ = wait.PollInfinite(5*time.Second, func() (bool, error) { + // keep polling until the yurthub client certificate is signed + if clientCertManager.GetAPIServerClientCert() != nil { + return true, nil + } + klog.Infof("waiting for the controller-manager to sign the %s client certificate", projectinfo.GetHubName()) + return false, nil + }) + + if !yurtutil.IsNil(client) { + return client, nil + } + + return kubeconfigutil.ClientSetFromFile(clientCertManager.GetHubConfFile()) + } + + hubServerCertManager, sErr := certfactory.NewCertManagerFactoryWithFnAndStore(kubeClientFn, hubServerCertStore).New(&certfactory.CertManagerConfig{ + ComponentName: fmt.Sprintf("%s-server", projectinfo.GetHubName()), + SignerName: certificatesv1.KubeletServingSignerName, + ForServerUsage: true, + CommonName: fmt.Sprintf("system:node:%s", nodeName), + Organizations: []string{user.NodesGroup}, + IPs: certIPs, + }) + if sErr != nil { + return nil, sErr + } + + return &hubServerCertificateManager{ + hubServerCertManager: hubServerCertManager, + hubServerCertStore: hubServerCertStore, + }, nil +} + +func (hcm *hubServerCertificateManager) Start() { + hcm.hubServerCertManager.Start() +} + +func (hcm *hubServerCertificateManager) Stop() { + hcm.hubServerCertManager.Stop() +} + +func (hcm *hubServerCertificateManager) GetHubServerCert() *tls.Certificate { + return hcm.hubServerCertManager.Current() +} + +func (hcm *hubServerCertificateManager) GetHubServerCertFile() string { + return hcm.hubServerCertStore.CurrentPath() +} diff --git a/pkg/yurthub/certificate/server/server_test.go b/pkg/yurthub/certificate/server/server_test.go new file mode 100644 index 00000000000..eaa4257f787 --- /dev/null +++ b/pkg/yurthub/certificate/server/server_test.go @@ -0,0 +1,55 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/openyurtio/openyurt/pkg/projectinfo" +) + +func TestGetHubServerCertFile(t *testing.T) { + nodeName := "foo" + testcases := map[string]struct { + rootDir string + path string + }{ + "use default root dir": { + rootDir: filepath.Join("/var/lib", projectinfo.GetHubName(), "pki"), + path: filepath.Join("/var/lib", projectinfo.GetHubName(), "pki", fmt.Sprintf("%s-server-current.pem", projectinfo.GetHubName())), + }, + "define root dir": { + rootDir: "/tmp/pki", + path: filepath.Join("/tmp", "pki", fmt.Sprintf("%s-server-current.pem", projectinfo.GetHubName())), + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + mgr, err := NewHubServerCertificateManager(nil, nil, nodeName, tc.rootDir, nil) + if err != nil { + t.Errorf("failed to new cert manager, %v", err) + } + + if mgr.GetHubServerCertFile() != tc.path { + t.Errorf("expect hub server cert file %s, but got %s", tc.path, mgr.GetHubServerCertFile()) + } + }) + } +} diff --git a/pkg/yurthub/certificate/token/testdata/ca.crt b/pkg/yurthub/certificate/testdata/ca.crt similarity index 100% rename from pkg/yurthub/certificate/token/testdata/ca.crt rename to pkg/yurthub/certificate/testdata/ca.crt diff --git a/pkg/yurthub/certificate/token/testdata/ca.key b/pkg/yurthub/certificate/testdata/ca.key similarity index 100% rename from pkg/yurthub/certificate/token/testdata/ca.key rename to pkg/yurthub/certificate/testdata/ca.key diff --git a/pkg/yurthub/certificate/token/testdata/fake_client.go b/pkg/yurthub/certificate/testdata/fake_client.go similarity index 100% rename from pkg/yurthub/certificate/token/testdata/fake_client.go rename to pkg/yurthub/certificate/testdata/fake_client.go diff --git a/pkg/yurthub/certificate/testdata/kubelet.conf b/pkg/yurthub/certificate/testdata/kubelet.conf new file mode 100644 index 00000000000..81f572fffa5 --- /dev/null +++ b/pkg/yurthub/certificate/testdata/kubelet.conf @@ -0,0 +1,14 @@ +apiVersion: v1 +clusters: +- cluster: + server: http://127.0.0.1:10261 + name: default-cluster +contexts: +- context: + cluster: default-cluster + namespace: default + user: default-auth + name: default-context +current-context: default-context +kind: Config +preferences: {} \ No newline at end of file diff --git a/pkg/yurthub/certificate/testdata/kubelet.pem b/pkg/yurthub/certificate/testdata/kubelet.pem new file mode 100644 index 00000000000..adb600c9a01 --- /dev/null +++ b/pkg/yurthub/certificate/testdata/kubelet.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIICtDCCAZygAwIBAgIRAO5qKHQa5BXX7iEe6CWQ3pAwDQYJKoZIhvcNAQELBQAw +PjEnMA8GA1UEChMIaGFuZ3pob3UwFAYDVQQKEw1hbGliYWJhIGNsb3VkMRMwEQYD +VQQDEwprdWJlcm5ldGVzMB4XDTIzMDcxOTA2MzMwOFoXDTMzMDcxNjA2MzMwOFow +STEVMBMGA1UEChMMc3lzdGVtOm5vZGVzMTAwLgYDVQQDEydzeXN0ZW06bm9kZTpp +LTV5aHE3YnIwZG5rdjgwcXZmM3hyZnUzYXAwWTATBgcqhkjOPQIBBggqhkjOPQMB +BwNCAARyoup5dNsDp+GOT0nNyowfSp85coVhJ275rqrZOIHIBlhvzJCezK1PVe4r +J9QzzJC03pwl6xoFsFvfI6UG8+G1o20wazAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0l +BAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAfBgNVHSMEGDAWgBQtE/d8pSus +mZzIdzVsFkKmxhhvpjAVBgNVHREEDjAMhwSp/gIBhwR/AAABMA0GCSqGSIb3DQEB +CwUAA4IBAQAoRPEFz1mPq/UzLzSMvxIbmz+FiPH3kuX3/j3FAN+kVelz7MeW/L5/ +HvoxfXWXKm+C0XczNj8Oo2GayNCh4VnHdoWIE2d4XMxZsH1PCJjYHTthJ6WQD1b+ +29VxBQXSthx1WumYkCMDWEduTnTsN3jAbayYBAAWvz+qgBn1Lb/HpJofSCnrZ5je +n596LOHS0UXZDyO5aNVXq0+hydtk/KdR33iA1Tp3X16wqM4C7xeKHixwbYucuUyV +/0P5S0UPp9V03V86xe9p0VHm//1CIS00/wuOh7ituWWX7CU68ZyxyGirAr8r4ojF +wqhGDLZ1ek34ufoADQut/XIhpdeYjNq7 +-----END CERTIFICATE----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIJ5xeWx4KNT2gjD4GgWATD19ZSJ00BHQHO61vqYziWm7oAoGCCqGSM49 +AwEHoUQDQgAEcqLqeXTbA6fhjk9JzcqMH0qfOXKFYSdu+a6q2TiByAZYb8yQnsyt +T1XuKyfUM8yQtN6cJesaBbBb3yOlBvPhtQ== +-----END EC PRIVATE KEY----- \ No newline at end of file diff --git a/pkg/yurthub/certificate/token/token.go b/pkg/yurthub/certificate/token/token.go index 6ce163ce2cc..459470e5b41 100644 --- a/pkg/yurthub/certificate/token/token.go +++ b/pkg/yurthub/certificate/token/token.go @@ -28,8 +28,6 @@ import ( "github.com/pkg/errors" certificatesv1 "k8s.io/api/certificates/v1" - utilerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/user" clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" @@ -51,39 +49,32 @@ import ( const ( YurtHubCSROrg = "openyurt:yurthub" - DefaultRootDir = "/var/lib" hubPkiDirName = "pki" hubCaFileName = "ca.crt" bootstrapConfigFileName = "bootstrap-hub.conf" ) var ( - hubConfigFileName = fmt.Sprintf("%s.conf", projectinfo.GetHubName()) - serverCertNotReadyError = errors.New("hub server certificate") - apiServerClientCertNotReadyError = errors.New("APIServer client certificate") - caCertIsNotReadyError = errors.New("ca.crt file") + hubConfigFileName = fmt.Sprintf("%s.conf", projectinfo.GetHubName()) ) -type CertificateManagerConfiguration struct { - RootDir string +type ClientCertificateManagerConfiguration struct { + WorkDir string NodeName string JoinToken string BootstrapFile string CaCertHashes []string YurtHubCertOrganizations []string - CertIPs []net.IP RemoteServers []*url.URL Client clientset.Interface } -type yurtHubCertManager struct { +type yurtHubClientCertManager struct { client clientset.Interface remoteServers []*url.URL caCertHashes []string apiServerClientCertManager certificate.Manager - hubServerCertManager certificate.Manager apiServerClientCertStore certificate.FileStore - hubServerCertStore certificate.FileStore hubRunDir string hubName string joinToken string @@ -91,19 +82,13 @@ type yurtHubCertManager struct { dialer *util.Dialer } -// NewYurtHubCertManager new a YurtCertificateManager instance -func NewYurtHubCertManager(cfg *CertificateManagerConfiguration) (hubCert.YurtCertificateManager, error) { +// NewYurtHubClientCertManager new a YurtCertificateManager instance +func NewYurtHubClientCertManager(cfg *ClientCertificateManagerConfiguration) (hubCert.YurtClientCertificateManager, error) { var err error - - hubRunDir := cfg.RootDir - if len(cfg.RootDir) == 0 { - hubRunDir = filepath.Join(DefaultRootDir, projectinfo.GetHubName()) - } - - ycm := &yurtHubCertManager{ + ycm := &yurtHubClientCertManager{ client: cfg.Client, remoteServers: cfg.RemoteServers, - hubRunDir: hubRunDir, + hubRunDir: cfg.WorkDir, hubName: projectinfo.GetHubName(), joinToken: cfg.JoinToken, bootstrapFile: cfg.BootstrapFile, @@ -124,17 +109,6 @@ func NewYurtHubCertManager(cfg *CertificateManagerConfiguration) (hubCert.YurtCe return ycm, errors.Wrap(err, "couldn't new apiserver client certificate manager") } - // 3. prepare yurthub server certificate manager - ycm.hubServerCertStore, err = store.NewFileStoreWrapper(fmt.Sprintf("%s-server", ycm.hubName), ycm.getPkiDir(), ycm.getPkiDir(), "", "") - if err != nil { - return ycm, errors.Wrap(err, "couldn't new hub server cert store") - } - - ycm.hubServerCertManager, err = ycm.newHubServerCertificateManager(ycm.hubServerCertStore, cfg.NodeName, cfg.CertIPs) - if err != nil { - return ycm, errors.Wrap(err, "couldn't new hub server certificate manager") - } - return ycm, nil } @@ -152,7 +126,7 @@ func removeDirContents(dir string) error { return nil } -func (ycm *yurtHubCertManager) verifyServerAddrOrCleanup(servers []*url.URL) { +func (ycm *yurtHubClientCertManager) verifyServerAddrOrCleanup(servers []*url.URL) { if cfg, err := clientcmd.LoadFromFile(ycm.GetHubConfFile()); err == nil { cluster := kubeconfigutil.GetClusterFromKubeConfig(cfg) if serverURL, err := url.Parse(cluster.Server); err != nil { @@ -172,7 +146,7 @@ func (ycm *yurtHubCertManager) verifyServerAddrOrCleanup(servers []*url.URL) { } // Start init certificate manager and certs for hub agent -func (ycm *yurtHubCertManager) Start() { +func (ycm *yurtHubClientCertManager) Start() { err := ycm.prepareConfigAndCaFile() if err != nil { klog.Errorf("failed to prepare config and ca file, %v", err) @@ -180,7 +154,6 @@ func (ycm *yurtHubCertManager) Start() { } ycm.apiServerClientCertManager.Start() - ycm.hubServerCertManager.Start() } // prepareConfigAndCaFile is used to create the following three files. @@ -188,7 +161,7 @@ func (ycm *yurtHubCertManager) Start() { // - /var/lib/yurthub/yurthub.conf // - /var/lib/yurthub/pki/ca.crt // if these files already exist, just reuse them. -func (ycm *yurtHubCertManager) prepareConfigAndCaFile() error { +func (ycm *yurtHubClientCertManager) prepareConfigAndCaFile() error { var tlsBootstrapCfg *clientcmdapi.Config var hubKubeConfig *clientcmdapi.Config var err error @@ -288,46 +261,23 @@ func (ycm *yurtHubCertManager) prepareConfigAndCaFile() error { } // Stop the cert manager loop -func (ycm *yurtHubCertManager) Stop() { +func (ycm *yurtHubClientCertManager) Stop() { ycm.apiServerClientCertManager.Stop() - ycm.hubServerCertManager.Stop() -} - -// Ready is used for checking client/server/ca certificates are prepared completely or not. -func (ycm *yurtHubCertManager) Ready() bool { - var errs []error - if ycm.GetHubServerCert() == nil { - errs = append(errs, serverCertNotReadyError) - } - - if ycm.GetAPIServerClientCert() == nil { - errs = append(errs, apiServerClientCertNotReadyError) - } - - if exist, _ := util.FileExists(ycm.GetCaFile()); !exist { - errs = append(errs, caCertIsNotReadyError) - } - - if len(errs) != 0 { - klog.Errorf("hub certificates are not ready: %s", utilerrors.NewAggregate(errs).Error()) - return false - } - return true } // UpdateBootstrapConf is used for revising bootstrap conf file by new bearer token. -func (ycm *yurtHubCertManager) UpdateBootstrapConf(joinToken string) error { +func (ycm *yurtHubClientCertManager) UpdateBootstrapConf(joinToken string) error { _, err := ycm.retrieveHubBootstrapConfig(joinToken) return err } // getPkiDir returns the directory for storing hub agent pki -func (ycm *yurtHubCertManager) getPkiDir() string { +func (ycm *yurtHubClientCertManager) getPkiDir() string { return filepath.Join(ycm.hubRunDir, hubPkiDirName) } // getBootstrapConfFile returns the path of yurthub bootstrap conf file -func (ycm *yurtHubCertManager) getBootstrapConfFile() string { +func (ycm *yurtHubClientCertManager) getBootstrapConfFile() string { if len(ycm.bootstrapFile) != 0 { return ycm.bootstrapFile } @@ -335,30 +285,22 @@ func (ycm *yurtHubCertManager) getBootstrapConfFile() string { } // GetCaFile returns the path of ca file -func (ycm *yurtHubCertManager) GetCaFile() string { +func (ycm *yurtHubClientCertManager) GetCaFile() string { return filepath.Join(ycm.getPkiDir(), hubCaFileName) } // GetHubConfFile returns the path of yurtHub config file path -func (ycm *yurtHubCertManager) GetHubConfFile() string { +func (ycm *yurtHubClientCertManager) GetHubConfFile() string { return filepath.Join(ycm.hubRunDir, hubConfigFileName) } -func (ycm *yurtHubCertManager) GetAPIServerClientCert() *tls.Certificate { +func (ycm *yurtHubClientCertManager) GetAPIServerClientCert() *tls.Certificate { return ycm.apiServerClientCertManager.Current() } -func (ycm *yurtHubCertManager) GetHubServerCert() *tls.Certificate { - return ycm.hubServerCertManager.Current() -} - -func (ycm *yurtHubCertManager) GetHubServerCertFile() string { - return ycm.hubServerCertStore.CurrentPath() -} - // newAPIServerClientCertificateManager create a certificate manager for yurthub component to prepare client certificate // that used to proxy requests to remote kube-apiserver. -func (ycm *yurtHubCertManager) newAPIServerClientCertificateManager(fileStore certificate.FileStore, nodeName string, hubCertOrganizations []string) (certificate.Manager, error) { +func (ycm *yurtHubClientCertManager) newAPIServerClientCertificateManager(fileStore certificate.FileStore, nodeName string, hubCertOrganizations []string) (certificate.Manager, error) { orgs := []string{YurtHubCSROrg, user.NodesGroup} for _, v := range hubCertOrganizations { if v != YurtHubCSROrg && v != user.NodesGroup { @@ -374,7 +316,7 @@ func (ycm *yurtHubCertManager) newAPIServerClientCertificateManager(fileStore ce }) } -func (ycm *yurtHubCertManager) generateCertClientFn(current *tls.Certificate) (clientset.Interface, error) { +func (ycm *yurtHubClientCertManager) generateCertClientFn(current *tls.Certificate) (clientset.Interface, error) { var kubeconfig *restclient.Config var err error if !yurtutil.IsNil(ycm.client) { @@ -412,39 +354,7 @@ func (ycm *yurtHubCertManager) generateCertClientFn(current *tls.Certificate) (c return clientset.NewForConfig(kubeconfig) } -// newHubServerCertificateManager create a certificate manager for yurthub component to prepare server certificate -// that used to handle requests from clients on edge nodes. -func (ycm *yurtHubCertManager) newHubServerCertificateManager(fileStore certificate.FileStore, nodeName string, certIPs []net.IP) (certificate.Manager, error) { - kubeClientFn := func(current *tls.Certificate) (clientset.Interface, error) { - // waiting for the certificate is generated - _ = wait.PollInfinite(5*time.Second, func() (bool, error) { - // keep polling until the yurthub client certificate is signed - if ycm.apiServerClientCertManager.Current() != nil { - return true, nil - } - klog.Infof("waiting for the controller-manager to sign the %s client certificate", ycm.hubName) - return false, nil - }) - - if !yurtutil.IsNil(ycm.client) { - return ycm.client, nil - } - - return kubeconfigutil.ClientSetFromFile(ycm.GetHubConfFile()) - } - - // create a certificate manager for the yurthub server and run the csr approver for both yurthub - return certfactory.NewCertManagerFactoryWithFnAndStore(kubeClientFn, fileStore).New(&certfactory.CertManagerConfig{ - ComponentName: fmt.Sprintf("%s-server", ycm.hubName), - SignerName: certificatesv1.KubeletServingSignerName, - ForServerUsage: true, - CommonName: fmt.Sprintf("system:node:%s", nodeName), - Organizations: []string{user.NodesGroup}, - IPs: certIPs, - }) -} - -func (ycm *yurtHubCertManager) retrieveHubBootstrapConfig(joinToken string) (*clientcmdapi.Config, error) { +func (ycm *yurtHubClientCertManager) retrieveHubBootstrapConfig(joinToken string) (*clientcmdapi.Config, error) { // retrieve bootstrap config info from cluster-info configmap by bootstrap token serverAddr := findActiveRemoteServer(ycm.remoteServers).Host if cfg, err := token.RetrieveValidatedConfigInfo(ycm.client, &token.BootstrapData{ diff --git a/pkg/yurthub/certificate/token/token_test.go b/pkg/yurthub/certificate/token/token_test.go index eebc1856340..8a9d07a1ae4 100644 --- a/pkg/yurthub/certificate/token/token_test.go +++ b/pkg/yurthub/certificate/token/token_test.go @@ -18,17 +18,13 @@ package token import ( "fmt" - "net" "net/url" "os" "path/filepath" "testing" - "time" - - "k8s.io/apimachinery/pkg/util/wait" "github.com/openyurtio/openyurt/pkg/projectinfo" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token/testdata" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/testdata" ) func Test_removeDirContents(t *testing.T) { @@ -78,31 +74,29 @@ func TestGetHubConfFile(t *testing.T) { nodeName := "foo" u, _ := url.Parse("http://127.0.0.1") remoteServers := []*url.URL{u} - certIPs := []net.IP{net.ParseIP("127.0.0.1")} testcases := map[string]struct { - rootDir string + workDir string path string }{ "use default root dir": { - rootDir: "", + workDir: filepath.Join("/var/lib", projectinfo.GetHubName()), path: filepath.Join("/var/lib", projectinfo.GetHubName(), fmt.Sprintf("%s.conf", projectinfo.GetHubName())), }, "define root dir": { - rootDir: "/tmp", + workDir: "/tmp", path: filepath.Join("/tmp", fmt.Sprintf("%s.conf", projectinfo.GetHubName())), }, } for k, tc := range testcases { t.Run(k, func(t *testing.T) { - cfg := &CertificateManagerConfiguration{ + cfg := &ClientCertificateManagerConfiguration{ NodeName: nodeName, RemoteServers: remoteServers, - CertIPs: certIPs, - RootDir: tc.rootDir, + WorkDir: tc.workDir, } - mgr, err := NewYurtHubCertManager(cfg) + mgr, err := NewYurtHubClientCertManager(cfg) if err != nil { t.Errorf("failed to new cert manager, %v", err) } @@ -118,31 +112,29 @@ func TestGetCaFile(t *testing.T) { nodeName := "foo" u, _ := url.Parse("http://127.0.0.1") remoteServers := []*url.URL{u} - certIPs := []net.IP{net.ParseIP("127.0.0.1")} testcases := map[string]struct { - rootDir string + workDir string path string }{ "use default root dir": { - rootDir: "", + workDir: filepath.Join("/var/lib", projectinfo.GetHubName()), path: filepath.Join("/var/lib", projectinfo.GetHubName(), "pki", "ca.crt"), }, "define root dir": { - rootDir: "/tmp", + workDir: "/tmp", path: filepath.Join("/tmp", "pki", "ca.crt"), }, } for k, tc := range testcases { t.Run(k, func(t *testing.T) { - cfg := &CertificateManagerConfiguration{ + cfg := &ClientCertificateManagerConfiguration{ NodeName: nodeName, RemoteServers: remoteServers, - CertIPs: certIPs, - RootDir: tc.rootDir, + WorkDir: tc.workDir, } - mgr, err := NewYurtHubCertManager(cfg) + mgr, err := NewYurtHubClientCertManager(cfg) if err != nil { t.Errorf("failed to new cert manager, %v", err) } @@ -154,56 +146,12 @@ func TestGetCaFile(t *testing.T) { } } -func TestGetHubServerCertFile(t *testing.T) { - nodeName := "foo" - u, _ := url.Parse("http://127.0.0.1") - remoteServers := []*url.URL{u} - certIPs := []net.IP{net.ParseIP("127.0.0.1")} - testcases := map[string]struct { - rootDir string - path string - }{ - "use default root dir": { - rootDir: "", - path: filepath.Join("/var/lib", projectinfo.GetHubName(), "pki", fmt.Sprintf("%s-server-current.pem", projectinfo.GetHubName())), - }, - "define root dir": { - rootDir: "/tmp", - path: filepath.Join("/tmp", "pki", fmt.Sprintf("%s-server-current.pem", projectinfo.GetHubName())), - }, - } - - for k, tc := range testcases { - t.Run(k, func(t *testing.T) { - cfg := &CertificateManagerConfiguration{ - NodeName: nodeName, - RemoteServers: remoteServers, - CertIPs: certIPs, - RootDir: tc.rootDir, - } - - mgr, err := NewYurtHubCertManager(cfg) - if err != nil { - t.Errorf("failed to new cert manager, %v", err) - } - - if mgr.GetHubServerCertFile() != tc.path { - t.Errorf("expect hub server cert file %s, but got %s", tc.path, mgr.GetHubServerCertFile()) - } - }) - } -} - -var ( - joinToken = "123456.abcdef1234567890" - rootDir = "/tmp/token/cert" -) - func TestUpdateBootstrapConf(t *testing.T) { + joinToken := "123456.abcdef1234567890" + workDir := "/tmp/token/cert" nodeName := "foo" u, _ := url.Parse("http://127.0.0.1") remoteServers := []*url.URL{u} - certIPs := []net.IP{net.ParseIP("127.0.0.1")} testcases := map[string]struct { joinToken string err error @@ -216,17 +164,16 @@ func TestUpdateBootstrapConf(t *testing.T) { for k, tc := range testcases { t.Run(k, func(t *testing.T) { - client, err := testdata.CreateCertFakeClient("./testdata") + client, err := testdata.CreateCertFakeClient("../testdata") if err != nil { t.Errorf("failed to create cert fake client, %v", err) return } - mgr, err := NewYurtHubCertManager(&CertificateManagerConfiguration{ + mgr, err := NewYurtHubClientCertManager(&ClientCertificateManagerConfiguration{ NodeName: nodeName, RemoteServers: remoteServers, - CertIPs: certIPs, - RootDir: rootDir, + WorkDir: workDir, JoinToken: tc.joinToken, Client: client, }) @@ -242,69 +189,5 @@ func TestUpdateBootstrapConf(t *testing.T) { mgr.Stop() }) } - os.RemoveAll(rootDir) -} - -func TestReady(t *testing.T) { - nodeName := "foo" - u, _ := url.Parse("http://127.0.0.1") - remoteServers := []*url.URL{u} - certIPs := []net.IP{net.ParseIP("127.0.0.1")} - - client, err := testdata.CreateCertFakeClient("./testdata") - if err != nil { - t.Errorf("failed to create cert fake client, %v", err) - return - } - - mgr, err := NewYurtHubCertManager(&CertificateManagerConfiguration{ - NodeName: nodeName, - RemoteServers: remoteServers, - CertIPs: certIPs, - RootDir: rootDir, - JoinToken: joinToken, - YurtHubCertOrganizations: []string{"yurthub:tenant:foo"}, - Client: client, - }) - if err != nil { - t.Errorf("failed to new yurt cert manager, %v", err) - return - } - mgr.Start() - - err = wait.PollImmediate(2*time.Second, 1*time.Minute, func() (done bool, err error) { - if mgr.Ready() { - return true, nil - } - return false, nil - }) - - if err != nil { - t.Errorf("certificates are not ready, %v", err) - } - - mgr.Stop() - - // reuse the config and ca file - t.Logf("go to check the reuse of config and ca file") - newMgr, err := NewYurtHubCertManager(&CertificateManagerConfiguration{ - NodeName: nodeName, - RemoteServers: remoteServers, - CertIPs: certIPs, - RootDir: rootDir, - JoinToken: joinToken, - YurtHubCertOrganizations: []string{"yurthub:tenant:foo"}, - Client: client, - }) - if err != nil { - t.Errorf("failed to new another yurt cert manager, %v", err) - return - } - newMgr.Start() - if !newMgr.Ready() { - t.Errorf("certificates can not be reused") - } - newMgr.Stop() - - os.RemoveAll(rootDir) + os.RemoveAll(workDir) } diff --git a/pkg/yurthub/kubernetes/rest/config_test.go b/pkg/yurthub/kubernetes/rest/config_test.go index ee5c450dfe0..bf76a17da04 100644 --- a/pkg/yurthub/kubernetes/rest/config_test.go +++ b/pkg/yurthub/kubernetes/rest/config_test.go @@ -17,7 +17,6 @@ limitations under the License. package rest import ( - "net" "net/url" "os" "testing" @@ -25,8 +24,9 @@ import ( "k8s.io/apimachinery/pkg/util/wait" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token/testdata" + "github.com/openyurtio/openyurt/cmd/yurthub/app/options" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/manager" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/testdata" "github.com/openyurtio/openyurt/pkg/yurthub/healthchecker" ) @@ -39,22 +39,20 @@ func TestGetRestConfig(t *testing.T) { servers := map[string]int{"https://10.10.10.113:6443": 2} u, _ := url.Parse("https://10.10.10.113:6443") remoteServers := []*url.URL{u} - certIPs := []net.IP{net.ParseIP("127.0.0.1")} fakeHealthyChecker := healthchecker.NewFakeChecker(false, servers) - client, err := testdata.CreateCertFakeClient("../../certificate/token/testdata") + client, err := testdata.CreateCertFakeClient("../../certificate/testdata") if err != nil { t.Errorf("failed to create cert fake client, %v", err) return } - certManager, err := token.NewYurtHubCertManager(&token.CertificateManagerConfiguration{ + certManager, err := manager.NewYurtHubCertManager(&options.YurtHubOptions{ NodeName: nodeName, - RemoteServers: remoteServers, - CertIPs: certIPs, RootDir: testDir, + YurtHubHost: "127.0.0.1", JoinToken: "123456.abcdef1234567890", - Client: client, - }) + ClientForTest: client, + }, remoteServers) if err != nil { t.Errorf("failed to create certManager, %v", err) return diff --git a/pkg/yurthub/server/certificate_test.go b/pkg/yurthub/server/certificate_test.go index fe2a7a30445..2451f1b057a 100644 --- a/pkg/yurthub/server/certificate_test.go +++ b/pkg/yurthub/server/certificate_test.go @@ -19,7 +19,6 @@ package server import ( "bytes" "encoding/json" - "net" "net/http" "net/http/httptest" "net/url" @@ -29,8 +28,9 @@ import ( "k8s.io/apimachinery/pkg/util/wait" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token/testdata" + "github.com/openyurtio/openyurt/cmd/yurthub/app/options" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/manager" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/testdata" ) var ( @@ -40,20 +40,17 @@ var ( func TestUpdateTokenHandler(t *testing.T) { u, _ := url.Parse("https://10.10.10.113:6443") remoteServers := []*url.URL{u} - certIPs := []net.IP{net.ParseIP("127.0.0.1")} - client, err := testdata.CreateCertFakeClient("../certificate/token/testdata") + client, err := testdata.CreateCertFakeClient("../certificate/testdata") if err != nil { t.Errorf("failed to create cert fake client, %v", err) return } - certManager, err := token.NewYurtHubCertManager(&token.CertificateManagerConfiguration{ + certManager, err := manager.NewYurtHubCertManager(&options.YurtHubOptions{ NodeName: "foo", - RemoteServers: remoteServers, - CertIPs: certIPs, RootDir: testDir, JoinToken: "123456.abcdef1234567890", - Client: client, - }) + ClientForTest: client, + }, remoteServers) if err != nil { t.Errorf("failed to create certManager, %v", err) return @@ -63,7 +60,7 @@ func TestUpdateTokenHandler(t *testing.T) { defer os.RemoveAll(testDir) err = wait.PollImmediate(2*time.Second, 1*time.Minute, func() (done bool, err error) { - if certManager.Ready() { + if certManager.GetAPIServerClientCert() != nil { return true, nil } return false, nil From 237fe8fc0aef17265dfe47738e1721a3f96b7dc2 Mon Sep 17 00:00:00 2001 From: vie-serendipity <60083692+vie-serendipity@users.noreply.github.com> Date: Fri, 28 Jul 2023 10:15:19 +0800 Subject: [PATCH 02/21] check whether yurt manager's container is ready in e2e (#1631) * modify * modify * modify * modify * move check to deployYurtManager * modify * modify * modify --- test/e2e/cmd/init/converter.go | 13 +++++++++---- test/e2e/cmd/init/init.go | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/test/e2e/cmd/init/converter.go b/test/e2e/cmd/init/converter.go index 5f61b907afb..9108248b90e 100644 --- a/test/e2e/cmd/init/converter.go +++ b/test/e2e/cmd/init/converter.go @@ -286,13 +286,18 @@ func (c *ClusterConverter) deployYurtManager() error { if podList.Items[0].Status.Phase == corev1.PodRunning { for i := range podList.Items[0].Status.Conditions { if podList.Items[0].Status.Conditions[i].Type == corev1.PodReady && - podList.Items[0].Status.Conditions[i].Status == corev1.ConditionTrue { - return true, nil + podList.Items[0].Status.Conditions[i].Status != corev1.ConditionTrue { + klog.Infof("pod(%s/%s): %#v", podList.Items[0].Namespace, podList.Items[0].Name, podList.Items[0]) + return false, nil + } + if podList.Items[0].Status.Conditions[i].Type == corev1.ContainersReady && + podList.Items[0].Status.Conditions[i].Status != corev1.ConditionTrue { + klog.Info("yurt manager's container is not ready") + return false, nil } } } - klog.Infof("pod(%s/%s): %#v", podList.Items[0].Namespace, podList.Items[0].Name, podList.Items[0]) - return false, nil + return true, nil }) } diff --git a/test/e2e/cmd/init/init.go b/test/e2e/cmd/init/init.go index 13cb1981fd8..8e4f7753835 100644 --- a/test/e2e/cmd/init/init.go +++ b/test/e2e/cmd/init/init.go @@ -374,6 +374,7 @@ func (ki *Initializer) prepareKindConfigFile(kindConfigPath string) error { } func (ki *Initializer) configureAddons() error { + if err := ki.configureCoreDnsAddon(); err != nil { return err } From 7cf623d38e91768acfc5192f1513de379c2149a6 Mon Sep 17 00:00:00 2001 From: Zhen Zhao <413621396@qq.com> Date: Wed, 9 Aug 2023 14:55:22 +0800 Subject: [PATCH 03/21] delete configmap when yurtstaticset is deleting (#1640) --- .../yurtstaticset/yurtstaticset_controller.go | 25 ++++++- .../yurtstaticset_controller_test.go | 65 ++++++++++++++++++- 2 files changed, 87 insertions(+), 3 deletions(-) diff --git a/pkg/controller/yurtstaticset/yurtstaticset_controller.go b/pkg/controller/yurtstaticset/yurtstaticset_controller.go index 0d59909e3f8..c3469a05645 100644 --- a/pkg/controller/yurtstaticset/yurtstaticset_controller.go +++ b/pkg/controller/yurtstaticset/yurtstaticset_controller.go @@ -283,12 +283,18 @@ func (r *ReconcileYurtStaticSet) Reconcile(_ context.Context, request reconcile. // Fetch the YurtStaticSet instance instance := &appsv1alpha1.YurtStaticSet{} if err := r.Get(context.TODO(), request.NamespacedName, instance); err != nil { + // if the yurtStaticSet does not exist, delete the specified configmap if exist. + if kerr.IsNotFound(err) { + return reconcile.Result{}, r.deleteConfigMap(request.Name, request.Namespace) + } klog.Errorf("Fail to get YurtStaticSet %v, %v", request.NamespacedName, err) return ctrl.Result{}, client.IgnoreNotFound(err) } if instance.DeletionTimestamp != nil { - return reconcile.Result{}, nil + // handle the deletion event + // delete the configMap which is created by yurtStaticSet + return reconcile.Result{}, r.deleteConfigMap(request.Name, request.Namespace) } var ( @@ -541,3 +547,20 @@ func (r *ReconcileYurtStaticSet) updateYurtStaticSetStatus(instance *appsv1alpha return reconcile.Result{}, nil } + +// deleteConfigMap delete the configMap if YurtStaticSet is deleting +func (r *ReconcileYurtStaticSet) deleteConfigMap(name, namespace string) error { + cmName := util.WithConfigMapPrefix(name) + configMap := &corev1.ConfigMap{} + if err := r.Get(context.TODO(), types.NamespacedName{Name: cmName, Namespace: namespace}, configMap); err != nil { + if kerr.IsNotFound(err) { + return nil + } + return err + } + if err := r.Delete(context.TODO(), configMap, &client.DeleteOptions{}); err != nil { + return err + } + klog.Infof(Format("Delete ConfigMap %s from YurtStaticSet %s", configMap.Name, name)) + return nil +} diff --git a/pkg/controller/yurtstaticset/yurtstaticset_controller_test.go b/pkg/controller/yurtstaticset/yurtstaticset_controller_test.go index f46556561d2..76eece0d502 100644 --- a/pkg/controller/yurtstaticset/yurtstaticset_controller_test.go +++ b/pkg/controller/yurtstaticset/yurtstaticset_controller_test.go @@ -27,7 +27,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" - fakeclint "sigs.k8s.io/controller-runtime/pkg/client/fake" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -113,7 +113,7 @@ func TestReconcile(t *testing.T) { for _, s := range strategy { instance.Spec.UpgradeStrategy = s - c := fakeclint.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(instance).WithObjects(staticPods...).WithObjects(nodes...).Build() + c := fakeclient.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(instance).WithObjects(staticPods...).WithObjects(nodes...).Build() var req = reconcile.Request{NamespacedName: types.NamespacedName{Namespace: metav1.NamespaceDefault, Name: TestStaticPodName}} rsp := ReconcileYurtStaticSet{ @@ -157,3 +157,64 @@ func Test_nodeTurnReady(t *testing.T) { } }) } + +func TestReconcileYurtStaticSetDeleteConfigMap(t *testing.T) { + staticPods := prepareStaticPods() + instance := &appsv1alpha1.YurtStaticSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: TestStaticPodName, + Namespace: metav1.NamespaceDefault, + }, + Spec: appsv1alpha1.YurtStaticSetSpec{ + StaticPodManifest: "nginx", + Template: corev1.PodTemplateSpec{}, + }, + } + cmList := []client.Object{ + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "yurt-static-set-nginx", + Namespace: metav1.NamespaceDefault, + }, + }, + } + + scheme := runtime.NewScheme() + if err := appsv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal("Fail to add yurt custom resource") + } + if err := clientgoscheme.AddToScheme(scheme); err != nil { + t.Fatal("Fail to add kubernetes clint-go custom resource") + } + c := fakeclient.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(instance).WithObjects(staticPods...).WithObjects(cmList...).Build() + + tests := []struct { + name string + yssName string + namespace string + wantErr bool + }{ + { + name: "test1", + yssName: TestStaticPodName, + namespace: metav1.NamespaceDefault, + wantErr: false, + }, + { + name: "test2", + yssName: TestStaticPodName, + namespace: metav1.NamespaceDefault, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &ReconcileYurtStaticSet{ + Client: c, + } + if err := r.deleteConfigMap(tt.yssName, tt.namespace); (err != nil) != tt.wantErr { + t.Errorf("deleteConfigMap() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} From d89a805b2910431564bd6584d3c413ec8219946f Mon Sep 17 00:00:00 2001 From: Abyss <45425302+wangxye@users.noreply.github.com> Date: Wed, 9 Aug 2023 16:05:03 +0800 Subject: [PATCH 04/21] feat: move yurt-device-controller into yurt-manager (#1607) * add a new crd iot.device/deviceservice/deviceprofile and incorporate it Signed-off-by: wangxye <1031989637@qq.com> --- .github/workflows/trivy-scan.yml | 2 +- Makefile | 2 + .../crds/iot.openyurt.io_deviceprofiles.yaml | 172 ++++++ .../crds/iot.openyurt.io_devices.yaml | 195 +++++++ .../crds/iot.openyurt.io_deviceservices.yaml | 141 +++++ .../yurt-manager-auto-generated.yaml | 78 +++ .../templates/yurt-manager-iot.yaml | 88 +++ cmd/yurt-iot-dock/app/core.go | 199 +++++++ cmd/yurt-iot-dock/app/options/options.go | 82 +++ cmd/yurt-iot-dock/yurt-iot-dock.go | 43 ++ go.mod | 11 +- go.sum | 31 ++ .../build/Dockerfile.yurt-iot-dock | 8 + .../release/Dockerfile.yurt-iot-dock | 14 + hack/make-rules/build.sh | 1 + hack/make-rules/image_build.sh | 1 + hack/make-rules/kustomize_to_chart.sh | 4 + hack/make-rules/local-up-openyurt.sh | 1 + pkg/apis/iot/v1alpha1/condition_const.go | 26 + pkg/apis/iot/v1alpha1/device_types.go | 175 ++++++ pkg/apis/iot/v1alpha1/deviceprofile_types.go | 122 +++++ pkg/apis/iot/v1alpha1/deviceservice_types.go | 121 +++++ .../iot/v1alpha1/zz_generated.deepcopy.go | 506 ++++++++++++++++++ .../platformadmin/platformadmin_controller.go | 6 + pkg/controller/platformadmin/util.go | 126 +++++ .../utils/{util.go => conditions.go} | 0 pkg/controller/platformadmin/utils/version.go | 31 ++ .../clients/edgex-foundry/device_client.go | 371 +++++++++++++ .../edgex-foundry/device_client_test.go | 202 +++++++ .../edgex-foundry/deviceprofile_client.go | 141 +++++ .../deviceprofile_client_test.go | 107 ++++ .../edgex-foundry/deviceservice_client.go | 166 ++++++ .../deviceservice_client_test.go | 126 +++++ .../clients/edgex-foundry/edgexobject.go | 21 + pkg/yurtiotdock/clients/edgex-foundry/util.go | 456 ++++++++++++++++ pkg/yurtiotdock/clients/errors.go | 29 + pkg/yurtiotdock/clients/interface.go | 93 ++++ .../controllers/device_controller.go | 295 ++++++++++ pkg/yurtiotdock/controllers/device_syncer.go | 237 ++++++++ .../controllers/deviceprofile_controller.go | 165 ++++++ .../controllers/deviceprofile_syncer.go | 214 ++++++++ .../controllers/deviceservice_controller.go | 220 ++++++++ .../controllers/deviceservice_syncer.go | 237 ++++++++ pkg/yurtiotdock/controllers/predicate.go | 48 ++ .../controllers/util/conditions.go | 120 +++++ .../controllers/util/fieldindexer.go | 62 +++ pkg/yurtiotdock/controllers/util/string.go | 30 ++ .../controllers/util/string_test.go | 70 +++ pkg/yurtiotdock/controllers/util/tools.go | 99 ++++ .../controllers/util/tools_test.go | 55 ++ .../controllers/well_known_labels.go | 21 + 51 files changed, 5769 insertions(+), 2 deletions(-) create mode 100644 charts/yurt-manager/crds/iot.openyurt.io_deviceprofiles.yaml create mode 100644 charts/yurt-manager/crds/iot.openyurt.io_devices.yaml create mode 100644 charts/yurt-manager/crds/iot.openyurt.io_deviceservices.yaml create mode 100644 charts/yurt-manager/templates/yurt-manager-iot.yaml create mode 100644 cmd/yurt-iot-dock/app/core.go create mode 100644 cmd/yurt-iot-dock/app/options/options.go create mode 100644 cmd/yurt-iot-dock/yurt-iot-dock.go create mode 100644 hack/dockerfiles/build/Dockerfile.yurt-iot-dock create mode 100644 hack/dockerfiles/release/Dockerfile.yurt-iot-dock create mode 100644 pkg/apis/iot/v1alpha1/device_types.go create mode 100644 pkg/apis/iot/v1alpha1/deviceprofile_types.go create mode 100644 pkg/apis/iot/v1alpha1/deviceservice_types.go create mode 100644 pkg/controller/platformadmin/util.go rename pkg/controller/platformadmin/utils/{util.go => conditions.go} (100%) create mode 100644 pkg/controller/platformadmin/utils/version.go create mode 100644 pkg/yurtiotdock/clients/edgex-foundry/device_client.go create mode 100644 pkg/yurtiotdock/clients/edgex-foundry/device_client_test.go create mode 100644 pkg/yurtiotdock/clients/edgex-foundry/deviceprofile_client.go create mode 100644 pkg/yurtiotdock/clients/edgex-foundry/deviceprofile_client_test.go create mode 100644 pkg/yurtiotdock/clients/edgex-foundry/deviceservice_client.go create mode 100644 pkg/yurtiotdock/clients/edgex-foundry/deviceservice_client_test.go create mode 100644 pkg/yurtiotdock/clients/edgex-foundry/edgexobject.go create mode 100644 pkg/yurtiotdock/clients/edgex-foundry/util.go create mode 100644 pkg/yurtiotdock/clients/errors.go create mode 100644 pkg/yurtiotdock/clients/interface.go create mode 100644 pkg/yurtiotdock/controllers/device_controller.go create mode 100644 pkg/yurtiotdock/controllers/device_syncer.go create mode 100644 pkg/yurtiotdock/controllers/deviceprofile_controller.go create mode 100644 pkg/yurtiotdock/controllers/deviceprofile_syncer.go create mode 100644 pkg/yurtiotdock/controllers/deviceservice_controller.go create mode 100644 pkg/yurtiotdock/controllers/deviceservice_syncer.go create mode 100644 pkg/yurtiotdock/controllers/predicate.go create mode 100644 pkg/yurtiotdock/controllers/util/conditions.go create mode 100644 pkg/yurtiotdock/controllers/util/fieldindexer.go create mode 100644 pkg/yurtiotdock/controllers/util/string.go create mode 100644 pkg/yurtiotdock/controllers/util/string_test.go create mode 100644 pkg/yurtiotdock/controllers/util/tools.go create mode 100644 pkg/yurtiotdock/controllers/util/tools_test.go create mode 100644 pkg/yurtiotdock/controllers/well_known_labels.go diff --git a/.github/workflows/trivy-scan.yml b/.github/workflows/trivy-scan.yml index eb2b54867c6..31dc32bb5d2 100644 --- a/.github/workflows/trivy-scan.yml +++ b/.github/workflows/trivy-scan.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - target: [ yurthub, node-servant, yurt-manager ] + target: [ yurthub, node-servant, yurt-manager, yurt-iot-dock ] steps: - uses: actions/checkout@v3 with: diff --git a/Makefile b/Makefile index 49fe17030f4..fefb7fe0ca0 100644 --- a/Makefile +++ b/Makefile @@ -171,6 +171,8 @@ docker-push-yurt-tunnel-server: docker-buildx-builder docker-push-yurt-tunnel-agent: docker-buildx-builder docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.yurt-tunnel-agent . -t ${IMAGE_REPO}/yurt-tunnel-agent:${GIT_VERSION} +docker-push-yurt-iot-dock: docker-buildx-builder + docker buildx build --no-cache --push ${DOCKER_BUILD_ARGS} --platform ${TARGET_PLATFORMS} -f hack/dockerfiles/release/Dockerfile.yurt-iot-dock . -t ${IMAGE_REPO}/yurt-iot-dock:${GIT_VERSION} generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. # hack/make-rule/generate_openapi.sh // TODO by kadisi diff --git a/charts/yurt-manager/crds/iot.openyurt.io_deviceprofiles.yaml b/charts/yurt-manager/crds/iot.openyurt.io_deviceprofiles.yaml new file mode 100644 index 00000000000..c1d92ed7a8e --- /dev/null +++ b/charts/yurt-manager/crds/iot.openyurt.io_deviceprofiles.yaml @@ -0,0 +1,172 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + name: deviceprofiles.iot.openyurt.io +spec: + group: iot.openyurt.io + names: + kind: DeviceProfile + listKind: DeviceProfileList + plural: deviceprofiles + shortNames: + - dp + singular: deviceprofile + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The nodepool of deviceProfile + jsonPath: .spec.nodePool + name: NODEPOOL + type: string + - description: The synced status of deviceProfile + jsonPath: .status.synced + name: SYNCED + type: boolean + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: DeviceProfile represents the attributes and operational capabilities + of a device. It is a template for which there can be multiple matching devices + within a given system. NOTE This struct is derived from edgex/go-mod-core-contracts/models/deviceprofile.go + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: DeviceProfileSpec defines the desired state of DeviceProfile + properties: + description: + type: string + deviceCommands: + items: + properties: + isHidden: + type: boolean + name: + type: string + readWrite: + type: string + resourceOperations: + items: + properties: + defaultValue: + type: string + deviceResource: + type: string + mappings: + additionalProperties: + type: string + type: object + required: + - defaultValue + type: object + type: array + required: + - isHidden + - name + - readWrite + - resourceOperations + type: object + type: array + deviceResources: + items: + properties: + attributes: + additionalProperties: + type: string + type: object + description: + type: string + isHidden: + type: boolean + name: + type: string + properties: + properties: + assertion: + type: string + base: + type: string + defaultValue: + type: string + mask: + type: string + maximum: + type: string + mediaType: + type: string + minimum: + type: string + offset: + type: string + readWrite: + type: string + scale: + type: string + shift: + type: string + units: + type: string + valueType: + type: string + type: object + tag: + type: string + required: + - description + - isHidden + - name + - properties + type: object + type: array + labels: + description: Labels used to search for groups of profiles on EdgeX + Foundry + items: + type: string + type: array + manufacturer: + description: Manufacturer of the device + type: string + model: + description: Model of the device + type: string + nodePool: + description: NodePool specifies which nodePool the deviceProfile belongs + to + type: string + type: object + status: + description: DeviceProfileStatus defines the observed state of DeviceProfile + properties: + id: + type: string + synced: + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/yurt-manager/crds/iot.openyurt.io_devices.yaml b/charts/yurt-manager/crds/iot.openyurt.io_devices.yaml new file mode 100644 index 00000000000..a284a0f5b8f --- /dev/null +++ b/charts/yurt-manager/crds/iot.openyurt.io_devices.yaml @@ -0,0 +1,195 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + name: devices.iot.openyurt.io +spec: + group: iot.openyurt.io + names: + kind: Device + listKind: DeviceList + plural: devices + shortNames: + - dev + singular: device + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The nodepool of device + jsonPath: .spec.nodePool + name: NODEPOOL + type: string + - description: The synced status of device + jsonPath: .status.synced + name: SYNCED + type: boolean + - description: The managed status of device + jsonPath: .spec.managed + name: MANAGED + priority: 1 + type: boolean + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: Device is the Schema for the devices API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: DeviceSpec defines the desired state of Device + properties: + adminState: + description: Admin state (locked/unlocked) + type: string + description: + description: Information describing the device + type: string + deviceProperties: + additionalProperties: + properties: + desiredValue: + type: string + name: + type: string + putURL: + type: string + required: + - desiredValue + - name + type: object + description: TODO support the following field A list of auto-generated + events coming from the device AutoEvents []AutoEvent `json:"autoEvents"` + DeviceProperties represents the expected state of the device's properties + type: object + labels: + description: Other labels applied to the device to help with searching + items: + type: string + type: array + location: + description: 'Device service specific location (interface{} is an + empty interface so it can be anything) TODO: location type in edgex + is interface{}' + type: string + managed: + description: True means device is managed by cloud, cloud can update + the related fields False means cloud can't update the fields + type: boolean + nodePool: + description: NodePool indicates which nodePool the device comes from + type: string + notify: + type: boolean + operatingState: + description: Operating state (enabled/disabled) + type: string + profileName: + description: Associated Device Profile - Describes the device + type: string + protocols: + additionalProperties: + additionalProperties: + type: string + type: object + description: A map of supported protocols for the given device + type: object + serviceName: + description: Associated Device Service - One per device + type: string + required: + - notify + - profileName + - serviceName + type: object + status: + description: DeviceStatus defines the observed state of Device + properties: + adminState: + description: Admin state (locked/unlocked) + type: string + conditions: + description: current device state + items: + description: DeviceCondition describes current state of a Device. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of in place set condition. + type: string + type: object + type: array + deviceProperties: + additionalProperties: + properties: + actualValue: + type: string + getURL: + type: string + name: + type: string + required: + - actualValue + - name + type: object + description: it represents the actual state of the device's properties + type: object + edgeId: + type: string + lastConnected: + description: Time (milliseconds) that the device last provided any + feedback or responded to any request + format: int64 + type: integer + lastReported: + description: Time (milliseconds) that the device reported data to + the core microservice + format: int64 + type: integer + operatingState: + description: Operating state (up/down/unknown) + type: string + synced: + description: Synced indicates whether the device already exists on + both OpenYurt and edge platform + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/yurt-manager/crds/iot.openyurt.io_deviceservices.yaml b/charts/yurt-manager/crds/iot.openyurt.io_deviceservices.yaml new file mode 100644 index 00000000000..d4722a55546 --- /dev/null +++ b/charts/yurt-manager/crds/iot.openyurt.io_deviceservices.yaml @@ -0,0 +1,141 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + name: deviceservices.iot.openyurt.io +spec: + group: iot.openyurt.io + names: + kind: DeviceService + listKind: DeviceServiceList + plural: deviceservices + shortNames: + - dsvc + singular: deviceservice + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The nodepool of deviceService + jsonPath: .spec.nodePool + name: NODEPOOL + type: string + - description: The synced status of deviceService + jsonPath: .status.synced + name: SYNCED + type: boolean + - description: The managed status of deviceService + jsonPath: .spec.managed + name: MANAGED + priority: 1 + type: boolean + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: DeviceService is the Schema for the deviceservices API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: DeviceServiceSpec defines the desired state of DeviceService + properties: + adminState: + description: Device Service Admin State + type: string + baseAddress: + type: string + description: + description: Information describing the device + type: string + labels: + description: tags or other labels applied to the device service for + search or other identification needs on the EdgeX Foundry + items: + type: string + type: array + managed: + description: True means deviceService is managed by cloud, cloud can + update the related fields False means cloud can't update the fields + type: boolean + nodePool: + description: NodePool indicates which nodePool the deviceService comes + from + type: string + required: + - baseAddress + type: object + status: + description: DeviceServiceStatus defines the observed state of DeviceService + properties: + adminState: + description: Device Service Admin State + type: string + conditions: + description: current deviceService state + items: + description: DeviceServiceCondition describes current state of a + Device. + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of in place set condition. + type: string + type: object + type: array + edgeId: + description: the Id assigned by the edge platform + type: string + lastConnected: + description: time in milliseconds that the device last reported data + to the core + format: int64 + type: integer + lastReported: + description: time in milliseconds that the device last reported data + to the core + format: int64 + type: integer + synced: + description: Synced indicates whether the device already exists on + both OpenYurt and edge platform + type: boolean + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml b/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml index 3dda7a3d1b4..72e35edad21 100644 --- a/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml +++ b/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml @@ -382,6 +382,84 @@ rules: - list - patch - watch +- apiGroups: + - iot.openyurt.io + resources: + - deviceprofiles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - iot.openyurt.io + resources: + - deviceprofiles/finalizers + verbs: + - update +- apiGroups: + - iot.openyurt.io + resources: + - deviceprofiles/status + verbs: + - get + - patch + - update +- apiGroups: + - iot.openyurt.io + resources: + - devices + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - iot.openyurt.io + resources: + - devices/finalizers + verbs: + - update +- apiGroups: + - iot.openyurt.io + resources: + - devices/status + verbs: + - get + - patch + - update +- apiGroups: + - iot.openyurt.io + resources: + - deviceservices + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - iot.openyurt.io + resources: + - deviceservices/finalizers + verbs: + - update +- apiGroups: + - iot.openyurt.io + resources: + - deviceservices/status + verbs: + - get + - patch + - update - apiGroups: - iot.openyurt.io resources: diff --git a/charts/yurt-manager/templates/yurt-manager-iot.yaml b/charts/yurt-manager/templates/yurt-manager-iot.yaml new file mode 100644 index 00000000000..ec415db5a72 --- /dev/null +++ b/charts/yurt-manager/templates/yurt-manager-iot.yaml @@ -0,0 +1,88 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: yurt-iot-dock +rules: + - apiGroups: + - "" + resources: + - events + verbs: + - get + - apiGroups: + - iot.openyurt.io + resources: + - devices + - deviceservices + - deviceprofiles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - iot.openyurt.io + resources: + - devices/status + - deviceprofiles/status + - deviceservices/status + verbs: + - get + - patch + - update + - apiGroups: + - iot.openyurt.io + resources: + - devices/finalizers + - deviceprofiles/finalizers + - deviceservices/finalizers + verbs: + - update + - apiGroups: + - apps.openyurt.io + resources: + - nodepools + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: yurt-iot-dock +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: yurt-iot-dock +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:authenticated \ No newline at end of file diff --git a/cmd/yurt-iot-dock/app/core.go b/cmd/yurt-iot-dock/app/core.go new file mode 100644 index 00000000000..39e30bcaf30 --- /dev/null +++ b/cmd/yurt-iot-dock/app/core.go @@ -0,0 +1,199 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "context" + "fmt" + "os" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + "k8s.io/klog/v2/klogr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + + "github.com/openyurtio/openyurt/cmd/yurt-iot-dock/app/options" + "github.com/openyurtio/openyurt/pkg/apis" + "github.com/openyurtio/openyurt/pkg/yurtiotdock/controllers" + "github.com/openyurtio/openyurt/pkg/yurtiotdock/controllers/util" +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + _ = clientgoscheme.AddToScheme(scheme) + + _ = apis.AddToScheme(clientgoscheme.Scheme) + _ = apis.AddToScheme(scheme) + + // +kubebuilder:scaffold:scheme +} + +func NewCmdYurtIoTDock(stopCh <-chan struct{}) *cobra.Command { + yurtIoTDockOptions := options.NewYurtIoTDockOptions() + cmd := &cobra.Command{ + Use: "yurt-iot-dock", + Short: "Launch yurt-iot-dock", + Long: "Launch yurt-iot-dock", + Run: func(cmd *cobra.Command, args []string) { + cmd.Flags().VisitAll(func(flag *pflag.Flag) { + klog.V(1).Infof("FLAG: --%s=%q", flag.Name, flag.Value) + }) + if err := options.ValidateOptions(yurtIoTDockOptions); err != nil { + klog.Fatalf("validate options: %v", err) + } + Run(yurtIoTDockOptions, stopCh) + }, + } + + yurtIoTDockOptions.AddFlags(cmd.Flags()) + return cmd +} + +func Run(opts *options.YurtIoTDockOptions, stopCh <-chan struct{}) { + ctrl.SetLogger(klogr.New()) + cfg := ctrl.GetConfigOrDie() + + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme, + MetricsBindAddress: opts.MetricsAddr, + HealthProbeBindAddress: opts.ProbeAddr, + LeaderElection: opts.EnableLeaderElection, + LeaderElectionID: "yurt-iot-dock", + Namespace: opts.Namespace, + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + // perform preflight check + setupLog.Info("[preflight] Running pre-flight checks") + if err := preflightCheck(mgr, opts); err != nil { + setupLog.Error(err, "failed to run pre-flight checks") + os.Exit(1) + } + + // register the field indexers + setupLog.Info("[preflight] Registering the field indexers") + if err := util.RegisterFieldIndexers(mgr.GetFieldIndexer()); err != nil { + setupLog.Error(err, "failed to register field indexers") + os.Exit(1) + } + + // get nodepool where yurt-iot-dock run + if opts.Nodepool == "" { + opts.Nodepool, err = util.GetNodePool(mgr.GetConfig()) + if err != nil { + setupLog.Error(err, "failed to get the nodepool where yurt-iot-dock run") + os.Exit(1) + } + } + + // setup the DeviceProfile Reconciler and Syncer + if err = (&controllers.DeviceProfileReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr, opts); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "DeviceProfile") + os.Exit(1) + } + dfs, err := controllers.NewDeviceProfileSyncer(mgr.GetClient(), opts) + if err != nil { + setupLog.Error(err, "unable to create syncer", "syncer", "DeviceProfile") + os.Exit(1) + } + err = mgr.Add(dfs.NewDeviceProfileSyncerRunnable()) + if err != nil { + setupLog.Error(err, "unable to create syncer runnable", "syncer", "DeviceProfile") + os.Exit(1) + } + + // setup the Device Reconciler and Syncer + if err = (&controllers.DeviceReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr, opts); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Device") + os.Exit(1) + } + ds, err := controllers.NewDeviceSyncer(mgr.GetClient(), opts) + if err != nil { + setupLog.Error(err, "unable to create syncer", "controller", "Device") + os.Exit(1) + } + err = mgr.Add(ds.NewDeviceSyncerRunnable()) + if err != nil { + setupLog.Error(err, "unable to create syncer runnable", "syncer", "Device") + os.Exit(1) + } + + // setup the DeviceService Reconciler and Syncer + if err = (&controllers.DeviceServiceReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr, opts); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "DeviceService") + os.Exit(1) + } + dss, err := controllers.NewDeviceServiceSyncer(mgr.GetClient(), opts) + if err != nil { + setupLog.Error(err, "unable to create syncer", "syncer", "DeviceService") + os.Exit(1) + } + err = mgr.Add(dss.NewDeviceServiceSyncerRunnable()) + if err != nil { + setupLog.Error(err, "unable to create syncer runnable", "syncer", "DeviceService") + os.Exit(1) + } + //+kubebuilder:scaffold:builder + + if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("check", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + + setupLog.Info("[run controllers] Starting manager, acting on " + fmt.Sprintf("[NodePool: %s, Namespace: %s]", opts.Nodepool, opts.Namespace)) + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "failed to running manager") + os.Exit(1) + } +} + +func preflightCheck(mgr ctrl.Manager, opts *options.YurtIoTDockOptions) error { + client, err := kubernetes.NewForConfig(mgr.GetConfig()) + if err != nil { + return err + } + if _, err := client.CoreV1().Namespaces().Get(context.TODO(), opts.Namespace, metav1.GetOptions{}); err != nil { + return err + } + return nil +} diff --git a/cmd/yurt-iot-dock/app/options/options.go b/cmd/yurt-iot-dock/app/options/options.go new file mode 100644 index 00000000000..08d9e37a920 --- /dev/null +++ b/cmd/yurt-iot-dock/app/options/options.go @@ -0,0 +1,82 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "fmt" + "net" + + "github.com/spf13/pflag" +) + +// YurtIoTDockOptions is the main settings for the yurt-iot-dock +type YurtIoTDockOptions struct { + MetricsAddr string + ProbeAddr string + EnableLeaderElection bool + Nodepool string + Namespace string + CoreDataAddr string + CoreMetadataAddr string + CoreCommandAddr string + EdgeSyncPeriod uint +} + +func NewYurtIoTDockOptions() *YurtIoTDockOptions { + return &YurtIoTDockOptions{ + MetricsAddr: ":8080", + ProbeAddr: ":8080", + EnableLeaderElection: false, + Nodepool: "", + Namespace: "default", + CoreDataAddr: "edgex-core-data:59880", + CoreMetadataAddr: "edgex-core-metadata:59881", + CoreCommandAddr: "edgex-core-command:59882", + EdgeSyncPeriod: 5, + } +} + +func ValidateOptions(options *YurtIoTDockOptions) error { + if err := ValidateEdgePlatformAddress(options); err != nil { + return err + } + return nil +} + +func (o *YurtIoTDockOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&o.MetricsAddr, "metrics-bind-address", o.MetricsAddr, "The address the metric endpoint binds to.") + fs.StringVar(&o.ProbeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + fs.BoolVar(&o.EnableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+"Enabling this will ensure there is only one active controller manager.") + fs.StringVar(&o.Nodepool, "nodepool", "", "The nodePool deviceController is deployed in.(just for debugging)") + fs.StringVar(&o.Namespace, "namespace", "default", "The cluster namespace for edge resources synchronization.") + fs.StringVar(&o.CoreDataAddr, "core-data-address", "edgex-core-data:59880", "The address of edge core-data service.") + fs.StringVar(&o.CoreMetadataAddr, "core-metadata-address", "edgex-core-metadata:59881", "The address of edge core-metadata service.") + fs.StringVar(&o.CoreCommandAddr, "core-command-address", "edgex-core-command:59882", "The address of edge core-command service.") + fs.UintVar(&o.EdgeSyncPeriod, "edge-sync-period", 5, "The period of the device management platform synchronizing the device status to the cloud.(in seconds,not less than 5 seconds)") +} + +func ValidateEdgePlatformAddress(options *YurtIoTDockOptions) error { + addrs := []string{options.CoreDataAddr, options.CoreMetadataAddr, options.CoreCommandAddr} + for _, addr := range addrs { + if addr != "" { + if _, _, err := net.SplitHostPort(addr); err != nil { + return fmt.Errorf("invalid address: %s", err) + } + } + } + return nil +} diff --git a/cmd/yurt-iot-dock/yurt-iot-dock.go b/cmd/yurt-iot-dock/yurt-iot-dock.go new file mode 100644 index 00000000000..c647e5241c0 --- /dev/null +++ b/cmd/yurt-iot-dock/yurt-iot-dock.go @@ -0,0 +1,43 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "math/rand" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/cmd/yurt-iot-dock/app" +) + +func main() { + rand.Seed(time.Now().UnixNano()) + klog.InitFlags(nil) + defer klog.Flush() + + cmd := app.NewCmdYurtIoTDock(wait.NeverStop) + cmd.Flags().AddGoFlagSet(flag.CommandLine) + if err := cmd.Execute(); err != nil { + panic(err) + } +} diff --git a/go.mod b/go.mod index f79da3cf042..53dcd2e2ad0 100644 --- a/go.mod +++ b/go.mod @@ -5,10 +5,13 @@ go 1.18 require ( github.com/aliyun/alibaba-cloud-sdk-go v1.62.156 github.com/davecgh/go-spew v1.1.1 + github.com/edgexfoundry/go-mod-core-contracts/v2 v2.3.0 + github.com/go-resty/resty/v2 v2.4.0 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 github.com/hashicorp/go-version v1.6.0 + github.com/jarcoal/httpmock v1.3.0 github.com/onsi/ginkgo/v2 v2.1.4 github.com/onsi/gomega v1.19.0 github.com/opencontainers/selinux v1.11.0 @@ -24,6 +27,7 @@ require ( go.etcd.io/etcd/api/v3 v3.5.0 go.etcd.io/etcd/client/pkg/v3 v3.5.0 go.etcd.io/etcd/client/v3 v3.5.0 + golang.org/x/net v0.9.0 golang.org/x/sys v0.10.0 google.golang.org/grpc v1.56.2 gopkg.in/cheggaaa/pb.v1 v1.0.28 @@ -86,10 +90,14 @@ require ( github.com/felixge/httpsnoop v1.0.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/fvbommel/sortorder v1.0.1 // indirect + github.com/fxamacker/cbor/v2 v2.4.0 // indirect github.com/go-logr/logr v0.4.0 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.5 // indirect github.com/go-openapi/swag v0.19.14 // indirect + github.com/go-playground/locales v0.14.0 // indirect + github.com/go-playground/universal-translator v0.18.0 // indirect + github.com/go-playground/validator/v10 v10.11.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -103,6 +111,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/leodido/go-urn v1.2.1 // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.7 // indirect @@ -124,6 +133,7 @@ require ( github.com/spf13/afero v1.6.0 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect + github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/contrib v0.20.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 // indirect @@ -139,7 +149,6 @@ require ( go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.24.0 // indirect golang.org/x/crypto v0.5.0 // indirect - golang.org/x/net v0.9.0 // indirect golang.org/x/oauth2 v0.7.0 // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/term v0.7.0 // indirect diff --git a/go.sum b/go.sum index 4c71d04330b..c77bd68f7d3 100644 --- a/go.sum +++ b/go.sum @@ -175,6 +175,8 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/edgexfoundry/go-mod-core-contracts/v2 v2.3.0 h1:8Svk1HTehXEgwxgyA4muVhSkP3D9n1q+oSHI3B1Ac90= +github.com/edgexfoundry/go-mod-core-contracts/v2 v2.3.0/go.mod h1:4/e61acxVkhQWCTjQ4XcHVJDnrMDloFsZZB1B6STCRw= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -214,6 +216,8 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/fvbommel/sortorder v1.0.1 h1:dSnXLt4mJYH25uDDGa3biZNQsozaUWDSWeKJ0qqFfzE= github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= +github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88= +github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -245,6 +249,16 @@ github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfT github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ= +github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= +github.com/go-resty/resty/v2 v2.4.0 h1:s6TItTLejEI+2mn98oijC5w/Rk2YU+OA6x0mnZN6r6k= +github.com/go-resty/resty/v2 v2.4.0/go.mod h1:B88+xCTEwvfD94NOuE6GS1wMlnoKNY8eEiNizfNwOwA= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -385,6 +399,8 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ishidawataru/sctp v0.0.0-20190723014705-7c296d48a2b5/go.mod h1:DM4VvS+hD/kDi1U1QsX2fnZowwBhqD0Dk3bRPKF/Oc8= +github.com/jarcoal/httpmock v1.3.0 h1:2RJ8GP0IIaWwcC9Fp2BmVi8Kog3v2Hn7VXM3fTd+nuc= +github.com/jarcoal/httpmock v1.3.0/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -422,6 +438,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -429,6 +446,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kubernetes/kubernetes v1.22.3 h1:0gYnqsr5nZiAO+iDkEU7RJ6Ne2CMyoinJXVm5qVSTiE= github.com/kubernetes/kubernetes v1.22.3/go.mod h1:Snea7fgIObGgHmLbUJ3OgjGEr5bjj16iEdp5oHS6eS8= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/libopenstorage/openstorage v1.0.0/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= @@ -452,6 +471,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989/go.mod h1:2eu9pRWp8mo84xCg6KswZ+USQHjwgRhNp06sozOdsTY= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= @@ -543,6 +563,7 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -591,6 +612,8 @@ github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzG github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= @@ -676,6 +699,8 @@ github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3C github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= @@ -756,6 +781,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -837,6 +863,7 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -914,8 +941,10 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -936,6 +965,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1086,6 +1116,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= diff --git a/hack/dockerfiles/build/Dockerfile.yurt-iot-dock b/hack/dockerfiles/build/Dockerfile.yurt-iot-dock new file mode 100644 index 00000000000..3e684bb6866 --- /dev/null +++ b/hack/dockerfiles/build/Dockerfile.yurt-iot-dock @@ -0,0 +1,8 @@ +# multi-arch image building for yurt-iot-dock + +FROM --platform=${TARGETPLATFORM} alpine:3.17 +ARG TARGETOS TARGETARCH MIRROR_REPO +RUN if [ ! -z "${MIRROR_REPO+x}" ]; then sed -i "s/dl-cdn.alpinelinux.org/${MIRROR_REPO}/g" /etc/apk/repositories; fi && \ + apk add ca-certificates bash libc6-compat iptables ip6tables && update-ca-certificates && rm /var/cache/apk/* +COPY ./_output/local/bin/${TARGETOS}/${TARGETARCH}/yurt-iot-dock /usr/local/bin/yurt-iot-dock +ENTRYPOINT ["/usr/local/bin/yurt-iot-dock"] \ No newline at end of file diff --git a/hack/dockerfiles/release/Dockerfile.yurt-iot-dock b/hack/dockerfiles/release/Dockerfile.yurt-iot-dock new file mode 100644 index 00000000000..d73a35cb790 --- /dev/null +++ b/hack/dockerfiles/release/Dockerfile.yurt-iot-dock @@ -0,0 +1,14 @@ +# multi-arch image building for yurt-iot-dock + +FROM --platform=${BUILDPLATFORM} golang:1.18 as builder +ADD . /build +ARG TARGETOS TARGETARCH GIT_VERSION GOPROXY MIRROR_REPO +WORKDIR /build/ +RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} GIT_VERSION=${GIT_VERSION} make build WHAT=cmd/yurt-iot-dock + +FROM --platform=${TARGETPLATFORM} alpine:3.17 +ARG TARGETOS TARGETARCH MIRROR_REPO +RUN if [ ! -z "${MIRROR_REPO+x}" ]; then sed -i "s/dl-cdn.alpinelinux.org/${MIRROR_REPO}/g" /etc/apk/repositories; fi && \ + apk add ca-certificates bash libc6-compat iptables ip6tables && update-ca-certificates && rm /var/cache/apk/* +COPY --from=builder /build/_output/local/bin/${TARGETOS}/${TARGETARCH}/yurt-iot-dock /usr/local/bin/yurt-iot-dock +ENTRYPOINT ["/usr/local/bin/yurt-iot-dock"] \ No newline at end of file diff --git a/hack/make-rules/build.sh b/hack/make-rules/build.sh index 1422dd5df7d..5e28026b658 100755 --- a/hack/make-rules/build.sh +++ b/hack/make-rules/build.sh @@ -25,6 +25,7 @@ readonly YURT_ALL_TARGETS=( yurt-node-servant yurthub yurt-manager + yurt-iot-dock ) # clean old binaries at GOOS and GOARCH diff --git a/hack/make-rules/image_build.sh b/hack/make-rules/image_build.sh index b069cfcbc0b..1f4afa71d07 100755 --- a/hack/make-rules/image_build.sh +++ b/hack/make-rules/image_build.sh @@ -25,6 +25,7 @@ readonly IMAGE_TARGETS=( yurt-node-servant yurthub yurt-manager + yurt-iot-dock ) http_proxy=${http_proxy:-} diff --git a/hack/make-rules/kustomize_to_chart.sh b/hack/make-rules/kustomize_to_chart.sh index a9442b31b2a..f8e0f98eff8 100755 --- a/hack/make-rules/kustomize_to_chart.sh +++ b/hack/make-rules/kustomize_to_chart.sh @@ -192,6 +192,10 @@ EOF mv ${crd_dir}/apiextensions.k8s.io_v1_customresourcedefinition_yurtappdaemons.apps.openyurt.io.yaml ${crd_dir}/apps.openyurt.io_yurtappdaemons.yaml mv ${crd_dir}/apiextensions.k8s.io_v1_customresourcedefinition_yurtappsets.apps.openyurt.io.yaml ${crd_dir}/apps.openyurt.io_yurtappsets.yaml mv ${crd_dir}/apiextensions.k8s.io_v1_customresourcedefinition_gateways.raven.openyurt.io.yaml ${crd_dir}/raven.openyurt.io_gateways.yaml + mv ${crd_dir}/apiextensions.k8s.io_v1_customresourcedefinition_platformadmins.iot.openyurt.io.yaml ${crd_dir}/iot.openyurt.io_platformadmins.yaml + mv ${crd_dir}/apiextensions.k8s.io_v1_customresourcedefinition_devices.iot.openyurt.io.yaml ${crd_dir}/iot.openyurt.io_devices.yaml + mv ${crd_dir}/apiextensions.k8s.io_v1_customresourcedefinition_deviceservices.iot.openyurt.io.yaml ${crd_dir}/iot.openyurt.io_deviceservices.yaml + mv ${crd_dir}/apiextensions.k8s.io_v1_customresourcedefinition_deviceprofiles.iot.openyurt.io.yaml ${crd_dir}/iot.openyurt.io_deviceprofiles.yaml # rbac dir local rbac_kustomization_resources="" diff --git a/hack/make-rules/local-up-openyurt.sh b/hack/make-rules/local-up-openyurt.sh index 5181eec46f8..88c709b0b4f 100755 --- a/hack/make-rules/local-up-openyurt.sh +++ b/hack/make-rules/local-up-openyurt.sh @@ -56,6 +56,7 @@ readonly REQUIRED_IMAGES=( openyurt/node-servant openyurt/yurt-manager openyurt/yurthub + openyurt/yurt-iot-dock ) readonly LOCAL_ARCH=$(go env GOHOSTARCH) diff --git a/pkg/apis/iot/v1alpha1/condition_const.go b/pkg/apis/iot/v1alpha1/condition_const.go index 7f1c8b5e84e..498c9c3e87e 100644 --- a/pkg/apis/iot/v1alpha1/condition_const.go +++ b/pkg/apis/iot/v1alpha1/condition_const.go @@ -35,4 +35,30 @@ const ( DeploymentProvisioningReason = "DeploymentProvisioning" DeploymentProvisioningFailedReason = "DeploymentProvisioningFailed" + + // DeviceSyncedCondition indicates that the device exists in both OpenYurt and edge platform + DeviceSyncedCondition DeviceConditionType = "DeviceSynced" + + DeviceManagingReason = "This device is not managed by openyurt" + + DeviceCreateSyncedReason = "Failed to create device on edge platform" + + // DeviceManagingCondition indicates that the device is being managed by cloud and its properties are being reconciled + DeviceManagingCondition DeviceConditionType = "DeviceManaging" + + DeviceVistedCoreMetadataSyncedReason = "Failed to visit the EdgeX core-metadata-service" + + DeviceUpdateStateReason = "Failed to update AdminState or OperatingState of device on edge platform" + + // DeviceServiceSyncedCondition indicates that the deviceService exists in both OpenYurt and edge platform + DeviceServiceSyncedCondition DeviceServiceConditionType = "DeviceServiceSynced" + + DeviceServiceManagingReason = "This deviceService is not managed by openyurt" + + // DeviceServiceManagingCondition indicates that the deviceService is being managed by cloud and its field are being reconciled + DeviceServiceManagingCondition DeviceServiceConditionType = "DeviceServiceManaging" + + DeviceServiceCreateSyncedReason = "Failed to add DeviceService to EdgeX" + + DeviceServiceUpdateStatusSyncedReason = "Failed to update DeviceService status" ) diff --git a/pkg/apis/iot/v1alpha1/device_types.go b/pkg/apis/iot/v1alpha1/device_types.go new file mode 100644 index 00000000000..9a1bafd257f --- /dev/null +++ b/pkg/apis/iot/v1alpha1/device_types.go @@ -0,0 +1,175 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + DeviceFinalizer = "iot.openyurt.io/device" +) + +// DeviceConditionType indicates valid conditions type of a Device. +type DeviceConditionType string + +type AdminState string + +const ( + Locked AdminState = "LOCKED" + UnLocked AdminState = "UNLOCKED" +) + +type OperatingState string + +const ( + Unknown OperatingState = "UNKNOWN" + Up OperatingState = "UP" + Down OperatingState = "DOWN" +) + +type ProtocolProperties map[string]string + +// DeviceSpec defines the desired state of Device +type DeviceSpec struct { + // Information describing the device + Description string `json:"description,omitempty"` + // Admin state (locked/unlocked) + AdminState AdminState `json:"adminState,omitempty"` + // Operating state (enabled/disabled) + OperatingState OperatingState `json:"operatingState,omitempty"` + // A map of supported protocols for the given device + Protocols map[string]ProtocolProperties `json:"protocols,omitempty"` + // Other labels applied to the device to help with searching + Labels []string `json:"labels,omitempty"` + // Device service specific location (interface{} is an empty interface so + // it can be anything) + // TODO: location type in edgex is interface{} + Location string `json:"location,omitempty"` + // Associated Device Service - One per device + Service string `json:"serviceName"` + // Associated Device Profile - Describes the device + Profile string `json:"profileName"` + Notify bool `json:"notify"` + // True means device is managed by cloud, cloud can update the related fields + // False means cloud can't update the fields + Managed bool `json:"managed,omitempty"` + // NodePool indicates which nodePool the device comes from + NodePool string `json:"nodePool,omitempty"` + // TODO support the following field + // A list of auto-generated events coming from the device + // AutoEvents []AutoEvent `json:"autoEvents"` + // DeviceProperties represents the expected state of the device's properties + DeviceProperties map[string]DesiredPropertyState `json:"deviceProperties,omitempty"` +} + +type DesiredPropertyState struct { + Name string `json:"name"` + PutURL string `json:"putURL,omitempty"` + DesiredValue string `json:"desiredValue"` +} + +type ActualPropertyState struct { + Name string `json:"name"` + GetURL string `json:"getURL,omitempty"` + ActualValue string `json:"actualValue"` +} + +// DeviceStatus defines the observed state of Device +type DeviceStatus struct { + // Time (milliseconds) that the device last provided any feedback or + // responded to any request + LastConnected int64 `json:"lastConnected,omitempty"` + // Time (milliseconds) that the device reported data to the core + // microservice + LastReported int64 `json:"lastReported,omitempty"` + // Synced indicates whether the device already exists on both OpenYurt and edge platform + Synced bool `json:"synced,omitempty"` + // it represents the actual state of the device's properties + DeviceProperties map[string]ActualPropertyState `json:"deviceProperties,omitempty"` + EdgeId string `json:"edgeId,omitempty"` + // Admin state (locked/unlocked) + AdminState AdminState `json:"adminState,omitempty"` + // Operating state (up/down/unknown) + OperatingState OperatingState `json:"operatingState,omitempty"` + // current device state + // +optional + Conditions []DeviceCondition `json:"conditions,omitempty"` +} + +// DeviceCondition describes current state of a Device. +type DeviceCondition struct { + // Type of in place set condition. + Type DeviceConditionType `json:"type,omitempty"` + + // Status of the condition, one of True, False, Unknown. + Status corev1.ConditionStatus `json:"status,omitempty"` + + // Last time the condition transitioned from one status to another. + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` + + // The reason for the condition's last transition. + Reason string `json:"reason,omitempty"` + + // A human readable message indicating details about the transition. + Message string `json:"message,omitempty"` +} + +// +genclient +// +k8s:openapi-gen=true +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:shortName=dev +// +kubebuilder:printcolumn:name="NODEPOOL",type="string",JSONPath=".spec.nodePool",description="The nodepool of device" +// +kubebuilder:printcolumn:name="SYNCED",type="boolean",JSONPath=".status.synced",description="The synced status of device" +// +kubebuilder:printcolumn:name="MANAGED",type="boolean",priority=1,JSONPath=".spec.managed",description="The managed status of device" +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" + +// Device is the Schema for the devices API +type Device struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec DeviceSpec `json:"spec,omitempty"` + Status DeviceStatus `json:"status,omitempty"` +} + +func (d *Device) SetConditions(conditions []DeviceCondition) { + d.Status.Conditions = conditions +} + +func (d *Device) GetConditions() []DeviceCondition { + return d.Status.Conditions +} + +func (d *Device) IsAddedToEdgeX() bool { + return d.Status.Synced +} + +//+kubebuilder:object:root=true + +// DeviceList contains a list of Device +type DeviceList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Device `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Device{}, &DeviceList{}) +} diff --git a/pkg/apis/iot/v1alpha1/deviceprofile_types.go b/pkg/apis/iot/v1alpha1/deviceprofile_types.go new file mode 100644 index 00000000000..2d25605bea9 --- /dev/null +++ b/pkg/apis/iot/v1alpha1/deviceprofile_types.go @@ -0,0 +1,122 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + DeviceProfileFinalizer = "iot.openyurt.io/deviceprofile" +) + +type DeviceResource struct { + Description string `json:"description"` + Name string `json:"name"` + Tag string `json:"tag,omitempty"` + IsHidden bool `json:"isHidden"` + Properties ResourceProperties `json:"properties"` + Attributes map[string]string `json:"attributes,omitempty"` +} + +type ResourceProperties struct { + ReadWrite string `json:"readWrite,omitempty"` // Read/Write Permissions set for this property + Minimum string `json:"minimum,omitempty"` // Minimum value that can be get/set from this property + Maximum string `json:"maximum,omitempty"` // Maximum value that can be get/set from this property + DefaultValue string `json:"defaultValue,omitempty"` // Default value set to this property if no argument is passed + Mask string `json:"mask,omitempty"` // Mask to be applied prior to get/set of property + Shift string `json:"shift,omitempty"` // Shift to be applied after masking, prior to get/set of property + Scale string `json:"scale,omitempty"` // Multiplicative factor to be applied after shifting, prior to get/set of property + Offset string `json:"offset,omitempty"` // Additive factor to be applied after multiplying, prior to get/set of property + Base string `json:"base,omitempty"` // Base for property to be applied to, leave 0 for no power operation (i.e. base ^ property: 2 ^ 10) + Assertion string `json:"assertion,omitempty"` + MediaType string `json:"mediaType,omitempty"` + Units string `json:"units,omitempty"` + ValueType string `json:"valueType,omitempty"` +} + +type DeviceCommand struct { + Name string `json:"name"` + IsHidden bool `json:"isHidden"` + ReadWrite string `json:"readWrite"` + ResourceOperations []ResourceOperation `json:"resourceOperations"` +} + +type ResourceOperation struct { + DeviceResource string `json:"deviceResource,omitempty"` + Mappings map[string]string `json:"mappings,omitempty"` + DefaultValue string `json:"defaultValue"` +} + +// DeviceProfileSpec defines the desired state of DeviceProfile +type DeviceProfileSpec struct { + // NodePool specifies which nodePool the deviceProfile belongs to + NodePool string `json:"nodePool,omitempty"` + Description string `json:"description,omitempty"` + // Manufacturer of the device + Manufacturer string `json:"manufacturer,omitempty"` + // Model of the device + Model string `json:"model,omitempty"` + // Labels used to search for groups of profiles on EdgeX Foundry + Labels []string `json:"labels,omitempty"` + DeviceResources []DeviceResource `json:"deviceResources,omitempty"` + DeviceCommands []DeviceCommand `json:"deviceCommands,omitempty"` +} + +// DeviceProfileStatus defines the observed state of DeviceProfile +type DeviceProfileStatus struct { + EdgeId string `json:"id,omitempty"` + Synced bool `json:"synced,omitempty"` +} + +// +genclient +// +k8s:openapi-gen=true +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:shortName=dp +// +kubebuilder:printcolumn:name="NODEPOOL",type="string",JSONPath=".spec.nodePool",description="The nodepool of deviceProfile" +// +kubebuilder:printcolumn:name="SYNCED",type="boolean",JSONPath=".status.synced",description="The synced status of deviceProfile" +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" + +// DeviceProfile represents the attributes and operational capabilities of a device. +// It is a template for which there can be multiple matching devices within a given system. +// NOTE This struct is derived from +// edgex/go-mod-core-contracts/models/deviceprofile.go +type DeviceProfile struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec DeviceProfileSpec `json:"spec,omitempty"` + Status DeviceProfileStatus `json:"status,omitempty"` +} + +func (dp *DeviceProfile) IsAddedToEdgeX() bool { + return dp.Status.Synced +} + +//+kubebuilder:object:root=true + +// DeviceProfileList contains a list of DeviceProfile +type DeviceProfileList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []DeviceProfile `json:"items"` +} + +func init() { + SchemeBuilder.Register(&DeviceProfile{}, &DeviceProfileList{}) +} diff --git a/pkg/apis/iot/v1alpha1/deviceservice_types.go b/pkg/apis/iot/v1alpha1/deviceservice_types.go new file mode 100644 index 00000000000..4720b4e46bf --- /dev/null +++ b/pkg/apis/iot/v1alpha1/deviceservice_types.go @@ -0,0 +1,121 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + DeviceServiceFinalizer = "iot.openyurt.io/deviceservice" +) + +// DeviceServiceConditionType indicates valid conditions type of a Device Service. +type DeviceServiceConditionType string + +// DeviceServiceSpec defines the desired state of DeviceService +type DeviceServiceSpec struct { + BaseAddress string `json:"baseAddress"` + // Information describing the device + Description string `json:"description,omitempty"` + // tags or other labels applied to the device service for search or other + // identification needs on the EdgeX Foundry + Labels []string `json:"labels,omitempty"` + // Device Service Admin State + AdminState AdminState `json:"adminState,omitempty"` + // True means deviceService is managed by cloud, cloud can update the related fields + // False means cloud can't update the fields + Managed bool `json:"managed,omitempty"` + // NodePool indicates which nodePool the deviceService comes from + NodePool string `json:"nodePool,omitempty"` +} + +// DeviceServiceStatus defines the observed state of DeviceService +type DeviceServiceStatus struct { + // Synced indicates whether the device already exists on both OpenYurt and edge platform + Synced bool `json:"synced,omitempty"` + // the Id assigned by the edge platform + EdgeId string `json:"edgeId,omitempty"` + // time in milliseconds that the device last reported data to the core + LastConnected int64 `json:"lastConnected,omitempty"` + // time in milliseconds that the device last reported data to the core + LastReported int64 `json:"lastReported,omitempty"` + // Device Service Admin State + AdminState AdminState `json:"adminState,omitempty"` + // current deviceService state + // +optional + Conditions []DeviceServiceCondition `json:"conditions,omitempty"` +} + +// DeviceServiceCondition describes current state of a Device. +type DeviceServiceCondition struct { + // Type of in place set condition. + Type DeviceServiceConditionType `json:"type,omitempty"` + + // Status of the condition, one of True, False, Unknown. + Status corev1.ConditionStatus `json:"status,omitempty"` + + // Last time the condition transitioned from one status to another. + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` + + // The reason for the condition's last transition. + Reason string `json:"reason,omitempty"` + + // A human readable message indicating details about the transition. + Message string `json:"message,omitempty"` +} + +// +genclient +// +k8s:openapi-gen=true +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:shortName=dsvc +// +kubebuilder:printcolumn:name="NODEPOOL",type="string",JSONPath=".spec.nodePool",description="The nodepool of deviceService" +// +kubebuilder:printcolumn:name="SYNCED",type="boolean",JSONPath=".status.synced",description="The synced status of deviceService" +// +kubebuilder:printcolumn:name="MANAGED",type="boolean",priority=1,JSONPath=".spec.managed",description="The managed status of deviceService" +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" + +// DeviceService is the Schema for the deviceservices API +type DeviceService struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec DeviceServiceSpec `json:"spec,omitempty"` + Status DeviceServiceStatus `json:"status,omitempty"` +} + +func (ds *DeviceService) SetConditions(conditions []DeviceServiceCondition) { + ds.Status.Conditions = conditions +} + +func (ds *DeviceService) GetConditions() []DeviceServiceCondition { + return ds.Status.Conditions +} + +//+kubebuilder:object:root=true + +// DeviceServiceList contains a list of DeviceService +type DeviceServiceList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []DeviceService `json:"items"` +} + +func init() { + SchemeBuilder.Register(&DeviceService{}, &DeviceServiceList{}) +} diff --git a/pkg/apis/iot/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/iot/v1alpha1/zz_generated.deepcopy.go index 62aaa062a52..3162ed46c6e 100644 --- a/pkg/apis/iot/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/iot/v1alpha1/zz_generated.deepcopy.go @@ -25,6 +25,21 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ActualPropertyState) DeepCopyInto(out *ActualPropertyState) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ActualPropertyState. +func (in *ActualPropertyState) DeepCopy() *ActualPropertyState { + if in == nil { + return nil + } + out := new(ActualPropertyState) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DeploymentTemplateSpec) DeepCopyInto(out *DeploymentTemplateSpec) { *out = *in @@ -42,6 +57,439 @@ func (in *DeploymentTemplateSpec) DeepCopy() *DeploymentTemplateSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DesiredPropertyState) DeepCopyInto(out *DesiredPropertyState) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DesiredPropertyState. +func (in *DesiredPropertyState) DeepCopy() *DesiredPropertyState { + if in == nil { + return nil + } + out := new(DesiredPropertyState) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Device) DeepCopyInto(out *Device) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Device. +func (in *Device) DeepCopy() *Device { + if in == nil { + return nil + } + out := new(Device) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Device) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeviceCommand) DeepCopyInto(out *DeviceCommand) { + *out = *in + if in.ResourceOperations != nil { + in, out := &in.ResourceOperations, &out.ResourceOperations + *out = make([]ResourceOperation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeviceCommand. +func (in *DeviceCommand) DeepCopy() *DeviceCommand { + if in == nil { + return nil + } + out := new(DeviceCommand) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeviceCondition) DeepCopyInto(out *DeviceCondition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeviceCondition. +func (in *DeviceCondition) DeepCopy() *DeviceCondition { + if in == nil { + return nil + } + out := new(DeviceCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeviceList) DeepCopyInto(out *DeviceList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Device, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeviceList. +func (in *DeviceList) DeepCopy() *DeviceList { + if in == nil { + return nil + } + out := new(DeviceList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DeviceList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeviceProfile) DeepCopyInto(out *DeviceProfile) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeviceProfile. +func (in *DeviceProfile) DeepCopy() *DeviceProfile { + if in == nil { + return nil + } + out := new(DeviceProfile) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DeviceProfile) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeviceProfileList) DeepCopyInto(out *DeviceProfileList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]DeviceProfile, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeviceProfileList. +func (in *DeviceProfileList) DeepCopy() *DeviceProfileList { + if in == nil { + return nil + } + out := new(DeviceProfileList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DeviceProfileList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeviceProfileSpec) DeepCopyInto(out *DeviceProfileSpec) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.DeviceResources != nil { + in, out := &in.DeviceResources, &out.DeviceResources + *out = make([]DeviceResource, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.DeviceCommands != nil { + in, out := &in.DeviceCommands, &out.DeviceCommands + *out = make([]DeviceCommand, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeviceProfileSpec. +func (in *DeviceProfileSpec) DeepCopy() *DeviceProfileSpec { + if in == nil { + return nil + } + out := new(DeviceProfileSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeviceProfileStatus) DeepCopyInto(out *DeviceProfileStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeviceProfileStatus. +func (in *DeviceProfileStatus) DeepCopy() *DeviceProfileStatus { + if in == nil { + return nil + } + out := new(DeviceProfileStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeviceResource) DeepCopyInto(out *DeviceResource) { + *out = *in + out.Properties = in.Properties + if in.Attributes != nil { + in, out := &in.Attributes, &out.Attributes + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeviceResource. +func (in *DeviceResource) DeepCopy() *DeviceResource { + if in == nil { + return nil + } + out := new(DeviceResource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeviceService) DeepCopyInto(out *DeviceService) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeviceService. +func (in *DeviceService) DeepCopy() *DeviceService { + if in == nil { + return nil + } + out := new(DeviceService) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DeviceService) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeviceServiceCondition) DeepCopyInto(out *DeviceServiceCondition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeviceServiceCondition. +func (in *DeviceServiceCondition) DeepCopy() *DeviceServiceCondition { + if in == nil { + return nil + } + out := new(DeviceServiceCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeviceServiceList) DeepCopyInto(out *DeviceServiceList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]DeviceService, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeviceServiceList. +func (in *DeviceServiceList) DeepCopy() *DeviceServiceList { + if in == nil { + return nil + } + out := new(DeviceServiceList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *DeviceServiceList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeviceServiceSpec) DeepCopyInto(out *DeviceServiceSpec) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeviceServiceSpec. +func (in *DeviceServiceSpec) DeepCopy() *DeviceServiceSpec { + if in == nil { + return nil + } + out := new(DeviceServiceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeviceServiceStatus) DeepCopyInto(out *DeviceServiceStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]DeviceServiceCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeviceServiceStatus. +func (in *DeviceServiceStatus) DeepCopy() *DeviceServiceStatus { + if in == nil { + return nil + } + out := new(DeviceServiceStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeviceSpec) DeepCopyInto(out *DeviceSpec) { + *out = *in + if in.Protocols != nil { + in, out := &in.Protocols, &out.Protocols + *out = make(map[string]ProtocolProperties, len(*in)) + for key, val := range *in { + var outVal map[string]string + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = make(ProtocolProperties, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + (*out)[key] = outVal + } + } + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.DeviceProperties != nil { + in, out := &in.DeviceProperties, &out.DeviceProperties + *out = make(map[string]DesiredPropertyState, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeviceSpec. +func (in *DeviceSpec) DeepCopy() *DeviceSpec { + if in == nil { + return nil + } + out := new(DeviceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeviceStatus) DeepCopyInto(out *DeviceStatus) { + *out = *in + if in.DeviceProperties != nil { + in, out := &in.DeviceProperties, &out.DeviceProperties + *out = make(map[string]ActualPropertyState, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]DeviceCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeviceStatus. +func (in *DeviceStatus) DeepCopy() *DeviceStatus { + if in == nil { + return nil + } + out := new(DeviceStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PlatformAdmin) DeepCopyInto(out *PlatformAdmin) { *out = *in @@ -168,6 +616,64 @@ func (in *PlatformAdminStatus) DeepCopy() *PlatformAdminStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in ProtocolProperties) DeepCopyInto(out *ProtocolProperties) { + { + in := &in + *out = make(ProtocolProperties, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProtocolProperties. +func (in ProtocolProperties) DeepCopy() ProtocolProperties { + if in == nil { + return nil + } + out := new(ProtocolProperties) + in.DeepCopyInto(out) + return *out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceOperation) DeepCopyInto(out *ResourceOperation) { + *out = *in + if in.Mappings != nil { + in, out := &in.Mappings, &out.Mappings + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceOperation. +func (in *ResourceOperation) DeepCopy() *ResourceOperation { + if in == nil { + return nil + } + out := new(ResourceOperation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceProperties) DeepCopyInto(out *ResourceProperties) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceProperties. +func (in *ResourceProperties) DeepCopy() *ResourceProperties { + if in == nil { + return nil + } + out := new(ResourceProperties) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceTemplateSpec) DeepCopyInto(out *ServiceTemplateSpec) { *out = *in diff --git a/pkg/controller/platformadmin/platformadmin_controller.go b/pkg/controller/platformadmin/platformadmin_controller.go index 542c147c0f2..dddb4178a9e 100644 --- a/pkg/controller/platformadmin/platformadmin_controller.go +++ b/pkg/controller/platformadmin/platformadmin_controller.go @@ -729,6 +729,12 @@ func (r *ReconcilePlatformAdmin) initFramework(ctx context.Context, platformAdmi platformAdminFramework.Components = r.Configration.NoSectyComponents[platformAdmin.Spec.Version] } + yurtIotDock, err := NewYurtIoTDockComponent(platformAdmin, platformAdminFramework) + if err != nil { + return err + } + platformAdminFramework.Components = append(platformAdminFramework.Components, yurtIotDock) + // For better serialization, the serialization method of the Kubernetes runtime library is used data, err := runtime.Encode(r.yamlSerializer, platformAdminFramework) if err != nil { diff --git a/pkg/controller/platformadmin/util.go b/pkg/controller/platformadmin/util.go new file mode 100644 index 00000000000..6abc2b5e780 --- /dev/null +++ b/pkg/controller/platformadmin/util.go @@ -0,0 +1,126 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package platformadmin + +import ( + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" + + iotv1alpha2 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha2" + "github.com/openyurtio/openyurt/pkg/controller/platformadmin/config" + utils "github.com/openyurtio/openyurt/pkg/controller/platformadmin/utils" +) + +// NewYurtIoTDockComponent initialize the configuration of yurt-iot-dock component +func NewYurtIoTDockComponent(platformAdmin *iotv1alpha2.PlatformAdmin, platformAdminFramework *PlatformAdminFramework) (*config.Component, error) { + var yurtIotDockComponent config.Component + + // If the configuration of the yurt-iot-dock component that customized in the platformAdminFramework + for _, cp := range platformAdminFramework.Components { + if cp.Name != utils.IotDockName { + continue + } + return cp, nil + } + + // Otherwise, the default configuration is used to start + ver, ns, err := utils.DefaultVersion(platformAdmin) + if err != nil { + return nil, err + } + + yurtIotDockComponent.Name = utils.IotDockName + yurtIotDockComponent.Deployment = &appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": utils.IotDockName, + "control-plane": utils.IotDockControlPlane, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": utils.IotDockName, + "control-plane": utils.IotDockControlPlane, + }, + Namespace: ns, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: utils.IotDockName, + Image: fmt.Sprintf("%s:%s", utils.IotDockImage, ver), + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{ + "--health-probe-bind-address=:8081", + "--metrics-bind-address=127.0.0.1:8080", + "--leader-elect=false", + fmt.Sprintf("--namespace=%s", ns), + }, + LivenessProbe: &corev1.Probe{ + InitialDelaySeconds: 15, + PeriodSeconds: 20, + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(8081), + }, + }, + }, + ReadinessProbe: &corev1.Probe{ + InitialDelaySeconds: 5, + PeriodSeconds: 10, + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/readyz", + Port: intstr.FromInt(8081), + }, + }, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("512m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1024m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + }, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: pointer.Bool(false), + }, + }, + }, + TerminationGracePeriodSeconds: pointer.Int64(10), + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: pointer.Int64(65532), + }, + }, + }, + } + // YurtIoTDock doesn't need a service yet + yurtIotDockComponent.Service = nil + + return &yurtIotDockComponent, nil +} diff --git a/pkg/controller/platformadmin/utils/util.go b/pkg/controller/platformadmin/utils/conditions.go similarity index 100% rename from pkg/controller/platformadmin/utils/util.go rename to pkg/controller/platformadmin/utils/conditions.go diff --git a/pkg/controller/platformadmin/utils/version.go b/pkg/controller/platformadmin/utils/version.go new file mode 100644 index 00000000000..08542f4c97c --- /dev/null +++ b/pkg/controller/platformadmin/utils/version.go @@ -0,0 +1,31 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + iotv1alpha2 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha2" +) + +const IotDockName = "yurt-iot-dock" +const IotDockImage = "openyurt/yurt-iot-dock" +const IotDockControlPlane = "platformadmin-controller" + +func DefaultVersion(platformAdmin *iotv1alpha2.PlatformAdmin) (string, string, error) { + version := "latest" + ns := platformAdmin.Namespace + return version, ns, nil +} diff --git a/pkg/yurtiotdock/clients/edgex-foundry/device_client.go b/pkg/yurtiotdock/clients/edgex-foundry/device_client.go new file mode 100644 index 00000000000..72ef381d8fa --- /dev/null +++ b/pkg/yurtiotdock/clients/edgex-foundry/device_client.go @@ -0,0 +1,371 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package edgex_foundry + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/cookiejar" + "strings" + "time" + + "github.com/edgexfoundry/go-mod-core-contracts/v2/dtos" + "github.com/edgexfoundry/go-mod-core-contracts/v2/dtos/common" + edgex_resp "github.com/edgexfoundry/go-mod-core-contracts/v2/dtos/responses" + "github.com/go-resty/resty/v2" + "golang.org/x/net/publicsuffix" + "k8s.io/klog/v2" + + iotv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" + "github.com/openyurtio/openyurt/pkg/yurtiotdock/clients" +) + +type EdgexDeviceClient struct { + *resty.Client + CoreMetaAddr string + CoreCommandAddr string +} + +func NewEdgexDeviceClient(coreMetaAddr, coreCommandAddr string) *EdgexDeviceClient { + cookieJar, _ := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) + instance := resty.NewWithClient(&http.Client{ + Jar: cookieJar, + Timeout: 10 * time.Second, + }) + return &EdgexDeviceClient{ + Client: instance, + CoreMetaAddr: coreMetaAddr, + CoreCommandAddr: coreCommandAddr, + } +} + +// Create function sends a POST request to EdgeX to add a new device +func (efc *EdgexDeviceClient) Create(ctx context.Context, device *iotv1alpha1.Device, options clients.CreateOptions) (*iotv1alpha1.Device, error) { + devs := []*iotv1alpha1.Device{device} + req := makeEdgeXDeviceRequest(devs) + klog.V(5).Infof("will add the Device: %s", device.Name) + reqBody, err := json.Marshal(req) + if err != nil { + return nil, err + } + postPath := fmt.Sprintf("http://%s%s", efc.CoreMetaAddr, DevicePath) + resp, err := efc.R(). + SetBody(reqBody).Post(postPath) + if err != nil { + return nil, err + } else if resp.StatusCode() != http.StatusMultiStatus { + return nil, fmt.Errorf("create device on edgex foundry failed, the response is : %s", resp.Body()) + } + + var edgexResps []*common.BaseWithIdResponse + if err = json.Unmarshal(resp.Body(), &edgexResps); err != nil { + return nil, err + } + createdDevice := device.DeepCopy() + if len(edgexResps) == 1 { + if edgexResps[0].StatusCode == http.StatusCreated { + createdDevice.Status.EdgeId = edgexResps[0].Id + createdDevice.Status.Synced = true + } else { + return nil, fmt.Errorf("create device on edgex foundry failed, the response is : %s", resp.Body()) + } + } else { + return nil, fmt.Errorf("edgex BaseWithIdResponse count mismatch device cound, the response is : %s", resp.Body()) + } + return createdDevice, err +} + +// Delete function sends a request to EdgeX to delete a device +func (efc *EdgexDeviceClient) Delete(ctx context.Context, name string, options clients.DeleteOptions) error { + klog.V(5).Infof("will delete the Device: %s", name) + delURL := fmt.Sprintf("http://%s%s/name/%s", efc.CoreMetaAddr, DevicePath, name) + resp, err := efc.R().Delete(delURL) + if err != nil { + return err + } + if resp.StatusCode() != http.StatusOK { + return errors.New(string(resp.Body())) + } + return nil +} + +// Update is used to set the admin or operating state of the device by unique name of the device. +// TODO support to update other fields +func (efc *EdgexDeviceClient) Update(ctx context.Context, device *iotv1alpha1.Device, options clients.UpdateOptions) (*iotv1alpha1.Device, error) { + actualDeviceName := getEdgeXName(device) + patchURL := fmt.Sprintf("http://%s%s", efc.CoreMetaAddr, DevicePath) + if device == nil { + return nil, nil + } + devs := []*iotv1alpha1.Device{device} + req := makeEdgeXDeviceUpdateRequest(devs) + reqBody, err := json.Marshal(req) + if err != nil { + return nil, err + } + rep, err := efc.R(). + SetHeader("Content-Type", "application/json"). + SetBody(reqBody). + Patch(patchURL) + if err != nil { + return nil, err + } else if rep.StatusCode() != http.StatusMultiStatus { + return nil, fmt.Errorf("failed to update device: %s, get response: %s", actualDeviceName, string(rep.Body())) + } + return device, nil +} + +// Get is used to query the device information corresponding to the device name +func (efc *EdgexDeviceClient) Get(ctx context.Context, deviceName string, options clients.GetOptions) (*iotv1alpha1.Device, error) { + klog.V(5).Infof("will get Devices: %s", deviceName) + var dResp edgex_resp.DeviceResponse + getURL := fmt.Sprintf("http://%s%s/name/%s", efc.CoreMetaAddr, DevicePath, deviceName) + resp, err := efc.R().Get(getURL) + if err != nil { + return nil, err + } + if resp.StatusCode() == http.StatusNotFound { + return nil, fmt.Errorf("Device %s not found", deviceName) + } + err = json.Unmarshal(resp.Body(), &dResp) + if err != nil { + return nil, err + } + device := toKubeDevice(dResp.Device, options.Namespace) + return &device, err +} + +// List is used to get all device objects on edge platform +// TODO:support label filtering according to options +func (efc *EdgexDeviceClient) List(ctx context.Context, options clients.ListOptions) ([]iotv1alpha1.Device, error) { + lp := fmt.Sprintf("http://%s%s/all?limit=-1", efc.CoreMetaAddr, DevicePath) + resp, err := efc.R().EnableTrace().Get(lp) + if err != nil { + return nil, err + } + var mdResp edgex_resp.MultiDevicesResponse + if err := json.Unmarshal(resp.Body(), &mdResp); err != nil { + return nil, err + } + var res []iotv1alpha1.Device + for _, dp := range mdResp.Devices { + res = append(res, toKubeDevice(dp, options.Namespace)) + } + return res, nil +} + +func (efc *EdgexDeviceClient) GetPropertyState(ctx context.Context, propertyName string, d *iotv1alpha1.Device, options clients.GetOptions) (*iotv1alpha1.ActualPropertyState, error) { + actualDeviceName := getEdgeXName(d) + // get the old property from status + oldAps, exist := d.Status.DeviceProperties[propertyName] + propertyGetURL := "" + // 1. query the Get URL of a property + if !exist || (exist && oldAps.GetURL == "") { + coreCommands, err := efc.GetCommandResponseByName(actualDeviceName) + if err != nil { + return &iotv1alpha1.ActualPropertyState{}, err + } + for _, c := range coreCommands { + if c.Name == propertyName && c.Get { + propertyGetURL = fmt.Sprintf("%s%s", c.Url, c.Path) + break + } + } + if propertyGetURL == "" { + return nil, &clients.NotFoundError{} + } + } else { + propertyGetURL = oldAps.GetURL + } + // 2. get the actual property value by the getURL + actualPropertyState := iotv1alpha1.ActualPropertyState{ + Name: propertyName, + GetURL: propertyGetURL, + } + if resp, err := efc.getPropertyState(propertyGetURL); err != nil { + return nil, err + } else { + var eResp edgex_resp.EventResponse + if err := json.Unmarshal(resp.Body(), &eResp); err != nil { + return nil, err + } + actualPropertyState.ActualValue = getPropertyValueFromEvent(propertyName, eResp.Event) + } + return &actualPropertyState, nil +} + +// getPropertyState returns different error messages according to the status code +func (efc *EdgexDeviceClient) getPropertyState(getURL string) (*resty.Response, error) { + resp, err := efc.R().Get(getURL) + if err != nil { + return resp, err + } + if resp.StatusCode() == 400 { + err = errors.New("request is in an invalid state") + } else if resp.StatusCode() == 404 { + err = errors.New("the requested resource does not exist") + } else if resp.StatusCode() == 423 { + err = errors.New("the device is locked (AdminState) or down (OperatingState)") + } else if resp.StatusCode() == 500 { + err = errors.New("an unexpected error occurred on the server") + } + return resp, err +} + +func (efc *EdgexDeviceClient) UpdatePropertyState(ctx context.Context, propertyName string, d *iotv1alpha1.Device, options clients.UpdateOptions) error { + // Get the actual device name + acturalDeviceName := getEdgeXName(d) + + dps := d.Spec.DeviceProperties[propertyName] + parameterName := dps.Name + if dps.PutURL == "" { + putCmd, err := efc.getPropertyPut(acturalDeviceName, dps.Name) + if err != nil { + return err + } + dps.PutURL = fmt.Sprintf("%s%s", putCmd.Url, putCmd.Path) + if len(putCmd.Parameters) == 1 { + parameterName = putCmd.Parameters[0].ResourceName + } + } + // set the device property to desired state + bodyMap := make(map[string]string) + bodyMap[parameterName] = dps.DesiredValue + body, _ := json.Marshal(bodyMap) + klog.V(5).Infof("setting the property to desired value", "propertyName", parameterName, "desiredValue", string(body)) + rep, err := efc.R(). + SetHeader("Content-Type", "application/json"). + SetBody(body). + Put(dps.PutURL) + if err != nil { + return err + } else if rep.StatusCode() != http.StatusOK { + return fmt.Errorf("failed to set property: %s, get response: %s", dps.Name, string(rep.Body())) + } else if rep.Body() != nil { + // If the parameters are illegal, such as out of range, the 200 status code is also returned, but the description appears in the body + a := string(rep.Body()) + if strings.Contains(a, "execWriteCmd") { + return fmt.Errorf("failed to set property: %s, get response: %s", dps.Name, string(rep.Body())) + } + } + return nil +} + +// Gets the models.Put from edgex foundry which is used to set the device property's value +func (efc *EdgexDeviceClient) getPropertyPut(deviceName, cmdName string) (dtos.CoreCommand, error) { + coreCommands, err := efc.GetCommandResponseByName(deviceName) + if err != nil { + return dtos.CoreCommand{}, err + } + for _, c := range coreCommands { + if cmdName == c.Name && c.Set { + return c, nil + } + } + return dtos.CoreCommand{}, errors.New("corresponding command is not found") +} + +// ListPropertiesState gets all the actual property information about a device +func (efc *EdgexDeviceClient) ListPropertiesState(ctx context.Context, device *iotv1alpha1.Device, options clients.ListOptions) (map[string]iotv1alpha1.DesiredPropertyState, map[string]iotv1alpha1.ActualPropertyState, error) { + actualDeviceName := getEdgeXName(device) + + dpsm := map[string]iotv1alpha1.DesiredPropertyState{} + apsm := map[string]iotv1alpha1.ActualPropertyState{} + coreCommands, err := efc.GetCommandResponseByName(actualDeviceName) + if err != nil { + return dpsm, apsm, err + } + + for _, c := range coreCommands { + // DesiredPropertyState only store the basic information and does not set DesiredValue + if c.Get { + getURL := fmt.Sprintf("%s%s", c.Url, c.Path) + aps, ok := apsm[c.Name] + if ok { + aps.GetURL = getURL + } else { + aps = iotv1alpha1.ActualPropertyState{Name: c.Name, GetURL: getURL} + } + apsm[c.Name] = aps + resp, err := efc.getPropertyState(getURL) + if err != nil { + klog.V(5).ErrorS(err, "getPropertyState failed", "propertyName", c.Name, "deviceName", actualDeviceName) + } else { + var eResp edgex_resp.EventResponse + if err := json.Unmarshal(resp.Body(), &eResp); err != nil { + klog.V(5).ErrorS(err, "failed to decode the response ", "response", resp) + continue + } + event := eResp.Event + readingName := c.Name + expectParams := c.Parameters + if len(expectParams) == 1 { + readingName = expectParams[0].ResourceName + } + klog.V(5).Infof("get reading name %s for command %s of device %s", readingName, c.Name, device.Name) + actualValue := getPropertyValueFromEvent(readingName, event) + aps.ActualValue = actualValue + apsm[c.Name] = aps + } + } + } + return dpsm, apsm, nil +} + +// The actual property value is resolved from the returned event +func getPropertyValueFromEvent(resName string, event dtos.Event) string { + actualValue := "" + for _, r := range event.Readings { + if resName == r.ResourceName { + if r.SimpleReading.Value != "" { + actualValue = r.SimpleReading.Value + } else if len(r.BinaryReading.BinaryValue) != 0 { + // TODO: how to demonstrate binary data + actualValue = fmt.Sprintf("%s:%s", r.BinaryReading.MediaType, "blob value") + } else if r.ObjectReading.ObjectValue != nil { + serializedBytes, _ := json.Marshal(r.ObjectReading.ObjectValue) + actualValue = string(serializedBytes) + } + break + } + } + return actualValue +} + +// GetCommandResponseByName gets all commands supported by the device +func (efc *EdgexDeviceClient) GetCommandResponseByName(deviceName string) ([]dtos.CoreCommand, error) { + klog.V(5).Infof("will get CommandResponses of device: %s", deviceName) + + var dcr edgex_resp.DeviceCoreCommandResponse + getURL := fmt.Sprintf("http://%s%s/name/%s", efc.CoreCommandAddr, CommandResponsePath, deviceName) + + resp, err := efc.R().Get(getURL) + if err != nil { + return nil, err + } + if resp.StatusCode() == http.StatusNotFound { + return nil, errors.New("Item not found") + } + err = json.Unmarshal(resp.Body(), &dcr) + if err != nil { + return nil, err + } + return dcr.DeviceCoreCommand.CoreCommands, nil +} diff --git a/pkg/yurtiotdock/clients/edgex-foundry/device_client_test.go b/pkg/yurtiotdock/clients/edgex-foundry/device_client_test.go new file mode 100644 index 00000000000..5873a540d74 --- /dev/null +++ b/pkg/yurtiotdock/clients/edgex-foundry/device_client_test.go @@ -0,0 +1,202 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package edgex_foundry + +import ( + "context" + "encoding/json" + "testing" + + edgex_resp "github.com/edgexfoundry/go-mod-core-contracts/v2/dtos/responses" + "github.com/jarcoal/httpmock" + "github.com/stretchr/testify/assert" + + iotv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" + "github.com/openyurtio/openyurt/pkg/yurtiotdock/clients" +) + +const ( + DeviceListMetadata = `{"apiVersion":"v2","statusCode":200,"totalCount":5,"devices":[{"created":1661829206505,"modified":1661829206505,"id":"f6255845-f4b2-4182-bd3c-abc9eac4a649","name":"Random-Float-Device","description":"Example of Device Virtual","adminState":"UNLOCKED","operatingState":"UP","labels":["device-virtual-example"],"serviceName":"device-virtual","profileName":"Random-Float-Device","autoEvents":[{"interval":"30s","onChange":false,"sourceName":"Float32"},{"interval":"30s","onChange":false,"sourceName":"Float64"}],"protocols":{"other":{"Address":"device-virtual-float-01","Protocol":"300"}}},{"created":1661829206506,"modified":1661829206506,"id":"d29efe20-fdec-4aeb-90e5-99528cb6ca28","name":"Random-Binary-Device","description":"Example of Device Virtual","adminState":"UNLOCKED","operatingState":"UP","labels":["device-virtual-example"],"serviceName":"device-virtual","profileName":"Random-Binary-Device","protocols":{"other":{"Address":"device-virtual-binary-01","Port":"300"}}},{"created":1661829206504,"modified":1661829206504,"id":"6a7f00a4-9536-48b2-9380-a9fc202ac517","name":"Random-Integer-Device","description":"Example of Device Virtual","adminState":"UNLOCKED","operatingState":"UP","labels":["device-virtual-example"],"serviceName":"device-virtual","profileName":"Random-Integer-Device","autoEvents":[{"interval":"15s","onChange":false,"sourceName":"Int8"},{"interval":"15s","onChange":false,"sourceName":"Int16"},{"interval":"15s","onChange":false,"sourceName":"Int32"},{"interval":"15s","onChange":false,"sourceName":"Int64"}],"protocols":{"other":{"Address":"device-virtual-int-01","Protocol":"300"}}},{"created":1661829206503,"modified":1661829206503,"id":"439d47a2-fa72-4c27-9f47-c19356cc0c3b","name":"Random-Boolean-Device","description":"Example of Device Virtual","adminState":"UNLOCKED","operatingState":"UP","labels":["device-virtual-example"],"serviceName":"device-virtual","profileName":"Random-Boolean-Device","autoEvents":[{"interval":"10s","onChange":false,"sourceName":"Bool"}],"protocols":{"other":{"Address":"device-virtual-bool-01","Port":"300"}}},{"created":1661829206505,"modified":1661829206505,"id":"2890ab86-3ae4-4b5e-98ab-aad85fc540e6","name":"Random-UnsignedInteger-Device","description":"Example of Device Virtual","adminState":"UNLOCKED","operatingState":"UP","labels":["device-virtual-example"],"serviceName":"device-virtual","profileName":"Random-UnsignedInteger-Device","autoEvents":[{"interval":"20s","onChange":false,"sourceName":"Uint8"},{"interval":"20s","onChange":false,"sourceName":"Uint16"},{"interval":"20s","onChange":false,"sourceName":"Uint32"},{"interval":"20s","onChange":false,"sourceName":"Uint64"}],"protocols":{"other":{"Address":"device-virtual-uint-01","Protocol":"300"}}}]}` + DeviceMetadata = `{"apiVersion":"v2","statusCode":200,"device":{"created":1661829206505,"modified":1661829206505,"id":"f6255845-f4b2-4182-bd3c-abc9eac4a649","name":"Random-Float-Device","description":"Example of Device Virtual","adminState":"UNLOCKED","operatingState":"UP","labels":["device-virtual-example"],"serviceName":"device-virtual","profileName":"Random-Float-Device","autoEvents":[{"interval":"30s","onChange":false,"sourceName":"Float32"},{"interval":"30s","onChange":false,"sourceName":"Float64"}],"protocols":{"other":{"Address":"device-virtual-float-01","Protocol":"300"}}}}` + + DeviceCreateSuccess = `[{"apiVersion":"v2","statusCode":201,"id":"2fff4f1a-7110-442f-b347-9f896338ba57"}]` + DeviceCreateFail = `[{"apiVersion":"v2","message":"device name test-Random-Float-Device already exists","statusCode":409}]` + + DeviceDeleteSuccess = `{"apiVersion":"v2","statusCode":200}` + DeviceDeleteFail = `{"apiVersion":"v2","message":"fail to query device by name test-Random-Float-Device","statusCode":404}` + + DeviceCoreCommands = `{"apiVersion":"v2","statusCode":200,"deviceCoreCommand":{"deviceName":"Random-Float-Device","profileName":"Random-Float-Device","coreCommands":[{"name":"WriteFloat32ArrayValue","set":true,"path":"/api/v2/device/name/Random-Float-Device/WriteFloat32ArrayValue","url":"http://edgex-core-command:59882","parameters":[{"resourceName":"Float32Array","valueType":"Float32Array"},{"resourceName":"EnableRandomization_Float32Array","valueType":"Bool"}]},{"name":"WriteFloat64ArrayValue","set":true,"path":"/api/v2/device/name/Random-Float-Device/WriteFloat64ArrayValue","url":"http://edgex-core-command:59882","parameters":[{"resourceName":"Float64Array","valueType":"Float64Array"},{"resourceName":"EnableRandomization_Float64Array","valueType":"Bool"}]},{"name":"Float32","get":true,"set":true,"path":"/api/v2/device/name/Random-Float-Device/Float32","url":"http://edgex-core-command:59882","parameters":[{"resourceName":"Float32","valueType":"Float32"}]},{"name":"Float64","get":true,"set":true,"path":"/api/v2/device/name/Random-Float-Device/Float64","url":"http://edgex-core-command:59882","parameters":[{"resourceName":"Float64","valueType":"Float64"}]},{"name":"Float32Array","get":true,"set":true,"path":"/api/v2/device/name/Random-Float-Device/Float32Array","url":"http://edgex-core-command:59882","parameters":[{"resourceName":"Float32Array","valueType":"Float32Array"}]},{"name":"Float64Array","get":true,"set":true,"path":"/api/v2/device/name/Random-Float-Device/Float64Array","url":"http://edgex-core-command:59882","parameters":[{"resourceName":"Float64Array","valueType":"Float64Array"}]},{"name":"WriteFloat32Value","set":true,"path":"/api/v2/device/name/Random-Float-Device/WriteFloat32Value","url":"http://edgex-core-command:59882","parameters":[{"resourceName":"Float32","valueType":"Float32"},{"resourceName":"EnableRandomization_Float32","valueType":"Bool"}]},{"name":"WriteFloat64Value","set":true,"path":"/api/v2/device/name/Random-Float-Device/WriteFloat64Value","url":"http://edgex-core-command:59882","parameters":[{"resourceName":"Float64","valueType":"Float64"},{"resourceName":"EnableRandomization_Float64","valueType":"Bool"}]}]}}` + DeviceCommandResp = `{"apiVersion":"v2","statusCode":200,"event":{"apiVersion":"v2","id":"095090e4-de39-45a1-a0fa-18bc340104e6","deviceName":"Random-Float-Device","profileName":"Random-Float-Device","sourceName":"Float32","origin":1661851070562067780,"readings":[{"id":"972bf6be-3b01-49fc-b211-a43ed51d207d","origin":1661851070562067780,"deviceName":"Random-Float-Device","resourceName":"Float32","profileName":"Random-Float-Device","valueType":"Float32","value":"-2.038811e+38"}]}}` + + DeviceUpdateSuccess = `[{"apiVersion":"v2","statusCode":200}] ` + + DeviceUpdateProperty = `{"apiVersion":"v2","statusCode":200}` +) + +var deviceClient = NewEdgexDeviceClient("edgex-core-metadata:59881", "edgex-core-command:59882") + +func Test_Get(t *testing.T) { + httpmock.ActivateNonDefault(deviceClient.Client.GetClient()) + defer httpmock.DeactivateAndReset() + httpmock.RegisterResponder("GET", "http://edgex-core-metadata:59881/api/v2/device/name/Random-Float-Device", + httpmock.NewStringResponder(200, DeviceMetadata)) + + device, err := deviceClient.Get(context.TODO(), "Random-Float-Device", clients.GetOptions{Namespace: "default"}) + assert.Nil(t, err) + + assert.Equal(t, "Random-Float-Device", device.Spec.Profile) +} + +func Test_List(t *testing.T) { + + httpmock.ActivateNonDefault(deviceClient.Client.GetClient()) + defer httpmock.DeactivateAndReset() + + httpmock.RegisterResponder("GET", "http://edgex-core-metadata:59881/api/v2/device/all?limit=-1", + httpmock.NewStringResponder(200, DeviceListMetadata)) + + devices, err := deviceClient.List(context.TODO(), clients.ListOptions{Namespace: "default"}) + assert.Nil(t, err) + + assert.Equal(t, len(devices), 5) +} + +func Test_Create(t *testing.T) { + httpmock.ActivateNonDefault(deviceClient.Client.GetClient()) + defer httpmock.DeactivateAndReset() + + httpmock.RegisterResponder("POST", "http://edgex-core-metadata:59881/api/v2/device", + httpmock.NewStringResponder(207, DeviceCreateSuccess)) + + var resp edgex_resp.DeviceResponse + + err := json.Unmarshal([]byte(DeviceMetadata), &resp) + assert.Nil(t, err) + + device := toKubeDevice(resp.Device, "default") + device.Name = "test-Random-Float-Device" + + create, err := deviceClient.Create(context.TODO(), &device, clients.CreateOptions{}) + assert.Nil(t, err) + + assert.Equal(t, "test-Random-Float-Device", create.Name) + + httpmock.RegisterResponder("POST", "http://edgex-core-metadata:59881/api/v2/device", + httpmock.NewStringResponder(207, DeviceCreateFail)) + + create, err = deviceClient.Create(context.TODO(), &device, clients.CreateOptions{}) + assert.NotNil(t, err) + assert.Nil(t, create) +} + +func Test_Delete(t *testing.T) { + httpmock.ActivateNonDefault(deviceClient.Client.GetClient()) + defer httpmock.DeactivateAndReset() + httpmock.RegisterResponder("DELETE", "http://edgex-core-metadata:59881/api/v2/device/name/test-Random-Float-Device", + httpmock.NewStringResponder(200, DeviceDeleteSuccess)) + + err := deviceClient.Delete(context.TODO(), "test-Random-Float-Device", clients.DeleteOptions{}) + assert.Nil(t, err) + + httpmock.RegisterResponder("DELETE", "http://edgex-core-metadata:59881/api/v2/device/name/test-Random-Float-Device", + httpmock.NewStringResponder(404, DeviceDeleteFail)) + + err = deviceClient.Delete(context.TODO(), "test-Random-Float-Device", clients.DeleteOptions{}) + assert.NotNil(t, err) +} + +func Test_GetPropertyState(t *testing.T) { + + httpmock.ActivateNonDefault(deviceClient.Client.GetClient()) + defer httpmock.DeactivateAndReset() + + httpmock.RegisterResponder("GET", "http://edgex-core-command:59882/api/v2/device/name/Random-Float-Device", + httpmock.NewStringResponder(200, DeviceCoreCommands)) + httpmock.RegisterResponder("GET", "http://edgex-core-command:59882/api/v2/device/name/Random-Float-Device/Float32", + httpmock.NewStringResponder(200, DeviceCommandResp)) + + var resp edgex_resp.DeviceResponse + + err := json.Unmarshal([]byte(DeviceMetadata), &resp) + assert.Nil(t, err) + + device := toKubeDevice(resp.Device, "default") + + _, err = deviceClient.GetPropertyState(context.TODO(), "Float32", &device, clients.GetOptions{}) + assert.Nil(t, err) +} + +func Test_ListPropertiesState(t *testing.T) { + httpmock.ActivateNonDefault(deviceClient.Client.GetClient()) + defer httpmock.DeactivateAndReset() + + httpmock.RegisterResponder("GET", "http://edgex-core-command:59882/api/v2/device/name/Random-Float-Device", + httpmock.NewStringResponder(200, DeviceCoreCommands)) + + var resp edgex_resp.DeviceResponse + + err := json.Unmarshal([]byte(DeviceMetadata), &resp) + assert.Nil(t, err) + + device := toKubeDevice(resp.Device, "default") + + _, _, err = deviceClient.ListPropertiesState(context.TODO(), &device, clients.ListOptions{}) + assert.Nil(t, err) +} + +func Test_UpdateDevice(t *testing.T) { + httpmock.ActivateNonDefault(deviceClient.Client.GetClient()) + defer httpmock.DeactivateAndReset() + + httpmock.RegisterResponder("PATCH", "http://edgex-core-metadata:59881/api/v2/device", + httpmock.NewStringResponder(207, DeviceUpdateSuccess)) + + var resp edgex_resp.DeviceResponse + + err := json.Unmarshal([]byte(DeviceMetadata), &resp) + assert.Nil(t, err) + + device := toKubeDevice(resp.Device, "default") + device.Spec.AdminState = "LOCKED" + + _, err = deviceClient.Update(context.TODO(), &device, clients.UpdateOptions{}) + assert.Nil(t, err) +} + +func Test_UpdatePropertyState(t *testing.T) { + httpmock.ActivateNonDefault(deviceClient.Client.GetClient()) + defer httpmock.DeactivateAndReset() + + httpmock.RegisterResponder("GET", "http://edgex-core-command:59882/api/v2/device/name/Random-Float-Device", + httpmock.NewStringResponder(200, DeviceCoreCommands)) + + httpmock.RegisterResponder("PUT", "http://edgex-core-command:59882/api/v2/device/name/Random-Float-Device/Float32", + httpmock.NewStringResponder(200, DeviceUpdateSuccess)) + var resp edgex_resp.DeviceResponse + err := json.Unmarshal([]byte(DeviceMetadata), &resp) + assert.Nil(t, err) + + device := toKubeDevice(resp.Device, "default") + device.Spec.DeviceProperties = map[string]iotv1alpha1.DesiredPropertyState{ + "Float32": { + Name: "Float32", + DesiredValue: "66.66", + }, + } + + err = deviceClient.UpdatePropertyState(context.TODO(), "Float32", &device, clients.UpdateOptions{}) + assert.Nil(t, err) +} diff --git a/pkg/yurtiotdock/clients/edgex-foundry/deviceprofile_client.go b/pkg/yurtiotdock/clients/edgex-foundry/deviceprofile_client.go new file mode 100644 index 00000000000..2cd9c9d40c6 --- /dev/null +++ b/pkg/yurtiotdock/clients/edgex-foundry/deviceprofile_client.go @@ -0,0 +1,141 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package edgex_foundry + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/edgexfoundry/go-mod-core-contracts/v2/dtos/common" + "github.com/edgexfoundry/go-mod-core-contracts/v2/dtos/responses" + "github.com/go-resty/resty/v2" + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" + devcli "github.com/openyurtio/openyurt/pkg/yurtiotdock/clients" +) + +type EdgexDeviceProfile struct { + *resty.Client + CoreMetaAddr string +} + +func NewEdgexDeviceProfile(coreMetaAddr string) *EdgexDeviceProfile { + return &EdgexDeviceProfile{ + Client: resty.New(), + CoreMetaAddr: coreMetaAddr, + } +} + +// TODO: support label filtering +func getListDeviceProfileURL(address string, opts devcli.ListOptions) (string, error) { + url := fmt.Sprintf("http://%s%s/all?limit=-1", address, DeviceProfilePath) + return url, nil +} + +func (cdc *EdgexDeviceProfile) List(ctx context.Context, opts devcli.ListOptions) ([]v1alpha1.DeviceProfile, error) { + klog.V(5).Info("will list DeviceProfiles") + lp, err := getListDeviceProfileURL(cdc.CoreMetaAddr, opts) + if err != nil { + return nil, err + } + resp, err := cdc.R().EnableTrace().Get(lp) + if err != nil { + return nil, err + } + var mdpResp responses.MultiDeviceProfilesResponse + if err := json.Unmarshal(resp.Body(), &mdpResp); err != nil { + return nil, err + } + var deviceProfiles []v1alpha1.DeviceProfile + for _, dp := range mdpResp.Profiles { + deviceProfiles = append(deviceProfiles, toKubeDeviceProfile(&dp, opts.Namespace)) + } + return deviceProfiles, nil +} + +func (cdc *EdgexDeviceProfile) Get(ctx context.Context, name string, opts devcli.GetOptions) (*v1alpha1.DeviceProfile, error) { + klog.V(5).Infof("will get DeviceProfiles: %s", name) + var dpResp responses.DeviceProfileResponse + getURL := fmt.Sprintf("http://%s%s/name/%s", cdc.CoreMetaAddr, DeviceProfilePath, name) + resp, err := cdc.R().Get(getURL) + if err != nil { + return nil, err + } + if resp.StatusCode() == http.StatusNotFound { + return nil, fmt.Errorf("DeviceProfile %s not found", name) + } + if err = json.Unmarshal(resp.Body(), &dpResp); err != nil { + return nil, err + } + kubedp := toKubeDeviceProfile(&dpResp.Profile, opts.Namespace) + return &kubedp, nil +} + +func (cdc *EdgexDeviceProfile) Create(ctx context.Context, deviceProfile *v1alpha1.DeviceProfile, opts devcli.CreateOptions) (*v1alpha1.DeviceProfile, error) { + dps := []*v1alpha1.DeviceProfile{deviceProfile} + req := makeEdgeXDeviceProfilesRequest(dps) + klog.V(5).Infof("will add the DeviceProfile: %s", deviceProfile.Name) + reqBody, err := json.Marshal(req) + if err != nil { + return nil, err + } + postURL := fmt.Sprintf("http://%s%s", cdc.CoreMetaAddr, DeviceProfilePath) + resp, err := cdc.R().SetBody(reqBody).Post(postURL) + if err != nil { + return nil, err + } + if resp.StatusCode() != http.StatusMultiStatus { + return nil, fmt.Errorf("create edgex deviceProfile err: %s", string(resp.Body())) // 假定 resp.Body() 存了 msg 信息 + } + var edgexResps []*common.BaseWithIdResponse + if err = json.Unmarshal(resp.Body(), &edgexResps); err != nil { + return nil, err + } + createdDeviceProfile := deviceProfile.DeepCopy() + if len(edgexResps) == 1 { + if edgexResps[0].StatusCode == http.StatusCreated { + createdDeviceProfile.Status.EdgeId = edgexResps[0].Id + createdDeviceProfile.Status.Synced = true + } else { + return nil, fmt.Errorf("create deviceprofile on edgex foundry failed, the response is : %s", resp.Body()) + } + } else { + return nil, fmt.Errorf("edgex BaseWithIdResponse count mismatch DeviceProfile count, the response is : %s", resp.Body()) + } + return createdDeviceProfile, err +} + +// TODO: edgex does not support update DeviceProfile +func (cdc *EdgexDeviceProfile) Update(ctx context.Context, deviceProfile *v1alpha1.DeviceProfile, opts devcli.UpdateOptions) (*v1alpha1.DeviceProfile, error) { + return nil, nil +} + +func (cdc *EdgexDeviceProfile) Delete(ctx context.Context, name string, opts devcli.DeleteOptions) error { + klog.V(5).Infof("will delete the DeviceProfile: %s", name) + delURL := fmt.Sprintf("http://%s%s/name/%s", cdc.CoreMetaAddr, DeviceProfilePath, name) + resp, err := cdc.R().Delete(delURL) + if err != nil { + return err + } + if resp.StatusCode() != http.StatusOK { + return fmt.Errorf("delete edgex deviceProfile err: %s", string(resp.Body())) // 假定 resp.Body() 存了 msg 信息 + } + return nil +} diff --git a/pkg/yurtiotdock/clients/edgex-foundry/deviceprofile_client_test.go b/pkg/yurtiotdock/clients/edgex-foundry/deviceprofile_client_test.go new file mode 100644 index 00000000000..0f32eecf687 --- /dev/null +++ b/pkg/yurtiotdock/clients/edgex-foundry/deviceprofile_client_test.go @@ -0,0 +1,107 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package edgex_foundry + +import ( + "context" + "encoding/json" + "testing" + + edgex_resp "github.com/edgexfoundry/go-mod-core-contracts/v2/dtos/responses" + "github.com/jarcoal/httpmock" + "github.com/stretchr/testify/assert" + + "github.com/openyurtio/openyurt/pkg/yurtiotdock/clients" +) + +const ( + DeviceProfileListMetaData = `{"apiVersion":"v2","statusCode":200,"totalCount":5,"profiles":[{"created":1661829206499,"modified":1661829206499,"id":"cf624c1f-c93a-48c0-b327-b00c7dc171f1","name":"Random-Binary-Device","manufacturer":"IOTech","description":"Example of Device-Virtual","model":"Device-Virtual-01","labels":["device-virtual-example"],"deviceResources":[{"description":"Generate random binary value","name":"Binary","isHidden":false,"tag":"","properties":{"valueType":"Binary","readWrite":"R","units":"","minimum":"","maximum":"","defaultValue":"","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":"random"},"attributes":null}],"deviceCommands":[]},{"created":1661829206501,"modified":1661829206501,"id":"adeafefa-2d11-4eee-8fe9-a4742f85f7fb","name":"Random-UnsignedInteger-Device","manufacturer":"IOTech","description":"Example of Device-Virtual","model":"Device-Virtual-01","labels":["device-virtual-example"],"deviceResources":[{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Uint8","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Uint16","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Uint32","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Uint64","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random uint8 value","name":"Uint8","isHidden":false,"tag":"","properties":{"valueType":"Uint8","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"0","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random uint16 value","name":"Uint16","isHidden":false,"tag":"","properties":{"valueType":"Uint16","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"0","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random uint32 value","name":"Uint32","isHidden":false,"tag":"","properties":{"valueType":"Uint32","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"0","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random uint64 value","name":"Uint64","isHidden":false,"tag":"","properties":{"valueType":"Uint64","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"0","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Uint8Array","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Uint16Array","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Uint32Array","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Uint64Array","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random uint8 array value","name":"Uint8Array","isHidden":false,"tag":"","properties":{"valueType":"Uint8Array","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"[0]","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random uint16 array value","name":"Uint16Array","isHidden":false,"tag":"","properties":{"valueType":"Uint16Array","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"[0]","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random uint32 array value","name":"Uint32Array","isHidden":false,"tag":"","properties":{"valueType":"Uint32Array","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"[0]","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random uint64 array value","name":"Uint64Array","isHidden":false,"tag":"","properties":{"valueType":"Uint64Array","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"[0]","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null}],"deviceCommands":[{"name":"WriteUint8Value","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Uint8","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Uint8","defaultValue":"false","mappings":null}]},{"name":"WriteUint16Value","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Uint16","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Uint16","defaultValue":"false","mappings":null}]},{"name":"WriteUint32Value","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Uint32","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Uint32","defaultValue":"false","mappings":null}]},{"name":"WriteUint64Value","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Uint64","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Uint64","defaultValue":"false","mappings":null}]},{"name":"WriteUint8ArrayValue","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Uint8Array","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Uint8Array","defaultValue":"false","mappings":null}]},{"name":"WriteUint16ArrayValue","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Uint16Array","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Uint16Array","defaultValue":"false","mappings":null}]},{"name":"WriteUint32ArrayValue","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Uint32Array","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Uint32Array","defaultValue":"false","mappings":null}]},{"name":"WriteUint64ArrayValue","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Uint64Array","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Uint64Array","defaultValue":"false","mappings":null}]}]},{"created":1661829206500,"modified":1661829206500,"id":"67f4a5a1-06e6-4051-b71d-655ec5dd4eb2","name":"Random-Integer-Device","manufacturer":"IOTech","description":"Example of Device-Virtual","model":"Device-Virtual-01","labels":["device-virtual-example"],"deviceResources":[{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Int8","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Int16","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Int32","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Int64","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random int8 value","name":"Int8","isHidden":false,"tag":"","properties":{"valueType":"Int8","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"0","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random int16 value","name":"Int16","isHidden":false,"tag":"","properties":{"valueType":"Int16","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"0","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random int32 value","name":"Int32","isHidden":false,"tag":"","properties":{"valueType":"Int32","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"0","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random int64 value","name":"Int64","isHidden":false,"tag":"","properties":{"valueType":"Int64","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"0","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Int8Array","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Int16Array","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Int32Array","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Int64Array","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random int8 array value","name":"Int8Array","isHidden":false,"tag":"","properties":{"valueType":"Int8Array","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"[0]","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random int16 array value","name":"Int16Array","isHidden":false,"tag":"","properties":{"valueType":"Int16Array","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"[0]","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random int32 array value","name":"Int32Array","isHidden":false,"tag":"","properties":{"valueType":"Int32Array","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"[0]","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random int64 array value","name":"Int64Array","isHidden":false,"tag":"","properties":{"valueType":"Int64Array","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"[0]","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null}],"deviceCommands":[{"name":"WriteInt8Value","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Int8","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Int8","defaultValue":"false","mappings":null}]},{"name":"WriteInt16Value","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Int16","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Int16","defaultValue":"false","mappings":null}]},{"name":"WriteInt32Value","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Int32","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Int32","defaultValue":"false","mappings":null}]},{"name":"WriteInt64Value","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Int64","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Int64","defaultValue":"false","mappings":null}]},{"name":"WriteInt8ArrayValue","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Int8Array","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Int8Array","defaultValue":"false","mappings":null}]},{"name":"WriteInt16ArrayValue","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Int16Array","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Int16Array","defaultValue":"false","mappings":null}]},{"name":"WriteInt32ArrayValue","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Int32Array","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Int32Array","defaultValue":"false","mappings":null}]},{"name":"WriteInt64ArrayValue","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Int64Array","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Int64Array","defaultValue":"false","mappings":null}]}]},{"created":1661829206500,"modified":1661829206500,"id":"30b8448f-0532-44fb-aed7-5fe4bca16f9a","name":"Random-Float-Device","manufacturer":"IOTech","description":"Example of Device-Virtual","model":"Device-Virtual-01","labels":["device-virtual-example"],"deviceResources":[{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Float32","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Float64","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random float32 value","name":"Float32","isHidden":false,"tag":"","properties":{"valueType":"Float32","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"0","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random float64 value","name":"Float64","isHidden":false,"tag":"","properties":{"valueType":"Float64","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"0","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Float32Array","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Float64Array","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random float32 array value","name":"Float32Array","isHidden":false,"tag":"","properties":{"valueType":"Float32Array","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"[0]","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random float64 array value","name":"Float64Array","isHidden":false,"tag":"","properties":{"valueType":"Float64Array","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"[0]","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null}],"deviceCommands":[{"name":"WriteFloat32Value","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Float32","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Float32","defaultValue":"false","mappings":null}]},{"name":"WriteFloat64Value","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Float64","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Float64","defaultValue":"false","mappings":null}]},{"name":"WriteFloat32ArrayValue","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Float32Array","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Float32Array","defaultValue":"false","mappings":null}]},{"name":"WriteFloat64ArrayValue","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Float64Array","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Float64Array","defaultValue":"false","mappings":null}]}]},{"created":1661829206499,"modified":1661829206499,"id":"01dfe04d-f361-41fd-b1c4-7ca0718f461a","name":"Random-Boolean-Device","manufacturer":"IOTech","description":"Example of Device-Virtual","model":"Device-Virtual-01","labels":["device-virtual-example"],"deviceResources":[{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Bool","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random boolean value","name":"Bool","isHidden":false,"tag":"","properties":{"valueType":"Bool","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_BoolArray","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random boolean array value","name":"BoolArray","isHidden":false,"tag":"","properties":{"valueType":"BoolArray","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"[true]","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null}],"deviceCommands":[{"name":"WriteBoolValue","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Bool","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Bool","defaultValue":"false","mappings":null}]},{"name":"WriteBoolArrayValue","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"BoolArray","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_BoolArray","defaultValue":"false","mappings":null}]}]}]}` + DeviceProfileMetaData = `{"apiVersion":"v2","statusCode":200,"profile":{"created":1661829206499,"modified":1661829206499,"id":"01dfe04d-f361-41fd-b1c4-7ca0718f461a","name":"Random-Boolean-Device","manufacturer":"IOTech","description":"Example of Device-Virtual","model":"Device-Virtual-01","labels":["device-virtual-example"],"deviceResources":[{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_Bool","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random boolean value","name":"Bool","isHidden":false,"tag":"","properties":{"valueType":"Bool","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"used to decide whether to re-generate a random value","name":"EnableRandomization_BoolArray","isHidden":true,"tag":"","properties":{"valueType":"Bool","readWrite":"W","units":"","minimum":"","maximum":"","defaultValue":"true","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null},{"description":"Generate random boolean array value","name":"BoolArray","isHidden":false,"tag":"","properties":{"valueType":"BoolArray","readWrite":"RW","units":"","minimum":"","maximum":"","defaultValue":"[true]","mask":"","shift":"","scale":"","offset":"","base":"","assertion":"","mediaType":""},"attributes":null}],"deviceCommands":[{"name":"WriteBoolValue","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"Bool","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_Bool","defaultValue":"false","mappings":null}]},{"name":"WriteBoolArrayValue","isHidden":false,"readWrite":"W","resourceOperations":[{"deviceResource":"BoolArray","defaultValue":"","mappings":null},{"deviceResource":"EnableRandomization_BoolArray","defaultValue":"false","mappings":null}]}]}}` + + ProfileCreateSuccess = `[{"apiVersion":"v2","statusCode":201,"id":"a583b97d-7c4d-4b7c-8b93-51da9e68518c"}]` + ProfileCreateFail = `[{"apiVersion":"v2","message":"device profile name test-Random-Boolean-Device exists","statusCode":409}]` + + ProfileDeleteSuccess = `{"apiVersion":"v2","statusCode":200}` + ProfileDeleteFail = `{"apiVersion":"v2","message":"fail to delete the device profile with name test-Random-Boolean-Device","statusCode":404}` +) + +var profileClient = NewEdgexDeviceProfile("edgex-core-metadata:59881") + +func Test_ListProfile(t *testing.T) { + + httpmock.ActivateNonDefault(profileClient.Client.GetClient()) + defer httpmock.DeactivateAndReset() + + httpmock.RegisterResponder("GET", "http://edgex-core-metadata:59881/api/v2/deviceprofile/all?limit=-1", + httpmock.NewStringResponder(200, DeviceProfileListMetaData)) + profiles, err := profileClient.List(context.TODO(), clients.ListOptions{Namespace: "default"}) + assert.Nil(t, err) + + assert.Equal(t, 5, len(profiles)) +} + +func Test_GetProfile(T *testing.T) { + httpmock.ActivateNonDefault(profileClient.Client.GetClient()) + defer httpmock.DeactivateAndReset() + + httpmock.RegisterResponder("GET", "http://edgex-core-metadata:59881/api/v2/deviceprofile/name/Random-Boolean-Device", + httpmock.NewStringResponder(200, DeviceProfileMetaData)) + + _, err := profileClient.Get(context.TODO(), "Random-Boolean-Device", clients.GetOptions{Namespace: "default"}) + assert.Nil(T, err) +} + +func Test_CreateProfile(t *testing.T) { + httpmock.ActivateNonDefault(profileClient.Client.GetClient()) + defer httpmock.DeactivateAndReset() + + httpmock.RegisterResponder("POST", "http://edgex-core-metadata:59881/api/v2/deviceprofile", + httpmock.NewStringResponder(207, ProfileCreateSuccess)) + + var resp edgex_resp.DeviceProfileResponse + + err := json.Unmarshal([]byte(DeviceProfileMetaData), &resp) + assert.Nil(t, err) + + profile := toKubeDeviceProfile(&resp.Profile, "default") + profile.Name = "test-Random-Boolean-Device" + + _, err = profileClient.Create(context.TODO(), &profile, clients.CreateOptions{}) + assert.Nil(t, err) + + httpmock.RegisterResponder("POST", "http://edgex-core-metadata:59881/api/v2/deviceprofile", + httpmock.NewStringResponder(207, ProfileCreateFail)) + + _, err = profileClient.Create(context.TODO(), &profile, clients.CreateOptions{}) + assert.NotNil(t, err) +} + +func Test_DeleteProfile(t *testing.T) { + httpmock.ActivateNonDefault(profileClient.Client.GetClient()) + defer httpmock.DeactivateAndReset() + + httpmock.RegisterResponder("DELETE", "http://edgex-core-metadata:59881/api/v2/deviceprofile/name/test-Random-Boolean-Device", + httpmock.NewStringResponder(200, ProfileDeleteSuccess)) + + err := profileClient.Delete(context.TODO(), "test-Random-Boolean-Device", clients.DeleteOptions{}) + assert.Nil(t, err) + + httpmock.RegisterResponder("DELETE", "http://edgex-core-metadata:59881/api/v2/deviceprofile/name/test-Random-Boolean-Device", + httpmock.NewStringResponder(404, ProfileDeleteFail)) + + err = profileClient.Delete(context.TODO(), "test-Random-Boolean-Device", clients.DeleteOptions{}) + assert.NotNil(t, err) +} diff --git a/pkg/yurtiotdock/clients/edgex-foundry/deviceservice_client.go b/pkg/yurtiotdock/clients/edgex-foundry/deviceservice_client.go new file mode 100644 index 00000000000..c7cd9f70f24 --- /dev/null +++ b/pkg/yurtiotdock/clients/edgex-foundry/deviceservice_client.go @@ -0,0 +1,166 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package edgex_foundry + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/edgexfoundry/go-mod-core-contracts/v2/dtos/common" + "github.com/edgexfoundry/go-mod-core-contracts/v2/dtos/responses" + "github.com/go-resty/resty/v2" + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" + edgeCli "github.com/openyurtio/openyurt/pkg/yurtiotdock/clients" +) + +type EdgexDeviceServiceClient struct { + *resty.Client + CoreMetaAddr string +} + +func NewEdgexDeviceServiceClient(coreMetaAddr string) *EdgexDeviceServiceClient { + return &EdgexDeviceServiceClient{ + Client: resty.New(), + CoreMetaAddr: coreMetaAddr, + } +} + +// Create function sends a POST request to EdgeX to add a new deviceService +func (eds *EdgexDeviceServiceClient) Create(ctx context.Context, deviceService *v1alpha1.DeviceService, options edgeCli.CreateOptions) (*v1alpha1.DeviceService, error) { + dss := []*v1alpha1.DeviceService{deviceService} + req := makeEdgeXDeviceService(dss) + klog.V(5).InfoS("will add the DeviceServices", "DeviceService", deviceService.Name) + jsonBody, err := json.Marshal(req) + if err != nil { + return nil, err + } + postPath := fmt.Sprintf("http://%s%s", eds.CoreMetaAddr, DeviceServicePath) + resp, err := eds.R(). + SetBody(jsonBody).Post(postPath) + if err != nil { + return nil, err + } else if resp.StatusCode() != http.StatusMultiStatus { + return nil, fmt.Errorf("create DeviceService on edgex foundry failed, the response is : %s", resp.Body()) + } + + var edgexResps []*common.BaseWithIdResponse + if err = json.Unmarshal(resp.Body(), &edgexResps); err != nil { + return nil, err + } + createdDeviceService := deviceService.DeepCopy() + if len(edgexResps) == 1 { + if edgexResps[0].StatusCode == http.StatusCreated { + createdDeviceService.Status.EdgeId = edgexResps[0].Id + createdDeviceService.Status.Synced = true + } else { + return nil, fmt.Errorf("create DeviceService on edgex foundry failed, the response is : %s", resp.Body()) + } + } else { + return nil, fmt.Errorf("edgex BaseWithIdResponse count mismatch DeviceService count, the response is : %s", resp.Body()) + } + return createdDeviceService, err +} + +// Delete function sends a request to EdgeX to delete a deviceService +func (eds *EdgexDeviceServiceClient) Delete(ctx context.Context, name string, option edgeCli.DeleteOptions) error { + klog.V(5).InfoS("will delete the DeviceService", "DeviceService", name) + delURL := fmt.Sprintf("http://%s%s/name/%s", eds.CoreMetaAddr, DeviceServicePath, name) + resp, err := eds.R().Delete(delURL) + if err != nil { + return err + } + if resp.StatusCode() != http.StatusOK { + return fmt.Errorf("delete edgex deviceservice err: %s", string(resp.Body())) + } + return nil +} + +// Update is used to set the admin or operating state of the deviceService by unique name of the deviceService. +// TODO support to update other fields +func (eds *EdgexDeviceServiceClient) Update(ctx context.Context, ds *v1alpha1.DeviceService, options edgeCli.UpdateOptions) (*v1alpha1.DeviceService, error) { + patchURL := fmt.Sprintf("http://%s%s", eds.CoreMetaAddr, DeviceServicePath) + if ds == nil { + return nil, nil + } + + if ds.Status.EdgeId == "" { + return nil, fmt.Errorf("failed to update deviceservice %s with empty edgex id", ds.Name) + } + edgeDs := toEdgexDeviceService(ds) + edgeDs.Id = ds.Status.EdgeId + dsJson, err := json.Marshal(&edgeDs) + if err != nil { + return nil, err + } + resp, err := eds.R(). + SetBody(dsJson).Patch(patchURL) + if err != nil { + return nil, err + } + + if resp.StatusCode() == http.StatusOK || resp.StatusCode() == http.StatusMultiStatus { + return ds, nil + } else { + return nil, fmt.Errorf("request to patch deviceservice failed, errcode:%d", resp.StatusCode()) + } +} + +// Get is used to query the deviceService information corresponding to the deviceService name +func (eds *EdgexDeviceServiceClient) Get(ctx context.Context, name string, options edgeCli.GetOptions) (*v1alpha1.DeviceService, error) { + klog.V(5).InfoS("will get DeviceServices", "DeviceService", name) + var dsResp responses.DeviceServiceResponse + getURL := fmt.Sprintf("http://%s%s/name/%s", eds.CoreMetaAddr, DeviceServicePath, name) + resp, err := eds.R().Get(getURL) + if err != nil { + return nil, err + } + if resp.StatusCode() == http.StatusNotFound { + return nil, fmt.Errorf("deviceservice %s not found", name) + } + err = json.Unmarshal(resp.Body(), &dsResp) + if err != nil { + return nil, err + } + ds := toKubeDeviceService(dsResp.Service, options.Namespace) + return &ds, nil +} + +// List is used to get all deviceService objects on edge platform +// The Hanoi version currently supports only a single label and does not support other filters +func (eds *EdgexDeviceServiceClient) List(ctx context.Context, options edgeCli.ListOptions) ([]v1alpha1.DeviceService, error) { + klog.V(5).Info("will list DeviceServices") + lp := fmt.Sprintf("http://%s%s/all?limit=-1", eds.CoreMetaAddr, DeviceServicePath) + resp, err := eds.R(). + EnableTrace(). + Get(lp) + if err != nil { + return nil, err + } + var mdsResponse responses.MultiDeviceServicesResponse + if err := json.Unmarshal(resp.Body(), &mdsResponse); err != nil { + return nil, err + } + var res []v1alpha1.DeviceService + for _, ds := range mdsResponse.Services { + res = append(res, toKubeDeviceService(ds, options.Namespace)) + } + return res, nil +} diff --git a/pkg/yurtiotdock/clients/edgex-foundry/deviceservice_client_test.go b/pkg/yurtiotdock/clients/edgex-foundry/deviceservice_client_test.go new file mode 100644 index 00000000000..c00d7b9118e --- /dev/null +++ b/pkg/yurtiotdock/clients/edgex-foundry/deviceservice_client_test.go @@ -0,0 +1,126 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package edgex_foundry + +import ( + "context" + "encoding/json" + "testing" + + edgex_resp "github.com/edgexfoundry/go-mod-core-contracts/v2/dtos/responses" + "github.com/jarcoal/httpmock" + "github.com/stretchr/testify/assert" + + "github.com/openyurtio/openyurt/pkg/yurtiotdock/clients" +) + +const ( + DeviceServiceListMetaData = `{"apiVersion":"v2","statusCode":200,"totalCount":1,"services":[{"created":1661829206490,"modified":1661850999190,"id":"74516e96-973d-4cad-bad1-afd4b3a8ea46","name":"device-virtual","baseAddress":"http://edgex-device-virtual:59900","adminState":"UNLOCKED"}]}` + DeviceServiceMetaData = `{"apiVersion":"v2","statusCode":200,"service":{"created":1661829206490,"modified":1661850999190,"id":"74516e96-973d-4cad-bad1-afd4b3a8ea46","name":"device-virtual","baseAddress":"http://edgex-device-virtual:59900","adminState":"UNLOCKED"}}` + ServiceCreateSuccess = `[{"apiVersion":"v2","statusCode":201,"id":"a583b97d-7c4d-4b7c-8b93-51da9e68518c"}]` + ServiceCreateFail = `[{"apiVersion":"v2","message":"device service name test-device-virtual exists","statusCode":409}]` + + ServiceDeleteSuccess = `{"apiVersion":"v2","statusCode":200}` + ServiceDeleteFail = `{"apiVersion":"v2","message":"fail to delete the device profile with name test-Random-Boolean-Device","statusCode":404}` + + ServiceUpdateSuccess = `[{"apiVersion":"v2","statusCode":200}]` + ServiceUpdateFail = `[{"apiVersion":"v2","message":"fail to query object *models.DeviceService, because id: md|ds:01dfe04d-f361-41fd-b1c4-7ca0718f461a doesn't exist in the database","statusCode":404}]` +) + +var serviceClient = NewEdgexDeviceServiceClient("edgex-core-metadata:59881") + +func Test_GetService(t *testing.T) { + httpmock.ActivateNonDefault(serviceClient.Client.GetClient()) + defer httpmock.DeactivateAndReset() + + httpmock.RegisterResponder("GET", "http://edgex-core-metadata:59881/api/v2/deviceservice/name/device-virtual", + httpmock.NewStringResponder(200, DeviceServiceMetaData)) + + _, err := serviceClient.Get(context.TODO(), "device-virtual", clients.GetOptions{Namespace: "default"}) + assert.Nil(t, err) +} + +func Test_ListService(t *testing.T) { + httpmock.ActivateNonDefault(serviceClient.Client.GetClient()) + defer httpmock.DeactivateAndReset() + + httpmock.RegisterResponder("GET", "http://edgex-core-metadata:59881/api/v2/deviceservice/all?limit=-1", + httpmock.NewStringResponder(200, DeviceServiceListMetaData)) + + services, err := serviceClient.List(context.TODO(), clients.ListOptions{}) + assert.Nil(t, err) + assert.Equal(t, 1, len(services)) +} + +func Test_CreateService(t *testing.T) { + httpmock.ActivateNonDefault(serviceClient.Client.GetClient()) + defer httpmock.DeactivateAndReset() + + httpmock.RegisterResponder("POST", "http://edgex-core-metadata:59881/api/v2/deviceservice", + httpmock.NewStringResponder(207, ServiceCreateSuccess)) + + var resp edgex_resp.DeviceServiceResponse + + err := json.Unmarshal([]byte(DeviceServiceMetaData), &resp) + assert.Nil(t, err) + + service := toKubeDeviceService(resp.Service, "default") + service.Name = "test-device-virtual" + + _, err = serviceClient.Create(context.TODO(), &service, clients.CreateOptions{}) + assert.Nil(t, err) + + httpmock.RegisterResponder("POST", "http://edgex-core-metadata:59881/api/v2/deviceservice", + httpmock.NewStringResponder(207, ServiceCreateFail)) +} + +func Test_DeleteService(t *testing.T) { + httpmock.ActivateNonDefault(serviceClient.Client.GetClient()) + defer httpmock.DeactivateAndReset() + + httpmock.RegisterResponder("DELETE", "http://edgex-core-metadata:59881/api/v2/deviceservice/name/test-device-virtual", + httpmock.NewStringResponder(200, ServiceDeleteSuccess)) + + err := serviceClient.Delete(context.TODO(), "test-device-virtual", clients.DeleteOptions{}) + assert.Nil(t, err) + + httpmock.RegisterResponder("DELETE", "http://edgex-core-metadata:59881/api/v2/deviceservice/name/test-device-virtual", + httpmock.NewStringResponder(404, ServiceDeleteFail)) + + err = serviceClient.Delete(context.TODO(), "test-device-virtual", clients.DeleteOptions{}) + assert.NotNil(t, err) +} + +func Test_UpdateService(t *testing.T) { + httpmock.ActivateNonDefault(serviceClient.Client.GetClient()) + defer httpmock.DeactivateAndReset() + httpmock.RegisterResponder("PATCH", "http://edgex-core-metadata:59881/api/v2/deviceservice", + httpmock.NewStringResponder(200, ServiceUpdateSuccess)) + var resp edgex_resp.DeviceServiceResponse + + err := json.Unmarshal([]byte(DeviceServiceMetaData), &resp) + assert.Nil(t, err) + + service := toKubeDeviceService(resp.Service, "default") + _, err = serviceClient.Update(context.TODO(), &service, clients.UpdateOptions{}) + assert.Nil(t, err) + + httpmock.RegisterResponder("PATCH", "http://edgex-core-metadata:59881/api/v2/deviceservice", + httpmock.NewStringResponder(404, ServiceUpdateFail)) + + _, err = serviceClient.Update(context.TODO(), &service, clients.UpdateOptions{}) + assert.NotNil(t, err) +} diff --git a/pkg/yurtiotdock/clients/edgex-foundry/edgexobject.go b/pkg/yurtiotdock/clients/edgex-foundry/edgexobject.go new file mode 100644 index 00000000000..4a093bf9834 --- /dev/null +++ b/pkg/yurtiotdock/clients/edgex-foundry/edgexobject.go @@ -0,0 +1,21 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package edgex_foundry + +type EdgeXObject interface { + IsAddedToEdgeX() bool +} diff --git a/pkg/yurtiotdock/clients/edgex-foundry/util.go b/pkg/yurtiotdock/clients/edgex-foundry/util.go new file mode 100644 index 00000000000..8fbf276ab30 --- /dev/null +++ b/pkg/yurtiotdock/clients/edgex-foundry/util.go @@ -0,0 +1,456 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package edgex_foundry + +import ( + "fmt" + "strings" + + "github.com/edgexfoundry/go-mod-core-contracts/v2/dtos" + "github.com/edgexfoundry/go-mod-core-contracts/v2/dtos/common" + "github.com/edgexfoundry/go-mod-core-contracts/v2/dtos/requests" + "github.com/edgexfoundry/go-mod-core-contracts/v2/models" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + iotv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" +) + +const ( + EdgeXObjectName = "yurt-iot-dock/edgex-object.name" + DeviceServicePath = "/api/v2/deviceservice" + DeviceProfilePath = "/api/v2/deviceprofile" + DevicePath = "/api/v2/device" + CommandResponsePath = "/api/v2/device" + + APIVersionV2 = "v2" +) + +type ClientURL struct { + Host string + Port int +} + +func getEdgeXName(provider metav1.Object) string { + var actualDeviceName string + if _, ok := provider.GetLabels()[EdgeXObjectName]; ok { + actualDeviceName = provider.GetLabels()[EdgeXObjectName] + } else { + actualDeviceName = provider.GetName() + } + return actualDeviceName +} + +func toEdgexDeviceService(ds *iotv1alpha1.DeviceService) dtos.DeviceService { + return dtos.DeviceService{ + Description: ds.Spec.Description, + Name: getEdgeXName(ds), + LastConnected: ds.Status.LastConnected, + LastReported: ds.Status.LastReported, + Labels: ds.Spec.Labels, + AdminState: string(ds.Spec.AdminState), + BaseAddress: ds.Spec.BaseAddress, + } +} + +func toEdgeXDeviceResourceSlice(drs []iotv1alpha1.DeviceResource) []dtos.DeviceResource { + var ret []dtos.DeviceResource + for _, dr := range drs { + ret = append(ret, toEdgeXDeviceResource(dr)) + } + return ret +} + +func toEdgeXDeviceResource(dr iotv1alpha1.DeviceResource) dtos.DeviceResource { + genericAttrs := make(map[string]interface{}) + for k, v := range dr.Attributes { + genericAttrs[k] = v + } + + return dtos.DeviceResource{ + Description: dr.Description, + Name: dr.Name, + Tag: dr.Tag, + Properties: toEdgeXProfileProperty(dr.Properties), + Attributes: genericAttrs, + } +} + +func toEdgeXProfileProperty(pp iotv1alpha1.ResourceProperties) dtos.ResourceProperties { + return dtos.ResourceProperties{ + ReadWrite: pp.ReadWrite, + Minimum: pp.Minimum, + Maximum: pp.Maximum, + DefaultValue: pp.DefaultValue, + Mask: pp.Mask, + Shift: pp.Shift, + Scale: pp.Scale, + Offset: pp.Offset, + Base: pp.Base, + Assertion: pp.Assertion, + MediaType: pp.MediaType, + Units: pp.Units, + ValueType: pp.ValueType, + } +} + +func toKubeDeviceService(ds dtos.DeviceService, namespace string) iotv1alpha1.DeviceService { + return iotv1alpha1.DeviceService{ + ObjectMeta: metav1.ObjectMeta{ + Name: toKubeName(ds.Name), + Namespace: namespace, + Labels: map[string]string{ + EdgeXObjectName: ds.Name, + }, + }, + Spec: iotv1alpha1.DeviceServiceSpec{ + Description: ds.Description, + Labels: ds.Labels, + AdminState: iotv1alpha1.AdminState(ds.AdminState), + BaseAddress: ds.BaseAddress, + }, + Status: iotv1alpha1.DeviceServiceStatus{ + EdgeId: ds.Id, + LastConnected: ds.LastConnected, + LastReported: ds.LastReported, + AdminState: iotv1alpha1.AdminState(ds.AdminState), + }, + } +} + +func toEdgeXDevice(d *iotv1alpha1.Device) dtos.Device { + md := dtos.Device{ + Description: d.Spec.Description, + Name: getEdgeXName(d), + AdminState: string(toEdgeXAdminState(d.Spec.AdminState)), + OperatingState: string(toEdgeXOperatingState(d.Spec.OperatingState)), + Protocols: toEdgeXProtocols(d.Spec.Protocols), + LastConnected: d.Status.LastConnected, + LastReported: d.Status.LastReported, + Labels: d.Spec.Labels, + Location: d.Spec.Location, + ServiceName: d.Spec.Service, + ProfileName: d.Spec.Profile, + } + if d.Status.EdgeId != "" { + md.Id = d.Status.EdgeId + } + return md +} + +func toEdgeXUpdateDevice(d *iotv1alpha1.Device) dtos.UpdateDevice { + adminState := string(toEdgeXAdminState(d.Spec.AdminState)) + operationState := string(toEdgeXOperatingState(d.Spec.OperatingState)) + md := dtos.UpdateDevice{ + Description: &d.Spec.Description, + Name: &d.Name, + AdminState: &adminState, + OperatingState: &operationState, + Protocols: toEdgeXProtocols(d.Spec.Protocols), + LastConnected: &d.Status.LastConnected, + LastReported: &d.Status.LastReported, + Labels: d.Spec.Labels, + Location: d.Spec.Location, + ServiceName: &d.Spec.Service, + ProfileName: &d.Spec.Profile, + Notify: &d.Spec.Notify, + } + if d.Status.EdgeId != "" { + md.Id = &d.Status.EdgeId + } + return md +} + +func toEdgeXProtocols( + pps map[string]iotv1alpha1.ProtocolProperties) map[string]dtos.ProtocolProperties { + ret := map[string]dtos.ProtocolProperties{} + for k, v := range pps { + ret[k] = dtos.ProtocolProperties(v) + } + return ret +} + +func toEdgeXAdminState(as iotv1alpha1.AdminState) models.AdminState { + if as == iotv1alpha1.Locked { + return models.Locked + } + return models.Unlocked +} + +func toEdgeXOperatingState(os iotv1alpha1.OperatingState) models.OperatingState { + if os == iotv1alpha1.Up { + return models.Up + } else if os == iotv1alpha1.Down { + return models.Down + } + return models.Unknown +} + +// toKubeDevice serialize the EdgeX Device to the corresponding Kubernetes Device +func toKubeDevice(ed dtos.Device, namespace string) iotv1alpha1.Device { + var loc string + if ed.Location != nil { + loc = ed.Location.(string) + } + return iotv1alpha1.Device{ + ObjectMeta: metav1.ObjectMeta{ + Name: toKubeName(ed.Name), + Namespace: namespace, + Labels: map[string]string{ + EdgeXObjectName: ed.Name, + }, + }, + Spec: iotv1alpha1.DeviceSpec{ + Description: ed.Description, + AdminState: iotv1alpha1.AdminState(ed.AdminState), + OperatingState: iotv1alpha1.OperatingState(ed.OperatingState), + Protocols: toKubeProtocols(ed.Protocols), + Labels: ed.Labels, + Location: loc, + Service: ed.ServiceName, + Profile: ed.ProfileName, + // TODO: Notify + }, + Status: iotv1alpha1.DeviceStatus{ + LastConnected: ed.LastConnected, + LastReported: ed.LastReported, + Synced: true, + EdgeId: ed.Id, + AdminState: iotv1alpha1.AdminState(ed.AdminState), + OperatingState: iotv1alpha1.OperatingState(ed.OperatingState), + }, + } +} + +// toKubeProtocols serialize the EdgeX ProtocolProperties to the corresponding +// Kubernetes OperatingState +func toKubeProtocols( + eps map[string]dtos.ProtocolProperties) map[string]iotv1alpha1.ProtocolProperties { + ret := map[string]iotv1alpha1.ProtocolProperties{} + for k, v := range eps { + ret[k] = iotv1alpha1.ProtocolProperties(v) + } + return ret +} + +// toKubeDeviceProfile create DeviceProfile in cloud according to devicProfile in edge +func toKubeDeviceProfile(dp *dtos.DeviceProfile, namespace string) iotv1alpha1.DeviceProfile { + return iotv1alpha1.DeviceProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: toKubeName(dp.Name), + Namespace: namespace, + Labels: map[string]string{ + EdgeXObjectName: dp.Name, + }, + }, + Spec: iotv1alpha1.DeviceProfileSpec{ + Description: dp.Description, + Manufacturer: dp.Manufacturer, + Model: dp.Model, + Labels: dp.Labels, + DeviceResources: toKubeDeviceResources(dp.DeviceResources), + DeviceCommands: toKubeDeviceCommand(dp.DeviceCommands), + }, + Status: iotv1alpha1.DeviceProfileStatus{ + EdgeId: dp.Id, + Synced: true, + }, + } +} + +func toKubeDeviceCommand(dcs []dtos.DeviceCommand) []iotv1alpha1.DeviceCommand { + var ret []iotv1alpha1.DeviceCommand + for _, dc := range dcs { + ret = append(ret, iotv1alpha1.DeviceCommand{ + Name: dc.Name, + ReadWrite: dc.ReadWrite, + IsHidden: dc.IsHidden, + ResourceOperations: toKubeResourceOperations(dc.ResourceOperations), + }) + } + return ret +} + +func toEdgeXDeviceCommand(dcs []iotv1alpha1.DeviceCommand) []dtos.DeviceCommand { + var ret []dtos.DeviceCommand + for _, dc := range dcs { + ret = append(ret, dtos.DeviceCommand{ + Name: dc.Name, + ReadWrite: dc.ReadWrite, + IsHidden: dc.IsHidden, + ResourceOperations: toEdgeXResourceOperations(dc.ResourceOperations), + }) + } + return ret +} + +func toKubeResourceOperations(ros []dtos.ResourceOperation) []iotv1alpha1.ResourceOperation { + var ret []iotv1alpha1.ResourceOperation + for _, ro := range ros { + ret = append(ret, iotv1alpha1.ResourceOperation{ + DeviceResource: ro.DeviceResource, + Mappings: ro.Mappings, + DefaultValue: ro.DefaultValue, + }) + } + return ret +} + +func toEdgeXResourceOperations(ros []iotv1alpha1.ResourceOperation) []dtos.ResourceOperation { + var ret []dtos.ResourceOperation + for _, ro := range ros { + ret = append(ret, dtos.ResourceOperation{ + DeviceResource: ro.DeviceResource, + Mappings: ro.Mappings, + DefaultValue: ro.DefaultValue, + }) + } + return ret +} + +func toKubeDeviceResources(drs []dtos.DeviceResource) []iotv1alpha1.DeviceResource { + var ret []iotv1alpha1.DeviceResource + for _, dr := range drs { + ret = append(ret, toKubeDeviceResource(dr)) + } + return ret +} + +func toKubeDeviceResource(dr dtos.DeviceResource) iotv1alpha1.DeviceResource { + concreteAttrs := make(map[string]string) + for k, v := range dr.Attributes { + switch asserted := v.(type) { + case string: + concreteAttrs[k] = asserted + continue + case int: + concreteAttrs[k] = fmt.Sprintf("%d", asserted) + continue + case float64: + concreteAttrs[k] = fmt.Sprintf("%f", asserted) + continue + case fmt.Stringer: + concreteAttrs[k] = asserted.String() + continue + } + } + + return iotv1alpha1.DeviceResource{ + Description: dr.Description, + Name: dr.Name, + Tag: dr.Tag, + IsHidden: dr.IsHidden, + Properties: toKubeProfileProperty(dr.Properties), + Attributes: concreteAttrs, + } +} + +func toKubeProfileProperty(rp dtos.ResourceProperties) iotv1alpha1.ResourceProperties { + return iotv1alpha1.ResourceProperties{ + ValueType: rp.ValueType, + ReadWrite: rp.ReadWrite, + Minimum: rp.Minimum, + Maximum: rp.Maximum, + DefaultValue: rp.DefaultValue, + Mask: rp.Mask, + Shift: rp.Shift, + Scale: rp.Scale, + Offset: rp.Offset, + Base: rp.Base, + Assertion: rp.Assertion, + MediaType: rp.MediaType, + Units: rp.Units, + } +} + +// toEdgeXDeviceProfile create DeviceProfile in edge according to devicProfile in cloud +func toEdgeXDeviceProfile(dp *iotv1alpha1.DeviceProfile) dtos.DeviceProfile { + return dtos.DeviceProfile{ + DeviceProfileBasicInfo: dtos.DeviceProfileBasicInfo{ + Description: dp.Spec.Description, + Name: getEdgeXName(dp), + Manufacturer: dp.Spec.Manufacturer, + Model: dp.Spec.Model, + Labels: dp.Spec.Labels, + }, + DeviceResources: toEdgeXDeviceResourceSlice(dp.Spec.DeviceResources), + DeviceCommands: toEdgeXDeviceCommand(dp.Spec.DeviceCommands), + } +} + +func makeEdgeXDeviceProfilesRequest(dps []*iotv1alpha1.DeviceProfile) []*requests.DeviceProfileRequest { + var req []*requests.DeviceProfileRequest + for _, dp := range dps { + req = append(req, &requests.DeviceProfileRequest{ + BaseRequest: common.BaseRequest{ + Versionable: common.Versionable{ + ApiVersion: APIVersionV2, + }, + }, + Profile: toEdgeXDeviceProfile(dp), + }) + } + return req +} + +func makeEdgeXDeviceUpdateRequest(devs []*iotv1alpha1.Device) []*requests.UpdateDeviceRequest { + var req []*requests.UpdateDeviceRequest + for _, dev := range devs { + req = append(req, &requests.UpdateDeviceRequest{ + BaseRequest: common.BaseRequest{ + Versionable: common.Versionable{ + ApiVersion: APIVersionV2, + }, + }, + Device: toEdgeXUpdateDevice(dev), + }) + } + return req +} + +func makeEdgeXDeviceRequest(devs []*iotv1alpha1.Device) []*requests.AddDeviceRequest { + var req []*requests.AddDeviceRequest + for _, dev := range devs { + req = append(req, &requests.AddDeviceRequest{ + BaseRequest: common.BaseRequest{ + Versionable: common.Versionable{ + ApiVersion: APIVersionV2, + }, + }, + Device: toEdgeXDevice(dev), + }) + } + return req +} + +func makeEdgeXDeviceService(dss []*iotv1alpha1.DeviceService) []*requests.AddDeviceServiceRequest { + var req []*requests.AddDeviceServiceRequest + for _, ds := range dss { + req = append(req, &requests.AddDeviceServiceRequest{ + BaseRequest: common.BaseRequest{ + Versionable: common.Versionable{ + ApiVersion: APIVersionV2, + }, + }, + Service: toEdgexDeviceService(ds), + }) + } + return req +} + +func toKubeName(edgexName string) string { + return strings.ReplaceAll(strings.ToLower(edgexName), "_", "-") +} diff --git a/pkg/yurtiotdock/clients/errors.go b/pkg/yurtiotdock/clients/errors.go new file mode 100644 index 00000000000..b5d6d5b6a2b --- /dev/null +++ b/pkg/yurtiotdock/clients/errors.go @@ -0,0 +1,29 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package clients + +import "strings" + +type NotFoundError struct{} + +func (e *NotFoundError) Error() string { return "Item not found" } + +func IsNotFoundErr(err error) bool { + if err == nil { + return false + } + return strings.Contains(err.Error(), "not found") || strings.HasPrefix(err.Error(), "no item found") +} diff --git a/pkg/yurtiotdock/clients/interface.go b/pkg/yurtiotdock/clients/interface.go new file mode 100644 index 00000000000..93ffc3574bd --- /dev/null +++ b/pkg/yurtiotdock/clients/interface.go @@ -0,0 +1,93 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clients + +import ( + "context" + + iotv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" +) + +// CreateOptions defines additional options when creating an object +// Additional general field definitions can be added +type CreateOptions struct{} + +// DeleteOptions defines additional options when deleting an object +// Additional general field definitions can be added +type DeleteOptions struct{} + +// UpdateOptions defines additional options when updating an object +// Additional general field definitions can be added +type UpdateOptions struct{} + +// GetOptions defines additional options when getting an object +// Additional general field definitions can be added +type GetOptions struct { + // Namespace represents the namespace to list for, or empty for + // non-namespaced objects, or to list across all namespaces. + Namespace string +} + +// ListOptions defines additional options when listing an object +type ListOptions struct { + // A selector to restrict the list of returned objects by their labels. + // Defaults to everything. + // +optional + LabelSelector map[string]string + // A selector to restrict the list of returned objects by their fields. + // Defaults to everything. + // +optional + FieldSelector map[string]string + // Namespace represents the namespace to list for, or empty for + // non-namespaced objects, or to list across all namespaces. + Namespace string +} + +// DeviceInterface defines the interfaces which used to create, delete, update, get and list Device objects on edge-side platform +type DeviceInterface interface { + DevicePropertyInterface + Create(ctx context.Context, device *iotv1alpha1.Device, options CreateOptions) (*iotv1alpha1.Device, error) + Delete(ctx context.Context, name string, options DeleteOptions) error + Update(ctx context.Context, device *iotv1alpha1.Device, options UpdateOptions) (*iotv1alpha1.Device, error) + Get(ctx context.Context, name string, options GetOptions) (*iotv1alpha1.Device, error) + List(ctx context.Context, options ListOptions) ([]iotv1alpha1.Device, error) +} + +// DevicePropertyInterface defines the interfaces which used to get, list and set the actual status value of the device properties +type DevicePropertyInterface interface { + GetPropertyState(ctx context.Context, propertyName string, device *iotv1alpha1.Device, options GetOptions) (*iotv1alpha1.ActualPropertyState, error) + UpdatePropertyState(ctx context.Context, propertyName string, device *iotv1alpha1.Device, options UpdateOptions) error + ListPropertiesState(ctx context.Context, device *iotv1alpha1.Device, options ListOptions) (map[string]iotv1alpha1.DesiredPropertyState, map[string]iotv1alpha1.ActualPropertyState, error) +} + +// DeviceServiceInterface defines the interfaces which used to create, delete, update, get and list DeviceService objects on edge-side platform +type DeviceServiceInterface interface { + Create(ctx context.Context, deviceService *iotv1alpha1.DeviceService, options CreateOptions) (*iotv1alpha1.DeviceService, error) + Delete(ctx context.Context, name string, options DeleteOptions) error + Update(ctx context.Context, deviceService *iotv1alpha1.DeviceService, options UpdateOptions) (*iotv1alpha1.DeviceService, error) + Get(ctx context.Context, name string, options GetOptions) (*iotv1alpha1.DeviceService, error) + List(ctx context.Context, options ListOptions) ([]iotv1alpha1.DeviceService, error) +} + +// DeviceProfileInterface defines the interfaces which used to create, delete, update, get and list DeviceProfile objects on edge-side platform +type DeviceProfileInterface interface { + Create(ctx context.Context, deviceProfile *iotv1alpha1.DeviceProfile, options CreateOptions) (*iotv1alpha1.DeviceProfile, error) + Delete(ctx context.Context, name string, options DeleteOptions) error + Update(ctx context.Context, deviceProfile *iotv1alpha1.DeviceProfile, options UpdateOptions) (*iotv1alpha1.DeviceProfile, error) + Get(ctx context.Context, name string, options GetOptions) (*iotv1alpha1.DeviceProfile, error) + List(ctx context.Context, options ListOptions) ([]iotv1alpha1.DeviceProfile, error) +} diff --git a/pkg/yurtiotdock/controllers/device_controller.go b/pkg/yurtiotdock/controllers/device_controller.go new file mode 100644 index 00000000000..dd61d5a6a96 --- /dev/null +++ b/pkg/yurtiotdock/controllers/device_controller.go @@ -0,0 +1,295 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "encoding/json" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openyurtio/openyurt/cmd/yurt-iot-dock/app/options" + iotv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" + "github.com/openyurtio/openyurt/pkg/yurtiotdock/clients" + edgexCli "github.com/openyurtio/openyurt/pkg/yurtiotdock/clients/edgex-foundry" + util "github.com/openyurtio/openyurt/pkg/yurtiotdock/controllers/util" +) + +// DeviceReconciler reconciles a Device object +type DeviceReconciler struct { + client.Client + Scheme *runtime.Scheme + deviceCli clients.DeviceInterface + // which nodePool deviceController is deployed in + NodePool string + Namespace string +} + +//+kubebuilder:rbac:groups=iot.openyurt.io,resources=devices,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=iot.openyurt.io,resources=devices/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=iot.openyurt.io,resources=devices/finalizers,verbs=update + +func (r *DeviceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var d iotv1alpha1.Device + if err := r.Get(ctx, req.NamespacedName, &d); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // If objects doesn't belong to the Edge platform to which the controller is connected, the controller does not handle events for that object + if d.Spec.NodePool != r.NodePool { + return ctrl.Result{}, nil + } + klog.V(3).Infof("Reconciling the Device: %s", d.GetName()) + + deviceStatus := d.Status.DeepCopy() + // Update the conditions for device + defer func() { + if !d.Spec.Managed { + util.SetDeviceCondition(deviceStatus, util.NewDeviceCondition(iotv1alpha1.DeviceManagingCondition, corev1.ConditionFalse, iotv1alpha1.DeviceManagingReason, "")) + } + + err := r.Status().Update(ctx, &d) + if client.IgnoreNotFound(err) != nil { + if !apierrors.IsConflict(err) { + klog.V(4).ErrorS(err, "update device conditions failed", "DeviceName", d.GetName()) + } + } + }() + + // 1. Handle the device deletion event + if err := r.reconcileDeleteDevice(ctx, &d); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } else if !d.ObjectMeta.DeletionTimestamp.IsZero() { + return ctrl.Result{}, nil + } + + if !d.Status.Synced { + // 2. Synchronize OpenYurt device objects to edge platform + if err := r.reconcileCreateDevice(ctx, &d, deviceStatus); err != nil { + if apierrors.IsConflict(err) { + return ctrl.Result{Requeue: true}, nil + } else { + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil + } else if d.Spec.Managed { + // 3. If the device has been synchronized and is managed by the cloud, reconcile the device properties + if err := r.reconcileUpdateDevice(ctx, &d, deviceStatus); err != nil { + if apierrors.IsConflict(err) { + return ctrl.Result{RequeueAfter: time.Second * 2}, nil + } + return ctrl.Result{}, err + } + } + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *DeviceReconciler) SetupWithManager(mgr ctrl.Manager, opts *options.YurtIoTDockOptions) error { + r.deviceCli = edgexCli.NewEdgexDeviceClient(opts.CoreMetadataAddr, opts.CoreCommandAddr) + r.NodePool = opts.Nodepool + r.Namespace = opts.Namespace + + return ctrl.NewControllerManagedBy(mgr). + For(&iotv1alpha1.Device{}). + WithEventFilter(genFirstUpdateFilter("device")). + Complete(r) +} + +func (r *DeviceReconciler) reconcileDeleteDevice(ctx context.Context, d *iotv1alpha1.Device) error { + // gets the actual name of the device on the Edge platform from the Label of the device + edgeDeviceName := util.GetEdgeDeviceName(d, EdgeXObjectName) + if d.ObjectMeta.DeletionTimestamp.IsZero() { + if len(d.GetFinalizers()) == 0 { + patchData, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "finalizers": []string{iotv1alpha1.DeviceFinalizer}, + }, + }) + if err := r.Patch(ctx, d, client.RawPatch(types.MergePatchType, patchData)); err != nil { + return err + } + } + } else { + // delete the device object on the edge platform + err := r.deviceCli.Delete(context.TODO(), edgeDeviceName, clients.DeleteOptions{}) + if err != nil && !clients.IsNotFoundErr(err) { + return err + } + + // delete the device in OpenYurt + patchData, _ := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "finalizers": []string{}, + }, + }) + if err = r.Patch(ctx, d, client.RawPatch(types.MergePatchType, patchData)); err != nil { + return err + } + } + return nil +} + +func (r *DeviceReconciler) reconcileCreateDevice(ctx context.Context, d *iotv1alpha1.Device, deviceStatus *iotv1alpha1.DeviceStatus) error { + // get the actual name of the device on the Edge platform from the Label of the device + edgeDeviceName := util.GetEdgeDeviceName(d, EdgeXObjectName) + newDeviceStatus := d.Status.DeepCopy() + klog.V(4).Infof("Checking if device already exist on the edge platform: %s", d.GetName()) + // Checking if device already exist on the edge platform + edgeDevice, err := r.deviceCli.Get(context.TODO(), edgeDeviceName, clients.GetOptions{Namespace: r.Namespace}) + if err == nil { + // a. If object exists, the status of the device on OpenYurt is updated + klog.V(4).Infof("Device already exists on edge platform: %s", d.GetName()) + newDeviceStatus.EdgeId = edgeDevice.Status.EdgeId + newDeviceStatus.Synced = true + } else if clients.IsNotFoundErr(err) { + // b. If the object does not exist, a request is sent to the edge platform to create a new device + klog.V(4).Infof("Adding device to the edge platform: %s", d.GetName()) + createdEdgeObj, err := r.deviceCli.Create(context.TODO(), d, clients.CreateOptions{}) + if err != nil { + util.SetDeviceCondition(deviceStatus, util.NewDeviceCondition(iotv1alpha1.DeviceSyncedCondition, corev1.ConditionFalse, iotv1alpha1.DeviceCreateSyncedReason, err.Error())) + return fmt.Errorf("fail to add Device to edge platform: %v", err) + } else { + klog.V(4).Infof("Successfully add Device to edge platform, Name: %s, EdgeId: %s", edgeDeviceName, createdEdgeObj.Status.EdgeId) + newDeviceStatus.EdgeId = createdEdgeObj.Status.EdgeId + newDeviceStatus.Synced = true + } + } else { + klog.V(4).ErrorS(err, "failed to visit the edge platform") + util.SetDeviceCondition(deviceStatus, util.NewDeviceCondition(iotv1alpha1.DeviceSyncedCondition, corev1.ConditionFalse, iotv1alpha1.DeviceVistedCoreMetadataSyncedReason, "")) + return nil + } + d.Status = *newDeviceStatus + util.SetDeviceCondition(deviceStatus, util.NewDeviceCondition(iotv1alpha1.DeviceSyncedCondition, corev1.ConditionTrue, "", "")) + + return r.Status().Update(ctx, d) +} + +func (r *DeviceReconciler) reconcileUpdateDevice(ctx context.Context, d *iotv1alpha1.Device, deviceStatus *iotv1alpha1.DeviceStatus) error { + // the device has been added to the edge platform, check if each device property are in the desired state + newDeviceStatus := d.Status.DeepCopy() + // This list is used to hold the names of properties that failed to reconcile + var failedPropertyNames []string + + // 1. reconciling the AdminState and OperatingState field of device + klog.V(3).Infof("DeviceName: %s, reconciling the AdminState and OperatingState field of device", d.GetName()) + updateDevice := d.DeepCopy() + if d.Spec.AdminState != "" && d.Spec.AdminState != d.Status.AdminState { + newDeviceStatus.AdminState = d.Spec.AdminState + } else { + updateDevice.Spec.AdminState = "" + } + + if d.Spec.OperatingState != "" && d.Spec.OperatingState != d.Status.OperatingState { + newDeviceStatus.OperatingState = d.Spec.OperatingState + } else { + updateDevice.Spec.OperatingState = "" + } + _, err := r.deviceCli.Update(context.TODO(), updateDevice, clients.UpdateOptions{}) + if err != nil { + util.SetDeviceCondition(deviceStatus, util.NewDeviceCondition(iotv1alpha1.DeviceManagingCondition, corev1.ConditionFalse, iotv1alpha1.DeviceUpdateStateReason, err.Error())) + return err + } + + // 2. reconciling the device properties' value + klog.V(3).Infof("DeviceName: %s, reconciling the device properties", d.GetName()) + // property updates are made only when the device is up and unlocked + if newDeviceStatus.OperatingState == iotv1alpha1.Up && newDeviceStatus.AdminState == iotv1alpha1.UnLocked { + newDeviceStatus, failedPropertyNames = r.reconcileDeviceProperties(d, newDeviceStatus) + } + + d.Status = *newDeviceStatus + + // 3. update the device status on OpenYurt + klog.V(3).Infof("DeviceName: %s, update the device status", d.GetName()) + if err := r.Status().Update(ctx, d); err != nil { + util.SetDeviceCondition(deviceStatus, util.NewDeviceCondition(iotv1alpha1.DeviceManagingCondition, corev1.ConditionFalse, iotv1alpha1.DeviceUpdateStateReason, err.Error())) + return err + } else if len(failedPropertyNames) != 0 { + err = fmt.Errorf("the following device properties failed to reconcile: %v", failedPropertyNames) + util.SetDeviceCondition(deviceStatus, util.NewDeviceCondition(iotv1alpha1.DeviceManagingCondition, corev1.ConditionFalse, err.Error(), "")) + return nil + } + + util.SetDeviceCondition(deviceStatus, util.NewDeviceCondition(iotv1alpha1.DeviceManagingCondition, corev1.ConditionTrue, "", "")) + return nil +} + +// Update the actual property value of the device on edge platform, +// return the latest status and the names of the property that failed to update +func (r *DeviceReconciler) reconcileDeviceProperties(d *iotv1alpha1.Device, deviceStatus *iotv1alpha1.DeviceStatus) (*iotv1alpha1.DeviceStatus, []string) { + newDeviceStatus := deviceStatus.DeepCopy() + // This list is used to hold the names of properties that failed to reconcile + var failedPropertyNames []string + // 2. reconciling the device properties' value + klog.V(3).Infof("DeviceName: %s, reconciling the value of device properties", d.GetName()) + for _, desiredProperty := range d.Spec.DeviceProperties { + if desiredProperty.DesiredValue == "" { + continue + } + propertyName := desiredProperty.Name + // 1.1. gets the actual property value of the current device from edge platform + klog.V(4).Infof("DeviceName: %s, getting the actual value of property: %s", d.GetName(), propertyName) + actualProperty, err := r.deviceCli.GetPropertyState(context.TODO(), propertyName, d, clients.GetOptions{}) + if err != nil { + if !clients.IsNotFoundErr(err) { + klog.Errorf("DeviceName: %s, failed to get actual property value of %s, err:%v", d.GetName(), propertyName, err) + failedPropertyNames = append(failedPropertyNames, propertyName) + continue + } + klog.Errorf("DeviceName: %s, property read command not found", d.GetName()) + } else { + klog.V(4).Infof("DeviceName: %s, got the actual property state, {Name: %s, GetURL: %s, ActualValue: %s}", + d.GetName(), propertyName, actualProperty.GetURL, actualProperty.ActualValue) + } + + if newDeviceStatus.DeviceProperties == nil { + newDeviceStatus.DeviceProperties = map[string]iotv1alpha1.ActualPropertyState{} + } else { + newDeviceStatus.DeviceProperties[propertyName] = *actualProperty + } + + // 1.2. set the device attribute in the edge platform to the expected value + if actualProperty == nil || desiredProperty.DesiredValue != actualProperty.ActualValue { + klog.V(4).Infof("DeviceName: %s, the desired value and the actual value are different, desired: %s, actual: %s", + d.GetName(), desiredProperty.DesiredValue, actualProperty.ActualValue) + if err := r.deviceCli.UpdatePropertyState(context.TODO(), propertyName, d, clients.UpdateOptions{}); err != nil { + klog.ErrorS(err, "failed to update property", "DeviceName", d.GetName(), "propertyName", propertyName) + failedPropertyNames = append(failedPropertyNames, propertyName) + continue + } + + klog.V(4).Infof("DeviceName: %s, successfully set the property %s to desired value", d.GetName(), propertyName) + newActualProperty := iotv1alpha1.ActualPropertyState{ + Name: propertyName, + GetURL: actualProperty.GetURL, + ActualValue: desiredProperty.DesiredValue, + } + newDeviceStatus.DeviceProperties[propertyName] = newActualProperty + } + } + return newDeviceStatus, failedPropertyNames +} diff --git a/pkg/yurtiotdock/controllers/device_syncer.go b/pkg/yurtiotdock/controllers/device_syncer.go new file mode 100644 index 00000000000..501ba6101dc --- /dev/null +++ b/pkg/yurtiotdock/controllers/device_syncer.go @@ -0,0 +1,237 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "strings" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/openyurtio/openyurt/cmd/yurt-iot-dock/app/options" + iotv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" + edgeCli "github.com/openyurtio/openyurt/pkg/yurtiotdock/clients" + efCli "github.com/openyurtio/openyurt/pkg/yurtiotdock/clients/edgex-foundry" + "github.com/openyurtio/openyurt/pkg/yurtiotdock/controllers/util" +) + +type DeviceSyncer struct { + // kubernetes client + client.Client + // which nodePool deviceController is deployed in + NodePool string + // edge platform's client + deviceCli edgeCli.DeviceInterface + // syncing period in seconds + syncPeriod time.Duration + Namespace string +} + +// NewDeviceSyncer initialize a New DeviceSyncer +func NewDeviceSyncer(client client.Client, opts *options.YurtIoTDockOptions) (DeviceSyncer, error) { + return DeviceSyncer{ + syncPeriod: time.Duration(opts.EdgeSyncPeriod) * time.Second, + deviceCli: efCli.NewEdgexDeviceClient(opts.CoreMetadataAddr, opts.CoreCommandAddr), + Client: client, + NodePool: opts.Nodepool, + Namespace: opts.Namespace, + }, nil +} + +// NewDeviceSyncerRunnable initialize a controller-runtime manager runnable +func (ds *DeviceSyncer) NewDeviceSyncerRunnable() ctrlmgr.RunnableFunc { + return func(ctx context.Context) error { + ds.Run(ctx.Done()) + return nil + } +} + +func (ds *DeviceSyncer) Run(stop <-chan struct{}) { + klog.V(1).Info("[Device] Starting the syncer...") + go func() { + for { + <-time.After(ds.syncPeriod) + klog.V(2).Info("[Device] Start a round of synchronization.") + // 1. get device on edge platform and OpenYurt + edgeDevices, kubeDevices, err := ds.getAllDevices() + if err != nil { + klog.V(3).ErrorS(err, "fail to list the devices") + continue + } + + // 2. find the device that need to be synchronized + redundantEdgeDevices, redundantKubeDevices, syncedDevices := ds.findDiffDevice(edgeDevices, kubeDevices) + klog.V(2).Infof("[Device] The number of objects waiting for synchronization { %s:%d, %s:%d, %s:%d }", + "Edge device should be added to OpenYurt", len(redundantEdgeDevices), + "OpenYurt device that should be deleted", len(redundantKubeDevices), + "Devices that should be synchronized", len(syncedDevices)) + + // 3. create device on OpenYurt which are exists in edge platform but not in OpenYurt + if err := ds.syncEdgeToKube(redundantEdgeDevices); err != nil { + klog.V(3).ErrorS(err, "fail to create devices on OpenYurt") + } + + // 4. delete redundant device on OpenYurt + if err := ds.deleteDevices(redundantKubeDevices); err != nil { + klog.V(3).ErrorS(err, "fail to delete redundant devices on OpenYurt") + } + + // 5. update device status on OpenYurt + if err := ds.updateDevices(syncedDevices); err != nil { + klog.V(3).ErrorS(err, "fail to update devices status") + } + klog.V(2).Info("[Device] One round of synchronization is complete") + } + }() + + <-stop + klog.V(1).Info("[Device] Stopping the syncer") +} + +// Get the existing Device on the Edge platform, as well as OpenYurt existing Device +// edgeDevice：map[actualName]device +// kubeDevice：map[actualName]device +func (ds *DeviceSyncer) getAllDevices() (map[string]iotv1alpha1.Device, map[string]iotv1alpha1.Device, error) { + edgeDevice := map[string]iotv1alpha1.Device{} + kubeDevice := map[string]iotv1alpha1.Device{} + // 1. list devices on edge platform + eDevs, err := ds.deviceCli.List(context.TODO(), edgeCli.ListOptions{Namespace: ds.Namespace}) + if err != nil { + klog.V(4).ErrorS(err, "fail to list the devices object on the Edge Platform") + return edgeDevice, kubeDevice, err + } + // 2. list devices on OpenYurt (filter objects belonging to edgeServer) + var kDevs iotv1alpha1.DeviceList + listOptions := client.MatchingFields{util.IndexerPathForNodepool: ds.NodePool} + if err = ds.List(context.TODO(), &kDevs, listOptions, client.InNamespace(ds.Namespace)); err != nil { + klog.V(4).ErrorS(err, "fail to list the devices object on the OpenYurt") + return edgeDevice, kubeDevice, err + } + for i := range eDevs { + deviceName := util.GetEdgeDeviceName(&eDevs[i], EdgeXObjectName) + edgeDevice[deviceName] = eDevs[i] + } + + for i := range kDevs.Items { + deviceName := util.GetEdgeDeviceName(&kDevs.Items[i], EdgeXObjectName) + kubeDevice[deviceName] = kDevs.Items[i] + } + return edgeDevice, kubeDevice, nil +} + +// Get the list of devices that need to be added, deleted and updated +func (ds *DeviceSyncer) findDiffDevice( + edgeDevices map[string]iotv1alpha1.Device, kubeDevices map[string]iotv1alpha1.Device) ( + redundantEdgeDevices map[string]*iotv1alpha1.Device, redundantKubeDevices map[string]*iotv1alpha1.Device, syncedDevices map[string]*iotv1alpha1.Device) { + + redundantEdgeDevices = map[string]*iotv1alpha1.Device{} + redundantKubeDevices = map[string]*iotv1alpha1.Device{} + syncedDevices = map[string]*iotv1alpha1.Device{} + + for i := range edgeDevices { + ed := edgeDevices[i] + edName := util.GetEdgeDeviceName(&ed, EdgeXObjectName) + if _, exists := kubeDevices[edName]; !exists { + klog.V(5).Infof("found redundant edge device %s", edName) + redundantEdgeDevices[edName] = ds.completeCreateContent(&ed) + } else { + klog.V(5).Infof("found device %s to be synced", edName) + kd := kubeDevices[edName] + syncedDevices[edName] = ds.completeUpdateContent(&kd, &ed) + } + } + + for i := range kubeDevices { + kd := kubeDevices[i] + if !kd.Status.Synced { + continue + } + kdName := util.GetEdgeDeviceName(&kd, EdgeXObjectName) + if _, exists := edgeDevices[kdName]; !exists { + redundantKubeDevices[kdName] = &kd + } + } + return +} + +// syncEdgeToKube creates device on OpenYurt which are exists in edge platform but not in OpenYurt +func (ds *DeviceSyncer) syncEdgeToKube(edgeDevs map[string]*iotv1alpha1.Device) error { + for _, ed := range edgeDevs { + if err := ds.Client.Create(context.TODO(), ed); err != nil { + if apierrors.IsAlreadyExists(err) { + continue + } + klog.V(5).ErrorS(err, "fail to create device on OpenYurt", "DeviceName", strings.ToLower(ed.Name)) + return err + } + } + return nil +} + +// deleteDevices deletes redundant device on OpenYurt +func (ds *DeviceSyncer) deleteDevices(redundantKubeDevices map[string]*iotv1alpha1.Device) error { + for _, kd := range redundantKubeDevices { + if err := ds.Client.Delete(context.TODO(), kd); err != nil { + klog.V(5).ErrorS(err, "fail to delete the device on OpenYurt", + "DeviceName", kd.Name) + return err + } + } + return nil +} + +// updateDevicesStatus updates device status on OpenYurt +func (ds *DeviceSyncer) updateDevices(syncedDevices map[string]*iotv1alpha1.Device) error { + for n := range syncedDevices { + if err := ds.Client.Status().Update(context.TODO(), syncedDevices[n]); err != nil { + if apierrors.IsConflict(err) { + klog.V(5).InfoS("update Conflicts", "Device", syncedDevices[n].Name) + continue + } + return err + } + } + return nil +} + +// completeCreateContent completes the content of the device which will be created on OpenYurt +func (ds *DeviceSyncer) completeCreateContent(edgeDevice *iotv1alpha1.Device) *iotv1alpha1.Device { + createDevice := edgeDevice.DeepCopy() + createDevice.Spec.NodePool = ds.NodePool + createDevice.Name = strings.Join([]string{ds.NodePool, createDevice.Name}, "-") + createDevice.Namespace = ds.Namespace + createDevice.Spec.Managed = false + + return createDevice +} + +// completeUpdateContent completes the content of the device which will be updated on OpenYurt +func (ds *DeviceSyncer) completeUpdateContent(kubeDevice *iotv1alpha1.Device, edgeDevice *iotv1alpha1.Device) *iotv1alpha1.Device { + updatedDevice := kubeDevice.DeepCopy() + _, aps, _ := ds.deviceCli.ListPropertiesState(context.TODO(), updatedDevice, edgeCli.ListOptions{}) + // update device status + updatedDevice.Status.LastConnected = edgeDevice.Status.LastConnected + updatedDevice.Status.LastReported = edgeDevice.Status.LastReported + updatedDevice.Status.AdminState = edgeDevice.Status.AdminState + updatedDevice.Status.OperatingState = edgeDevice.Status.OperatingState + updatedDevice.Status.DeviceProperties = aps + return updatedDevice +} diff --git a/pkg/yurtiotdock/controllers/deviceprofile_controller.go b/pkg/yurtiotdock/controllers/deviceprofile_controller.go new file mode 100644 index 00000000000..d174cc2ebb6 --- /dev/null +++ b/pkg/yurtiotdock/controllers/deviceprofile_controller.go @@ -0,0 +1,165 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "encoding/json" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openyurtio/openyurt/cmd/yurt-iot-dock/app/options" + iotv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" + "github.com/openyurtio/openyurt/pkg/yurtiotdock/clients" + edgexclis "github.com/openyurtio/openyurt/pkg/yurtiotdock/clients/edgex-foundry" + "github.com/openyurtio/openyurt/pkg/yurtiotdock/controllers/util" +) + +// DeviceProfileReconciler reconciles a DeviceProfile object +type DeviceProfileReconciler struct { + client.Client + Scheme *runtime.Scheme + edgeClient clients.DeviceProfileInterface + NodePool string + Namespace string +} + +//+kubebuilder:rbac:groups=iot.openyurt.io,resources=deviceprofiles,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=iot.openyurt.io,resources=deviceprofiles/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=iot.openyurt.io,resources=deviceprofiles/finalizers,verbs=update + +// Reconcile make changes to a deviceprofile object in EdgeX based on it in Kubernetes +func (r *DeviceProfileReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var dp iotv1alpha1.DeviceProfile + if err := r.Get(ctx, req.NamespacedName, &dp); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + if dp.Spec.NodePool != r.NodePool { + return ctrl.Result{}, nil + } + klog.V(3).Infof("Reconciling the DeviceProfile: %s", dp.GetName()) + + // gets the actual name of deviceProfile on the edge platform from the Label of the deviceProfile + dpActualName := util.GetEdgeDeviceProfileName(&dp, EdgeXObjectName) + + // 1. Handle the deviceProfile deletion event + if err := r.reconcileDeleteDeviceProfile(ctx, &dp, dpActualName); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } else if !dp.ObjectMeta.DeletionTimestamp.IsZero() { + return ctrl.Result{}, nil + } + + if !dp.Status.Synced { + // 2. Synchronize OpenYurt deviceProfile to edge platform + if err := r.reconcileCreateDeviceProfile(ctx, &dp, dpActualName); err != nil { + if apierrors.IsConflict(err) { + return ctrl.Result{Requeue: true}, nil + } else { + return ctrl.Result{}, err + } + } + } + // 3. Handle the deviceProfile update event + // TODO + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *DeviceProfileReconciler) SetupWithManager(mgr ctrl.Manager, opts *options.YurtIoTDockOptions) error { + r.edgeClient = edgexclis.NewEdgexDeviceProfile(opts.CoreMetadataAddr) + r.NodePool = opts.Nodepool + r.Namespace = opts.Namespace + + return ctrl.NewControllerManagedBy(mgr). + For(&iotv1alpha1.DeviceProfile{}). + WithEventFilter(genFirstUpdateFilter("deviceprofile")). + Complete(r) +} + +func (r *DeviceProfileReconciler) reconcileDeleteDeviceProfile(ctx context.Context, dp *iotv1alpha1.DeviceProfile, actualName string) error { + if dp.ObjectMeta.DeletionTimestamp.IsZero() { + if len(dp.GetFinalizers()) == 0 { + patchString := map[string]interface{}{ + "metadata": map[string]interface{}{ + "finalizers": []string{iotv1alpha1.DeviceProfileFinalizer}, + }, + } + if patchData, err := json.Marshal(patchString); err != nil { + return err + } else { + if err = r.Patch(ctx, dp, client.RawPatch(types.MergePatchType, patchData)); err != nil { + return err + } + } + } + } else { + patchString := map[string]interface{}{ + "metadata": map[string]interface{}{ + "finalizers": []string{}, + }, + } + // delete the deviceProfile in OpenYurt + if patchData, err := json.Marshal(patchString); err != nil { + return err + } else { + if err = r.Patch(ctx, dp, client.RawPatch(types.MergePatchType, patchData)); err != nil { + return err + } + } + + // delete the deviceProfile object on edge platform + err := r.edgeClient.Delete(context.TODO(), actualName, clients.DeleteOptions{}) + if err != nil && !clients.IsNotFoundErr(err) { + return err + } + } + return nil +} + +func (r *DeviceProfileReconciler) reconcileCreateDeviceProfile(ctx context.Context, dp *iotv1alpha1.DeviceProfile, actualName string) error { + klog.V(4).Infof("Checking if deviceProfile already exist on the edge platform: %s", dp.GetName()) + if edgeDp, err := r.edgeClient.Get(context.TODO(), actualName, clients.GetOptions{Namespace: r.Namespace}); err != nil { + if !clients.IsNotFoundErr(err) { + klog.V(4).ErrorS(err, "fail to visit the edge platform") + return nil + } + } else { + // a. If object exists, the status of the deviceProfile on OpenYurt is updated + klog.V(4).Info("DeviceProfile already exists on edge platform") + dp.Status.Synced = true + dp.Status.EdgeId = edgeDp.Status.EdgeId + return r.Status().Update(ctx, dp) + } + + // b. If object does not exist, a request is sent to the edge platform to create a new deviceProfile + createDp, err := r.edgeClient.Create(context.Background(), dp, clients.CreateOptions{}) + if err != nil { + klog.V(4).ErrorS(err, "failed to create deviceProfile on edge platform") + return fmt.Errorf("failed to add deviceProfile to edge platform: %v", err) + } + klog.V(3).Infof("Successfully add DeviceProfile to edge platform, Name: %s, EdgeId: %s", createDp.GetName(), createDp.Status.EdgeId) + dp.Status.EdgeId = createDp.Status.EdgeId + dp.Status.Synced = true + return r.Status().Update(ctx, dp) +} diff --git a/pkg/yurtiotdock/controllers/deviceprofile_syncer.go b/pkg/yurtiotdock/controllers/deviceprofile_syncer.go new file mode 100644 index 00000000000..72d67729b47 --- /dev/null +++ b/pkg/yurtiotdock/controllers/deviceprofile_syncer.go @@ -0,0 +1,214 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "strings" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/openyurtio/openyurt/cmd/yurt-iot-dock/app/options" + iotv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" + devcli "github.com/openyurtio/openyurt/pkg/yurtiotdock/clients" + edgexclis "github.com/openyurtio/openyurt/pkg/yurtiotdock/clients/edgex-foundry" + "github.com/openyurtio/openyurt/pkg/yurtiotdock/controllers/util" +) + +type DeviceProfileSyncer struct { + // syncing period in seconds + syncPeriod time.Duration + // edge platform client + edgeClient devcli.DeviceProfileInterface + // Kubernetes client + client.Client + NodePool string + Namespace string +} + +// NewDeviceProfileSyncer initialize a New DeviceProfileSyncer +func NewDeviceProfileSyncer(client client.Client, opts *options.YurtIoTDockOptions) (DeviceProfileSyncer, error) { + return DeviceProfileSyncer{ + syncPeriod: time.Duration(opts.EdgeSyncPeriod) * time.Second, + edgeClient: edgexclis.NewEdgexDeviceProfile(opts.CoreMetadataAddr), + Client: client, + NodePool: opts.Nodepool, + Namespace: opts.Namespace, + }, nil +} + +// NewDeviceProfileSyncerRunnable initialize a controller-runtime manager runnable +func (dps *DeviceProfileSyncer) NewDeviceProfileSyncerRunnable() ctrlmgr.RunnableFunc { + return func(ctx context.Context) error { + dps.Run(ctx.Done()) + return nil + } +} + +func (dps *DeviceProfileSyncer) Run(stop <-chan struct{}) { + klog.V(1).Info("[DeviceProfile] Starting the syncer...") + go func() { + for { + <-time.After(dps.syncPeriod) + klog.V(2).Info("[DeviceProfile] Start a round of synchronization.") + + // 1. get deviceProfiles on edge platform and OpenYurt + edgeDeviceProfiles, kubeDeviceProfiles, err := dps.getAllDeviceProfiles() + if err != nil { + klog.V(3).ErrorS(err, "fail to list the deviceProfiles") + continue + } + + // 2. find the deviceProfiles that need to be synchronized + redundantEdgeDeviceProfiles, redundantKubeDeviceProfiles, syncedDeviceProfiles := + dps.findDiffDeviceProfiles(edgeDeviceProfiles, kubeDeviceProfiles) + klog.V(2).Infof("[DeviceProfile] The number of objects waiting for synchronization { %s:%d, %s:%d, %s:%d }", + "Edge deviceProfiles should be added to OpenYurt", len(redundantEdgeDeviceProfiles), + "OpenYurt deviceProfiles that should be deleted", len(redundantKubeDeviceProfiles), + "DeviceProfiles that should be synchronized", len(syncedDeviceProfiles)) + + // 3. create deviceProfiles on OpenYurt which are exists in edge platform but not in OpenYurt + if err := dps.syncEdgeToKube(redundantEdgeDeviceProfiles); err != nil { + klog.V(3).ErrorS(err, "fail to create deviceProfiles on OpenYurt") + } + + // 4. delete redundant deviceProfiles on OpenYurt + if err := dps.deleteDeviceProfiles(redundantKubeDeviceProfiles); err != nil { + klog.V(3).ErrorS(err, "fail to delete redundant deviceProfiles on OpenYurt") + } + + // 5. update deviceProfiles on OpenYurt + // TODO + } + }() + + <-stop + klog.V(1).Info("[DeviceProfile] Stopping the syncer") +} + +// Get the existing DeviceProfile on the Edge platform, as well as OpenYurt existing DeviceProfile +// edgeDeviceProfiles：map[actualName]DeviceProfile +// kubeDeviceProfiles：map[actualName]DeviceProfile +func (dps *DeviceProfileSyncer) getAllDeviceProfiles() ( + map[string]iotv1alpha1.DeviceProfile, map[string]iotv1alpha1.DeviceProfile, error) { + + edgeDeviceProfiles := map[string]iotv1alpha1.DeviceProfile{} + kubeDeviceProfiles := map[string]iotv1alpha1.DeviceProfile{} + + // 1. list deviceProfiles on edge platform + eDps, err := dps.edgeClient.List(context.TODO(), devcli.ListOptions{Namespace: dps.Namespace}) + if err != nil { + klog.V(4).ErrorS(err, "fail to list the deviceProfiles on the edge platform") + return edgeDeviceProfiles, kubeDeviceProfiles, err + } + // 2. list deviceProfiles on OpenYurt (filter objects belonging to edgeServer) + var kDps iotv1alpha1.DeviceProfileList + listOptions := client.MatchingFields{util.IndexerPathForNodepool: dps.NodePool} + if err = dps.List(context.TODO(), &kDps, listOptions, client.InNamespace(dps.Namespace)); err != nil { + klog.V(4).ErrorS(err, "fail to list the deviceProfiles on the Kubernetes") + return edgeDeviceProfiles, kubeDeviceProfiles, err + } + for i := range eDps { + deviceProfilesName := util.GetEdgeDeviceProfileName(&eDps[i], EdgeXObjectName) + edgeDeviceProfiles[deviceProfilesName] = eDps[i] + } + + for i := range kDps.Items { + deviceProfilesName := util.GetEdgeDeviceProfileName(&kDps.Items[i], EdgeXObjectName) + kubeDeviceProfiles[deviceProfilesName] = kDps.Items[i] + } + return edgeDeviceProfiles, kubeDeviceProfiles, nil +} + +// Get the list of deviceProfiles that need to be added, deleted and updated +func (dps *DeviceProfileSyncer) findDiffDeviceProfiles( + edgeDeviceProfiles map[string]iotv1alpha1.DeviceProfile, kubeDeviceProfiles map[string]iotv1alpha1.DeviceProfile) ( + redundantEdgeDeviceProfiles map[string]*iotv1alpha1.DeviceProfile, redundantKubeDeviceProfiles map[string]*iotv1alpha1.DeviceProfile, syncedDeviceProfiles map[string]*iotv1alpha1.DeviceProfile) { + + redundantEdgeDeviceProfiles = map[string]*iotv1alpha1.DeviceProfile{} + redundantKubeDeviceProfiles = map[string]*iotv1alpha1.DeviceProfile{} + syncedDeviceProfiles = map[string]*iotv1alpha1.DeviceProfile{} + + for i := range edgeDeviceProfiles { + edp := edgeDeviceProfiles[i] + edpName := util.GetEdgeDeviceProfileName(&edp, EdgeXObjectName) + if _, exists := kubeDeviceProfiles[edpName]; !exists { + redundantEdgeDeviceProfiles[edpName] = dps.completeCreateContent(&edp) + } else { + kdp := kubeDeviceProfiles[edpName] + syncedDeviceProfiles[edpName] = dps.completeUpdateContent(&kdp, &edp) + } + } + + for i := range kubeDeviceProfiles { + kdp := kubeDeviceProfiles[i] + if !kdp.Status.Synced { + continue + } + kdpName := util.GetEdgeDeviceProfileName(&kdp, EdgeXObjectName) + if _, exists := edgeDeviceProfiles[kdpName]; !exists { + redundantKubeDeviceProfiles[kdpName] = &kdp + } + } + return +} + +// completeCreateContent completes the content of the deviceProfile which will be created on OpenYurt +func (dps *DeviceProfileSyncer) completeCreateContent(edgeDps *iotv1alpha1.DeviceProfile) *iotv1alpha1.DeviceProfile { + createDeviceProfile := edgeDps.DeepCopy() + createDeviceProfile.Namespace = dps.Namespace + createDeviceProfile.Name = strings.Join([]string{dps.NodePool, createDeviceProfile.Name}, "-") + createDeviceProfile.Spec.NodePool = dps.NodePool + return createDeviceProfile +} + +// completeUpdateContent completes the content of the deviceProfile which will be updated on OpenYurt +// TODO +func (dps *DeviceProfileSyncer) completeUpdateContent(kubeDps *iotv1alpha1.DeviceProfile, edgeDS *iotv1alpha1.DeviceProfile) *iotv1alpha1.DeviceProfile { + return kubeDps +} + +// syncEdgeToKube creates deviceProfiles on OpenYurt which are exists in edge platform but not in OpenYurt +func (dps *DeviceProfileSyncer) syncEdgeToKube(edgeDps map[string]*iotv1alpha1.DeviceProfile) error { + for _, edp := range edgeDps { + if err := dps.Client.Create(context.TODO(), edp); err != nil { + if apierrors.IsAlreadyExists(err) { + klog.V(5).Infof("DeviceProfile already exist on Kubernetes: %s", strings.ToLower(edp.Name)) + continue + } + klog.Infof("created deviceProfile failed: %s", strings.ToLower(edp.Name)) + return err + } + } + return nil +} + +// deleteDeviceProfiles deletes redundant deviceProfiles on OpenYurt +func (dps *DeviceProfileSyncer) deleteDeviceProfiles(redundantKubeDeviceProfiles map[string]*iotv1alpha1.DeviceProfile) error { + for _, kdp := range redundantKubeDeviceProfiles { + if err := dps.Client.Delete(context.TODO(), kdp); err != nil { + klog.V(5).ErrorS(err, "fail to delete the DeviceProfile on Kubernetes: %s ", + "DeviceProfile", kdp.Name) + return err + } + } + return nil +} diff --git a/pkg/yurtiotdock/controllers/deviceservice_controller.go b/pkg/yurtiotdock/controllers/deviceservice_controller.go new file mode 100644 index 00000000000..2822558b5f0 --- /dev/null +++ b/pkg/yurtiotdock/controllers/deviceservice_controller.go @@ -0,0 +1,220 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "encoding/json" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openyurtio/openyurt/cmd/yurt-iot-dock/app/options" + iotv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" + "github.com/openyurtio/openyurt/pkg/yurtiotdock/clients" + edgexCli "github.com/openyurtio/openyurt/pkg/yurtiotdock/clients/edgex-foundry" + util "github.com/openyurtio/openyurt/pkg/yurtiotdock/controllers/util" +) + +// DeviceServiceReconciler reconciles a DeviceService object +type DeviceServiceReconciler struct { + client.Client + Scheme *runtime.Scheme + deviceServiceCli clients.DeviceServiceInterface + NodePool string + Namespace string +} + +//+kubebuilder:rbac:groups=iot.openyurt.io,resources=deviceservices,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=iot.openyurt.io,resources=deviceservices/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=iot.openyurt.io,resources=deviceservices/finalizers,verbs=update + +func (r *DeviceServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var ds iotv1alpha1.DeviceService + if err := r.Get(ctx, req.NamespacedName, &ds); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // If objects doesn't belong to the edge platform to which the controller is connected, the controller does not handle events for that object + if ds.Spec.NodePool != r.NodePool { + return ctrl.Result{}, nil + } + klog.V(3).Infof("Reconciling the DeviceService: %s", ds.GetName()) + + deviceServiceStatus := ds.Status.DeepCopy() + // Update deviceService conditions + defer func() { + if !ds.Spec.Managed { + util.SetDeviceServiceCondition(deviceServiceStatus, util.NewDeviceServiceCondition(iotv1alpha1.DeviceServiceManagingCondition, corev1.ConditionFalse, iotv1alpha1.DeviceServiceManagingReason, "")) + } + + err := r.Status().Update(ctx, &ds) + if client.IgnoreNotFound(err) != nil { + if !apierrors.IsConflict(err) { + klog.V(4).ErrorS(err, "update deviceService conditions failed", "deviceService", ds.GetName()) + } + } + }() + + // 1. Handle the deviceService deletion event + if err := r.reconcileDeleteDeviceService(ctx, &ds); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } else if !ds.ObjectMeta.DeletionTimestamp.IsZero() { + return ctrl.Result{}, nil + } + + if !ds.Status.Synced { + // 2. Synchronize OpenYurt deviceService to edge platform + if err := r.reconcileCreateDeviceService(ctx, &ds, deviceServiceStatus); err != nil { + if apierrors.IsConflict(err) { + return ctrl.Result{Requeue: true}, nil + } else { + return ctrl.Result{}, err + } + } + } else if ds.Spec.Managed { + // 3. If the deviceService has been synchronized and is managed by the cloud, reconcile the deviceService fields + if err := r.reconcileUpdateDeviceService(ctx, &ds, deviceServiceStatus); err != nil { + if apierrors.IsConflict(err) { + return ctrl.Result{Requeue: true}, nil + } else { + return ctrl.Result{}, err + } + } + } + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *DeviceServiceReconciler) SetupWithManager(mgr ctrl.Manager, opts *options.YurtIoTDockOptions) error { + r.deviceServiceCli = edgexCli.NewEdgexDeviceServiceClient(opts.CoreMetadataAddr) + r.NodePool = opts.Nodepool + r.Namespace = opts.Namespace + + return ctrl.NewControllerManagedBy(mgr). + For(&iotv1alpha1.DeviceService{}). + Complete(r) +} + +func (r *DeviceServiceReconciler) reconcileDeleteDeviceService(ctx context.Context, ds *iotv1alpha1.DeviceService) error { + // gets the actual name of deviceService on the edge platform from the Label of the device + edgeDeviceServiceName := util.GetEdgeDeviceServiceName(ds, EdgeXObjectName) + if ds.ObjectMeta.DeletionTimestamp.IsZero() { + if len(ds.GetFinalizers()) == 0 { + patchString := map[string]interface{}{ + "metadata": map[string]interface{}{ + "finalizers": []string{iotv1alpha1.DeviceServiceFinalizer}, + }, + } + if patchData, err := json.Marshal(patchString); err != nil { + return err + } else { + if err = r.Patch(ctx, ds, client.RawPatch(types.MergePatchType, patchData)); err != nil { + return err + } + } + } + } else { + patchString := map[string]interface{}{ + "metadata": map[string]interface{}{ + "finalizers": []string{}, + }, + } + // delete the deviceService in OpenYurt + if patchData, err := json.Marshal(patchString); err != nil { + return err + } else { + if err = r.Patch(ctx, ds, client.RawPatch(types.MergePatchType, patchData)); err != nil { + return err + } + } + + // delete the deviceService object on edge platform + err := r.deviceServiceCli.Delete(context.TODO(), edgeDeviceServiceName, clients.DeleteOptions{}) + if err != nil && !clients.IsNotFoundErr(err) { + return err + } + } + return nil +} + +func (r *DeviceServiceReconciler) reconcileCreateDeviceService(ctx context.Context, ds *iotv1alpha1.DeviceService, deviceServiceStatus *iotv1alpha1.DeviceServiceStatus) error { + // get the actual name of deviceService on the Edge platform from the Label of the device + edgeDeviceServiceName := util.GetEdgeDeviceServiceName(ds, EdgeXObjectName) + klog.V(4).Infof("Checking if deviceService already exist on the edge platform: %s", ds.GetName()) + // Checking if deviceService already exist on the edge platform + if edgeDs, err := r.deviceServiceCli.Get(context.TODO(), edgeDeviceServiceName, clients.GetOptions{Namespace: r.Namespace}); err != nil { + if !clients.IsNotFoundErr(err) { + klog.V(4).ErrorS(err, "fail to visit the edge platform") + return nil + } else { + createdDs, err := r.deviceServiceCli.Create(context.TODO(), ds, clients.CreateOptions{}) + if err != nil { + klog.V(4).ErrorS(err, "failed to create deviceService on edge platform") + util.SetDeviceServiceCondition(deviceServiceStatus, util.NewDeviceServiceCondition(iotv1alpha1.DeviceServiceSyncedCondition, corev1.ConditionFalse, iotv1alpha1.DeviceServiceCreateSyncedReason, err.Error())) + return fmt.Errorf("fail to create DeviceService to edge platform: %v", err) + } + + klog.V(4).Infof("Successfully add DeviceService to Edge Platform, Name: %s, EdgeId: %s", ds.GetName(), createdDs.Status.EdgeId) + ds.Status.EdgeId = createdDs.Status.EdgeId + ds.Status.Synced = true + util.SetDeviceServiceCondition(deviceServiceStatus, util.NewDeviceServiceCondition(iotv1alpha1.DeviceServiceSyncedCondition, corev1.ConditionTrue, "", "")) + return r.Status().Update(ctx, ds) + } + } else { + // a. If object exists, the status of the device on OpenYurt is updated + klog.V(4).Infof("DeviceServiceName: %s, obj already exists on edge platform", ds.GetName()) + ds.Status.Synced = true + ds.Status.EdgeId = edgeDs.Status.EdgeId + return r.Status().Update(ctx, ds) + } +} + +func (r *DeviceServiceReconciler) reconcileUpdateDeviceService(ctx context.Context, ds *iotv1alpha1.DeviceService, deviceServiceStatus *iotv1alpha1.DeviceServiceStatus) error { + // 1. reconciling the AdminState field of deviceService + newDeviceServiceStatus := ds.Status.DeepCopy() + updateDeviceService := ds.DeepCopy() + + if ds.Spec.AdminState != "" && ds.Spec.AdminState != ds.Status.AdminState { + newDeviceServiceStatus.AdminState = ds.Spec.AdminState + } else { + updateDeviceService.Spec.AdminState = "" + } + + _, err := r.deviceServiceCli.Update(context.TODO(), updateDeviceService, clients.UpdateOptions{}) + if err != nil { + util.SetDeviceServiceCondition(deviceServiceStatus, util.NewDeviceServiceCondition(iotv1alpha1.DeviceServiceManagingCondition, corev1.ConditionFalse, iotv1alpha1.DeviceServiceUpdateStatusSyncedReason, err.Error())) + + return err + } + + // 2. update the device status on OpenYurt + ds.Status = *newDeviceServiceStatus + if err = r.Status().Update(ctx, ds); err != nil { + util.SetDeviceServiceCondition(deviceServiceStatus, util.NewDeviceServiceCondition(iotv1alpha1.DeviceServiceManagingCondition, corev1.ConditionFalse, iotv1alpha1.DeviceServiceUpdateStatusSyncedReason, err.Error())) + + return err + } + util.SetDeviceServiceCondition(deviceServiceStatus, util.NewDeviceServiceCondition(iotv1alpha1.DeviceServiceManagingCondition, corev1.ConditionTrue, "", "")) + return nil +} diff --git a/pkg/yurtiotdock/controllers/deviceservice_syncer.go b/pkg/yurtiotdock/controllers/deviceservice_syncer.go new file mode 100644 index 00000000000..78949c55069 --- /dev/null +++ b/pkg/yurtiotdock/controllers/deviceservice_syncer.go @@ -0,0 +1,237 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package controllers + +import ( + "context" + "strings" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/openyurtio/openyurt/cmd/yurt-iot-dock/app/options" + iotv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" + iotcli "github.com/openyurtio/openyurt/pkg/yurtiotdock/clients" + edgexCli "github.com/openyurtio/openyurt/pkg/yurtiotdock/clients/edgex-foundry" + "github.com/openyurtio/openyurt/pkg/yurtiotdock/controllers/util" +) + +type DeviceServiceSyncer struct { + // Kubernetes client + client.Client + // syncing period in seconds + syncPeriod time.Duration + deviceServiceCli iotcli.DeviceServiceInterface + NodePool string + Namespace string +} + +func NewDeviceServiceSyncer(client client.Client, opts *options.YurtIoTDockOptions) (DeviceServiceSyncer, error) { + return DeviceServiceSyncer{ + syncPeriod: time.Duration(opts.EdgeSyncPeriod) * time.Second, + deviceServiceCli: edgexCli.NewEdgexDeviceServiceClient(opts.CoreMetadataAddr), + Client: client, + NodePool: opts.Nodepool, + Namespace: opts.Namespace, + }, nil +} + +func (ds *DeviceServiceSyncer) NewDeviceServiceSyncerRunnable() ctrlmgr.RunnableFunc { + return func(ctx context.Context) error { + ds.Run(ctx.Done()) + return nil + } +} + +func (ds *DeviceServiceSyncer) Run(stop <-chan struct{}) { + klog.V(1).Info("[DeviceService] Starting the syncer...") + go func() { + for { + <-time.After(ds.syncPeriod) + klog.V(2).Info("[DeviceService] Start a round of synchronization.") + // 1. get deviceServices on edge platform and OpenYurt + edgeDeviceServices, kubeDeviceServices, err := ds.getAllDeviceServices() + if err != nil { + klog.V(3).ErrorS(err, "fail to list the deviceServices") + continue + } + + // 2. find the deviceServices that need to be synchronized + redundantEdgeDeviceServices, redundantKubeDeviceServices, syncedDeviceServices := + ds.findDiffDeviceServices(edgeDeviceServices, kubeDeviceServices) + klog.V(2).Infof("[DeviceService] The number of objects waiting for synchronization { %s:%d, %s:%d, %s:%d }", + "Edge deviceServices should be added to OpenYurt", len(redundantEdgeDeviceServices), + "OpenYurt deviceServices that should be deleted", len(redundantKubeDeviceServices), + "DeviceServices that should be synchronized", len(syncedDeviceServices)) + + // 3. create deviceServices on OpenYurt which are exists in edge platform but not in OpenYurt + if err := ds.syncEdgeToKube(redundantEdgeDeviceServices); err != nil { + klog.V(3).ErrorS(err, "fail to create deviceServices on OpenYurt") + } + + // 4. delete redundant deviceServices on OpenYurt + if err := ds.deleteDeviceServices(redundantKubeDeviceServices); err != nil { + klog.V(3).ErrorS(err, "fail to delete redundant deviceServices on OpenYurt") + } + + // 5. update deviceService status on OpenYurt + if err := ds.updateDeviceServices(syncedDeviceServices); err != nil { + klog.V(3).ErrorS(err, "fail to update deviceServices") + } + klog.V(2).Info("[DeviceService] One round of synchronization is complete") + } + }() + + <-stop + klog.V(1).Info("[DeviceService] Stopping the syncer") +} + +// Get the existing DeviceService on the Edge platform, as well as OpenYurt existing DeviceService +// edgeDeviceServices：map[actualName]DeviceService +// kubeDeviceServices：map[actualName]DeviceService +func (ds *DeviceServiceSyncer) getAllDeviceServices() ( + map[string]iotv1alpha1.DeviceService, map[string]iotv1alpha1.DeviceService, error) { + + edgeDeviceServices := map[string]iotv1alpha1.DeviceService{} + kubeDeviceServices := map[string]iotv1alpha1.DeviceService{} + + // 1. list deviceServices on edge platform + eDevSs, err := ds.deviceServiceCli.List(context.TODO(), iotcli.ListOptions{Namespace: ds.Namespace}) + if err != nil { + klog.V(4).ErrorS(err, "fail to list the deviceServices object on the edge platform") + return edgeDeviceServices, kubeDeviceServices, err + } + // 2. list deviceServices on OpenYurt (filter objects belonging to edgeServer) + var kDevSs iotv1alpha1.DeviceServiceList + listOptions := client.MatchingFields{util.IndexerPathForNodepool: ds.NodePool} + if err = ds.List(context.TODO(), &kDevSs, listOptions, client.InNamespace(ds.Namespace)); err != nil { + klog.V(4).ErrorS(err, "fail to list the deviceServices object on the Kubernetes") + return edgeDeviceServices, kubeDeviceServices, err + } + for i := range eDevSs { + deviceServicesName := util.GetEdgeDeviceServiceName(&eDevSs[i], EdgeXObjectName) + edgeDeviceServices[deviceServicesName] = eDevSs[i] + } + + for i := range kDevSs.Items { + deviceServicesName := util.GetEdgeDeviceServiceName(&kDevSs.Items[i], EdgeXObjectName) + kubeDeviceServices[deviceServicesName] = kDevSs.Items[i] + } + return edgeDeviceServices, kubeDeviceServices, nil +} + +// Get the list of deviceServices that need to be added, deleted and updated +func (ds *DeviceServiceSyncer) findDiffDeviceServices( + edgeDeviceService map[string]iotv1alpha1.DeviceService, kubeDeviceService map[string]iotv1alpha1.DeviceService) ( + redundantEdgeDeviceServices map[string]*iotv1alpha1.DeviceService, redundantKubeDeviceServices map[string]*iotv1alpha1.DeviceService, syncedDeviceServices map[string]*iotv1alpha1.DeviceService) { + + redundantEdgeDeviceServices = map[string]*iotv1alpha1.DeviceService{} + redundantKubeDeviceServices = map[string]*iotv1alpha1.DeviceService{} + syncedDeviceServices = map[string]*iotv1alpha1.DeviceService{} + + for i := range edgeDeviceService { + eds := edgeDeviceService[i] + edName := util.GetEdgeDeviceServiceName(&eds, EdgeXObjectName) + if _, exists := kubeDeviceService[edName]; !exists { + redundantEdgeDeviceServices[edName] = ds.completeCreateContent(&eds) + } else { + kd := kubeDeviceService[edName] + syncedDeviceServices[edName] = ds.completeUpdateContent(&kd, &eds) + } + } + + for i := range kubeDeviceService { + kds := kubeDeviceService[i] + if !kds.Status.Synced { + continue + } + kdName := util.GetEdgeDeviceServiceName(&kds, EdgeXObjectName) + if _, exists := edgeDeviceService[kdName]; !exists { + redundantKubeDeviceServices[kdName] = &kds + } + } + return +} + +// syncEdgeToKube creates deviceServices on OpenYurt which are exists in edge platform but not in OpenYurt +func (ds *DeviceServiceSyncer) syncEdgeToKube(edgeDevs map[string]*iotv1alpha1.DeviceService) error { + for _, ed := range edgeDevs { + if err := ds.Client.Create(context.TODO(), ed); err != nil { + if apierrors.IsAlreadyExists(err) { + klog.V(5).InfoS("DeviceService already exist on Kubernetes", + "DeviceService", strings.ToLower(ed.Name)) + continue + } + klog.InfoS("created deviceService failed:", "DeviceService", strings.ToLower(ed.Name)) + return err + } + } + return nil +} + +// deleteDeviceServices deletes redundant deviceServices on OpenYurt +func (ds *DeviceServiceSyncer) deleteDeviceServices(redundantKubeDeviceServices map[string]*iotv1alpha1.DeviceService) error { + for _, kds := range redundantKubeDeviceServices { + if err := ds.Client.Delete(context.TODO(), kds); err != nil { + klog.V(5).ErrorS(err, "fail to delete the DeviceService on Kubernetes", + "DeviceService", kds.Name) + return err + } + } + return nil +} + +// updateDeviceServices updates deviceServices status on OpenYurt +func (ds *DeviceServiceSyncer) updateDeviceServices(syncedDeviceServices map[string]*iotv1alpha1.DeviceService) error { + for _, sd := range syncedDeviceServices { + if sd.ObjectMeta.ResourceVersion == "" { + continue + } + if err := ds.Client.Status().Update(context.TODO(), sd); err != nil { + if apierrors.IsConflict(err) { + klog.V(5).InfoS("update Conflicts", "DeviceService", sd.Name) + continue + } + klog.V(5).ErrorS(err, "fail to update the DeviceService on Kubernetes", + "DeviceService", sd.Name) + return err + } + } + return nil +} + +// completeCreateContent completes the content of the deviceService which will be created on OpenYurt +func (ds *DeviceServiceSyncer) completeCreateContent(edgeDS *iotv1alpha1.DeviceService) *iotv1alpha1.DeviceService { + createDevice := edgeDS.DeepCopy() + createDevice.Spec.NodePool = ds.NodePool + createDevice.Namespace = ds.Namespace + createDevice.Name = strings.Join([]string{ds.NodePool, edgeDS.Name}, "-") + createDevice.Spec.Managed = false + return createDevice +} + +// completeUpdateContent completes the content of the deviceService which will be updated on OpenYurt +func (ds *DeviceServiceSyncer) completeUpdateContent(kubeDS *iotv1alpha1.DeviceService, edgeDS *iotv1alpha1.DeviceService) *iotv1alpha1.DeviceService { + updatedDS := kubeDS.DeepCopy() + // update device status + updatedDS.Status.LastConnected = edgeDS.Status.LastConnected + updatedDS.Status.LastReported = edgeDS.Status.LastReported + updatedDS.Status.AdminState = edgeDS.Status.AdminState + return updatedDS +} diff --git a/pkg/yurtiotdock/controllers/predicate.go b/pkg/yurtiotdock/controllers/predicate.go new file mode 100644 index 00000000000..0c6dc5eadb5 --- /dev/null +++ b/pkg/yurtiotdock/controllers/predicate.go @@ -0,0 +1,48 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + edgexCli "github.com/openyurtio/openyurt/pkg/yurtiotdock/clients/edgex-foundry" +) + +func genFirstUpdateFilter(objKind string) predicate.Predicate { + return predicate.Funcs{ + // ignore the update event that is generated due to a + // new deviceprofile being added to the Edgex Foundry + UpdateFunc: func(e event.UpdateEvent) bool { + oldDp, ok := e.ObjectOld.(edgexCli.EdgeXObject) + if !ok { + klog.Infof("fail to assert object to deviceprofile, object kind is %s", objKind) + return false + } + newDp, ok := e.ObjectNew.(edgexCli.EdgeXObject) + if !ok { + klog.Infof("fail to assert object to deviceprofile, object kind is %s", objKind) + return false + } + if !oldDp.IsAddedToEdgeX() && newDp.IsAddedToEdgeX() { + return false + } + return true + }, + } +} diff --git a/pkg/yurtiotdock/controllers/util/conditions.go b/pkg/yurtiotdock/controllers/util/conditions.go new file mode 100644 index 00000000000..db892cc1459 --- /dev/null +++ b/pkg/yurtiotdock/controllers/util/conditions.go @@ -0,0 +1,120 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + iotv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" +) + +// NewDeviceCondition creates a new Device condition. +func NewDeviceCondition(condType iotv1alpha1.DeviceConditionType, status corev1.ConditionStatus, reason, message string) *iotv1alpha1.DeviceCondition { + return &iotv1alpha1.DeviceCondition{ + Type: condType, + Status: status, + LastTransitionTime: metav1.Now(), + Reason: reason, + Message: message, + } +} + +// GetDeviceCondition returns the condition with the provided type. +func GetDeviceCondition(status iotv1alpha1.DeviceStatus, condType iotv1alpha1.DeviceConditionType) *iotv1alpha1.DeviceCondition { + for i := range status.Conditions { + c := status.Conditions[i] + if c.Type == condType { + return &c + } + } + return nil +} + +// SetDeviceCondition updates the Device to include the provided condition. If the condition that +// we are about to add already exists and has the same status, reason and message then we are not going to update. +func SetDeviceCondition(status *iotv1alpha1.DeviceStatus, condition *iotv1alpha1.DeviceCondition) { + currentCond := GetDeviceCondition(*status, condition.Type) + if currentCond != nil && currentCond.Status == condition.Status && currentCond.Reason == condition.Reason { + return + } + + if currentCond != nil && currentCond.Status == condition.Status { + condition.LastTransitionTime = currentCond.LastTransitionTime + } + newConditions := filterOutDeviceCondition(status.Conditions, condition.Type) + status.Conditions = append(newConditions, *condition) +} + +func filterOutDeviceCondition(conditions []iotv1alpha1.DeviceCondition, condType iotv1alpha1.DeviceConditionType) []iotv1alpha1.DeviceCondition { + var newConditions []iotv1alpha1.DeviceCondition + for _, c := range conditions { + if c.Type == condType { + continue + } + newConditions = append(newConditions, c) + } + return newConditions +} + +// NewDeviceServiceCondition creates a new DeviceService condition. +func NewDeviceServiceCondition(condType iotv1alpha1.DeviceServiceConditionType, status corev1.ConditionStatus, reason, message string) *iotv1alpha1.DeviceServiceCondition { + return &iotv1alpha1.DeviceServiceCondition{ + Type: condType, + Status: status, + LastTransitionTime: metav1.Now(), + Reason: reason, + Message: message, + } +} + +// GetDeviceServiceCondition returns the condition with the provided type. +func GetDeviceServiceCondition(status iotv1alpha1.DeviceServiceStatus, condType iotv1alpha1.DeviceServiceConditionType) *iotv1alpha1.DeviceServiceCondition { + for i := range status.Conditions { + c := status.Conditions[i] + if c.Type == condType { + return &c + } + } + return nil +} + +// SetDeviceServiceCondition updates the DeviceService to include the provided condition. If the condition that +// we are about to add already exists and has the same status, reason and message then we are not going to update. +func SetDeviceServiceCondition(status *iotv1alpha1.DeviceServiceStatus, condition *iotv1alpha1.DeviceServiceCondition) { + currentCond := GetDeviceServiceCondition(*status, condition.Type) + if currentCond != nil && currentCond.Status == condition.Status && currentCond.Reason == condition.Reason { + return + } + + if currentCond != nil && currentCond.Status == condition.Status { + condition.LastTransitionTime = currentCond.LastTransitionTime + } + newConditions := filterOutDeviceServiceCondition(status.Conditions, condition.Type) + status.Conditions = append(newConditions, *condition) +} + +func filterOutDeviceServiceCondition(conditions []iotv1alpha1.DeviceServiceCondition, condType iotv1alpha1.DeviceServiceConditionType) []iotv1alpha1.DeviceServiceCondition { + var newConditions []iotv1alpha1.DeviceServiceCondition + for _, c := range conditions { + if c.Type == condType { + continue + } + newConditions = append(newConditions, c) + } + return newConditions +} diff --git a/pkg/yurtiotdock/controllers/util/fieldindexer.go b/pkg/yurtiotdock/controllers/util/fieldindexer.go new file mode 100644 index 00000000000..8245d343274 --- /dev/null +++ b/pkg/yurtiotdock/controllers/util/fieldindexer.go @@ -0,0 +1,62 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "context" + "sync" + + "sigs.k8s.io/controller-runtime/pkg/client" + + iotv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" +) + +const ( + IndexerPathForNodepool = "spec.nodePool" +) + +var registerOnce sync.Once + +func RegisterFieldIndexers(fi client.FieldIndexer) error { + var err error + registerOnce.Do(func() { + // register the fieldIndexer for device + if err = fi.IndexField(context.TODO(), &iotv1alpha1.Device{}, IndexerPathForNodepool, func(rawObj client.Object) []string { + device := rawObj.(*iotv1alpha1.Device) + return []string{device.Spec.NodePool} + }); err != nil { + return + } + + // register the fieldIndexer for deviceService + if err = fi.IndexField(context.TODO(), &iotv1alpha1.DeviceService{}, IndexerPathForNodepool, func(rawObj client.Object) []string { + deviceService := rawObj.(*iotv1alpha1.DeviceService) + return []string{deviceService.Spec.NodePool} + }); err != nil { + return + } + + // register the fieldIndexer for deviceProfile + if err = fi.IndexField(context.TODO(), &iotv1alpha1.DeviceProfile{}, IndexerPathForNodepool, func(rawObj client.Object) []string { + profile := rawObj.(*iotv1alpha1.DeviceProfile) + return []string{profile.Spec.NodePool} + }); err != nil { + return + } + }) + return err +} diff --git a/pkg/yurtiotdock/controllers/util/string.go b/pkg/yurtiotdock/controllers/util/string.go new file mode 100644 index 00000000000..4e7607a4d56 --- /dev/null +++ b/pkg/yurtiotdock/controllers/util/string.go @@ -0,0 +1,30 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +// IsInStringLst checks if 'str' is in the 'strLst' +func IsInStringLst(strLst []string, str string) bool { + if len(strLst) == 0 { + return false + } + for _, s := range strLst { + if str == s { + return true + } + } + return false +} diff --git a/pkg/yurtiotdock/controllers/util/string_test.go b/pkg/yurtiotdock/controllers/util/string_test.go new file mode 100644 index 00000000000..7f56226ad8d --- /dev/null +++ b/pkg/yurtiotdock/controllers/util/string_test.go @@ -0,0 +1,70 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "testing" +) + +func TestIsInStringLst(t *testing.T) { + tests := []struct { + desc string + sl []string + s string + res bool + }{ + { + "test empty list", + []string{}, + "a", + false, + }, + { + "test not in list", + []string{"a", "b", "c"}, + "d", + false, + }, + { + "test not in list with one element", + []string{"a"}, + "b", + false, + }, + { + "test in list with one element", + []string{"aaa"}, + "aaa", + true, + }, + { + "test in list with one element", + []string{"aaa", "a", "bbb"}, + "a", + true, + }, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + res := IsInStringLst(tt.sl, tt.s) + if res != tt.res { + t.Errorf("expect %v, but %v returned", tt.res, res) + } + }) + } +} diff --git a/pkg/yurtiotdock/controllers/util/tools.go b/pkg/yurtiotdock/controllers/util/tools.go new file mode 100644 index 00000000000..a95dc50395a --- /dev/null +++ b/pkg/yurtiotdock/controllers/util/tools.go @@ -0,0 +1,99 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "context" + "fmt" + "io/ioutil" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + iotv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" +) + +const ( + PODHOSTNAME = "/etc/hostname" + PODNAMESPACE = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" +) + +// GetNodePool get nodepool where yurt-iot-dock run +func GetNodePool(cfg *rest.Config) (string, error) { + var nodePool string + client, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nodePool, err + } + + bn, err := ioutil.ReadFile(PODHOSTNAME) + if err != nil { + return nodePool, fmt.Errorf("Read file %s fail: %v", PODHOSTNAME, err) + } + bns, err := ioutil.ReadFile(PODNAMESPACE) + if err != nil { + return nodePool, fmt.Errorf("Read file %s fail: %v", PODNAMESPACE, err) + } + name := strings.Replace(string(bn), "\n", "", -1) + namespace := string(bns) + + pod, err := client.CoreV1().Pods(namespace).Get(context.Background(), name, metav1.GetOptions{}) + if err != nil { + return nodePool, fmt.Errorf("not found pod %s/%s: %v", namespace, name, err) + } + node, err := client.CoreV1().Nodes().Get(context.Background(), pod.Spec.NodeName, metav1.GetOptions{}) + if err != nil { + return nodePool, fmt.Errorf("not found node %s: %v", pod.Spec.NodeName, err) + } + nodePool, ok := node.Labels["apps.openyurt.io/nodepool"] + if !ok { + return nodePool, fmt.Errorf("node %s doesn't add to a nodepool", node.GetName()) + } + return nodePool, err +} + +func GetEdgeDeviceServiceName(ds *iotv1alpha1.DeviceService, label string) string { + var actualDSName string + if _, ok := ds.ObjectMeta.Labels[label]; ok { + actualDSName = ds.ObjectMeta.Labels[label] + } else { + actualDSName = ds.GetName() + } + return actualDSName +} + +func GetEdgeDeviceName(d *iotv1alpha1.Device, label string) string { + var actualDeviceName string + if _, ok := d.ObjectMeta.Labels[label]; ok { + actualDeviceName = d.ObjectMeta.Labels[label] + } else { + actualDeviceName = d.GetName() + } + return actualDeviceName +} + +func GetEdgeDeviceProfileName(dp *iotv1alpha1.DeviceProfile, label string) string { + var actualDPName string + if _, ok := dp.ObjectMeta.Labels[label]; ok { + actualDPName = dp.ObjectMeta.Labels[label] + } else { + actualDPName = dp.GetName() + } + return actualDPName +} diff --git a/pkg/yurtiotdock/controllers/util/tools_test.go b/pkg/yurtiotdock/controllers/util/tools_test.go new file mode 100644 index 00000000000..fde94c4fffa --- /dev/null +++ b/pkg/yurtiotdock/controllers/util/tools_test.go @@ -0,0 +1,55 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/client-go/rest" + + iotv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" +) + +func TestGetNodePool(t *testing.T) { + cfg := &rest.Config{} + res, err := GetNodePool(cfg) + if res != "" { + t.Errorf("expect nil on null config") + } + if err == nil { + t.Errorf("null config must cause error") + } +} + +func TestGetEdgeDeviceServiceName(t *testing.T) { + d := &iotv1alpha1.DeviceService{} + assert.Equal(t, GetEdgeDeviceServiceName(d, ""), "") + assert.Equal(t, GetEdgeDeviceServiceName(d, "a"), "") +} + +func TestGetEdgeDeviceName(t *testing.T) { + d := &iotv1alpha1.Device{} + assert.Equal(t, GetEdgeDeviceName(d, ""), "") + assert.Equal(t, GetEdgeDeviceName(d, "a"), "") +} + +func TestGetEdgeDeviceProfileName(t *testing.T) { + d := &iotv1alpha1.DeviceProfile{} + assert.Equal(t, GetEdgeDeviceProfileName(d, ""), "") + assert.Equal(t, GetEdgeDeviceProfileName(d, "a"), "") +} diff --git a/pkg/yurtiotdock/controllers/well_known_labels.go b/pkg/yurtiotdock/controllers/well_known_labels.go new file mode 100644 index 00000000000..fd414f13c9e --- /dev/null +++ b/pkg/yurtiotdock/controllers/well_known_labels.go @@ -0,0 +1,21 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +const ( + EdgeXObjectName = "yurt-iot-dock/edgex-object.name" +) From 3595932adc94859a3e70ddf8613e01a208dd4668 Mon Sep 17 00:00:00 2001 From: wesleysu <59680532+River-sh@users.noreply.github.com> Date: Fri, 11 Aug 2023 11:38:22 +0800 Subject: [PATCH 05/21] add new gateway version (#1641) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 珩轩 --- .../crds/raven.openyurt.io_gateways.yaml | 195 ++++++++++++++++ .../yurt-manager-auto-generated.yaml | 12 +- pkg/apis/addtoscheme_raven_v1beta1.go | 26 +++ .../apps/v1alpha1/zz_generated.deepcopy.go | 2 +- .../apps/v1beta1/zz_generated.deepcopy.go | 2 +- pkg/apis/raven/v1alpha1/gateway_conversion.go | 94 +++++++- .../raven/v1alpha1/zz_generated.deepcopy.go | 2 +- pkg/apis/raven/v1beta1/default.go | 43 ++++ pkg/apis/raven/v1beta1/gateway_conversion.go | 48 ++++ pkg/apis/raven/v1beta1/gateway_types.go | 139 +++++++++++ pkg/apis/raven/v1beta1/groupversion_info.go | 44 ++++ .../raven/v1beta1/zz_generated.deepcopy.go | 220 ++++++++++++++++++ pkg/controller/raven/common.go | 2 +- .../gateway/v1alpha1/gateway_default.go | 4 +- .../gateway/v1alpha1/gateway_handler.go | 3 - .../gateway/v1beta1/gateway_default.go | 39 ++++ .../gateway/v1beta1/gateway_handler.go | 56 +++++ .../gateway/v1beta1/gateway_validation.go | 128 ++++++++++ pkg/webhook/server.go | 4 +- 19 files changed, 1038 insertions(+), 25 deletions(-) create mode 100644 pkg/apis/addtoscheme_raven_v1beta1.go create mode 100644 pkg/apis/raven/v1beta1/default.go create mode 100644 pkg/apis/raven/v1beta1/gateway_conversion.go create mode 100644 pkg/apis/raven/v1beta1/gateway_types.go create mode 100644 pkg/apis/raven/v1beta1/groupversion_info.go create mode 100644 pkg/apis/raven/v1beta1/zz_generated.deepcopy.go create mode 100644 pkg/webhook/gateway/v1beta1/gateway_default.go create mode 100644 pkg/webhook/gateway/v1beta1/gateway_handler.go create mode 100644 pkg/webhook/gateway/v1beta1/gateway_validation.go diff --git a/charts/yurt-manager/crds/raven.openyurt.io_gateways.yaml b/charts/yurt-manager/crds/raven.openyurt.io_gateways.yaml index 0924060e97d..1fbbefafb3f 100644 --- a/charts/yurt-manager/crds/raven.openyurt.io_gateways.yaml +++ b/charts/yurt-manager/crds/raven.openyurt.io_gateways.yaml @@ -159,6 +159,201 @@ spec: type: object type: object served: true + storage: false + subresources: + status: {} + - name: v1beta1 + schema: + openAPIV3Schema: + description: Gateway is the Schema for the gateways API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: GatewaySpec defines the desired state of Gateway + properties: + endpoints: + description: Endpoints are a list of available Endpoint. + items: + description: Endpoint stores all essential data for establishing + the VPN tunnel and Proxy + properties: + config: + additionalProperties: + type: string + description: Config is a map to record config for the raven + agent of node + type: object + nodeName: + description: NodeName is the Node hosting this endpoint. + type: string + port: + description: Port is the exposed port of the node + type: integer + publicIP: + description: PublicIP is the exposed IP of the node + type: string + type: + description: Type is the service type of the node, proxy or + tunnel + type: string + underNAT: + description: UnderNAT indicates whether node is under NAT + type: boolean + required: + - nodeName + - type + type: object + type: array + exposeType: + description: ExposeType determines how the Gateway is exposed. + type: string + nodeSelector: + description: NodeSelector is a label query over nodes that managed + by the gateway. The nodes in the same gateway should share same + layer 3 network. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + proxyConfig: + description: ProxyConfig determine the l7 proxy configuration + properties: + Replicas: + description: Replicas is the number of gateway active endpoints + that enabled proxy + type: integer + proxyHTTPPort: + description: ProxyHTTPPort is the proxy http port of the cross-domain + request + type: string + proxyHTTPSPort: + description: ProxyHTTPSPort is the proxy https port of the cross-domain + request + type: string + required: + - Replicas + type: object + tunnelConfig: + description: TunnelConfig determine the l3 tunnel configuration + properties: + Replicas: + description: Replicas is the number of gateway active endpoints + that enabled tunnel + type: integer + required: + - Replicas + type: object + type: object + status: + description: GatewayStatus defines the observed state of Gateway + properties: + activeEndpoints: + description: ActiveEndpoints is the reference of the active endpoint. + items: + description: Endpoint stores all essential data for establishing + the VPN tunnel and Proxy + properties: + config: + additionalProperties: + type: string + description: Config is a map to record config for the raven + agent of node + type: object + nodeName: + description: NodeName is the Node hosting this endpoint. + type: string + port: + description: Port is the exposed port of the node + type: integer + publicIP: + description: PublicIP is the exposed IP of the node + type: string + type: + description: Type is the service type of the node, proxy or + tunnel + type: string + underNAT: + description: UnderNAT indicates whether node is under NAT + type: boolean + required: + - nodeName + - type + type: object + type: array + nodes: + description: Nodes contains all information of nodes managed by Gateway. + items: + description: NodeInfo stores information of node managed by Gateway. + properties: + nodeName: + description: NodeName is the Node host name. + type: string + privateIP: + description: PrivateIP is the node private ip address + type: string + subnets: + description: Subnets is the pod ip range of the node + items: + type: string + type: array + required: + - nodeName + - privateIP + - subnets + type: object + type: array + type: object + type: object + served: true storage: true subresources: status: {} diff --git a/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml b/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml index 72e35edad21..177e5500223 100644 --- a/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml +++ b/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml @@ -522,14 +522,14 @@ webhooks: service: name: yurt-manager-webhook-service namespace: {{ .Release.Namespace }} - path: /mutate-raven-openyurt-io-v1alpha1-gateway + path: /mutate-raven-openyurt-io-v1beta1-gateway failurePolicy: Fail - name: mutate.raven.v1alpha1.gateway.openyurt.io + name: mutate.gateway.v1beta1.raven.openyurt.io rules: - apiGroups: - raven.openyurt.io apiVersions: - - v1alpha1 + - v1beta1 operations: - CREATE - UPDATE @@ -652,14 +652,14 @@ webhooks: service: name: yurt-manager-webhook-service namespace: {{ .Release.Namespace }} - path: /validate-raven-openyurt-io-v1alpha1-gateway + path: /validate-raven-openyurt-io-v1beta1-gateway failurePolicy: Fail - name: validate.raven.v1alpha1.gateway.openyurt.io + name: validate.gateway.v1beta1.raven.openyurt.io rules: - apiGroups: - raven.openyurt.io apiVersions: - - v1alpha1 + - v1beta1 operations: - CREATE - UPDATE diff --git a/pkg/apis/addtoscheme_raven_v1beta1.go b/pkg/apis/addtoscheme_raven_v1beta1.go new file mode 100644 index 00000000000..f5f8a575d08 --- /dev/null +++ b/pkg/apis/addtoscheme_raven_v1beta1.go @@ -0,0 +1,26 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apis + +import ( + version "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" +) + +func init() { + // Register the types with the Scheme so the components can map objects to GroupVersionKinds and back + AddToSchemes = append(AddToSchemes, version.SchemeBuilder.AddToScheme) +} diff --git a/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go index a1352c1b16f..f732bdcaa9b 100644 --- a/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/apps/v1alpha1/zz_generated.deepcopy.go @@ -23,7 +23,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" ) diff --git a/pkg/apis/apps/v1beta1/zz_generated.deepcopy.go b/pkg/apis/apps/v1beta1/zz_generated.deepcopy.go index a62c4266aa2..ebb32758258 100644 --- a/pkg/apis/apps/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/apps/v1beta1/zz_generated.deepcopy.go @@ -23,7 +23,7 @@ package v1beta1 import ( corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/pkg/apis/raven/v1alpha1/gateway_conversion.go b/pkg/apis/raven/v1alpha1/gateway_conversion.go index ebcce13d9dc..6192bfc4a21 100644 --- a/pkg/apis/raven/v1alpha1/gateway_conversion.go +++ b/pkg/apis/raven/v1alpha1/gateway_conversion.go @@ -16,6 +16,13 @@ limitations under the License. package v1alpha1 +import ( + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/conversion" + + "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" +) + /* Implementing the hub method is pretty easy -- we just have to add an empty method called Hub() to serve as a @@ -31,15 +38,86 @@ method called Hub() to serve as a // NOTE !!!!!!! @kadisi // If this version is not storageversion, you need to implement the ConvertTo and ConvertFrom methods -// need import "sigs.k8s.io/controller-runtime/pkg/conversion" -//func (src *Gateway) ConvertTo(dstRaw conversion.Hub) error { -// return nil -//} +func (src *Gateway) ConvertTo(dstRaw conversion.Hub) error { + dst := dstRaw.(*v1beta1.Gateway) + dst.ObjectMeta = src.ObjectMeta + if src.Spec.NodeSelector != nil { + dst.Spec.NodeSelector = src.Spec.NodeSelector + } + dst.Spec.ExposeType = string(src.Spec.ExposeType) + dst.Spec.TunnelConfig.Replicas = 1 + dst.Spec.ProxyConfig.Replicas = 1 + for _, eps := range src.Spec.Endpoints { + dst.Spec.Endpoints = append(dst.Spec.Endpoints, v1beta1.Endpoint{ + NodeName: eps.NodeName, + PublicIP: eps.PublicIP, + UnderNAT: eps.UnderNAT, + Config: eps.Config, + Type: v1beta1.Tunnel, + Port: v1beta1.DefaultTunnelServerExposedPort, + }) + } + for _, node := range src.Status.Nodes { + dst.Status.Nodes = append(dst.Status.Nodes, v1beta1.NodeInfo{ + NodeName: node.NodeName, + PrivateIP: node.PrivateIP, + Subnets: node.Subnets, + }) + } + if src.Status.ActiveEndpoint != nil { + dst.Status.ActiveEndpoints = []*v1beta1.Endpoint{ + { + NodeName: src.Status.ActiveEndpoint.NodeName, + PublicIP: src.Status.ActiveEndpoint.PublicIP, + UnderNAT: src.Status.ActiveEndpoint.UnderNAT, + Config: src.Status.ActiveEndpoint.Config, + Type: v1beta1.Tunnel, + Port: v1beta1.DefaultTunnelServerExposedPort, + }, + } + } + + klog.Infof("convert from v1alpha1 to v1beta1 for %s", dst.Name) + return nil +} // NOTE !!!!!!! @kadisi // If this version is not storageversion, you need to implement the ConvertTo and ConvertFrom methods -// need import "sigs.k8s.io/controller-runtime/pkg/conversion" -//func (dst *Gateway) ConvertFrom(srcRaw conversion.Hub) error { -// return nil -//} +func (dst *Gateway) ConvertFrom(srcRaw conversion.Hub) error { + src := srcRaw.(*v1beta1.Gateway) + dst.ObjectMeta = src.ObjectMeta + dst.Spec.NodeSelector = src.Spec.NodeSelector + dst.Spec.ExposeType = ExposeType(src.Spec.ExposeType) + for _, eps := range src.Spec.Endpoints { + dst.Spec.Endpoints = append(dst.Spec.Endpoints, Endpoint{ + NodeName: eps.NodeName, + PublicIP: eps.PublicIP, + UnderNAT: eps.UnderNAT, + Config: eps.Config, + }) + } + for _, node := range src.Status.Nodes { + dst.Status.Nodes = append(dst.Status.Nodes, NodeInfo{ + NodeName: node.NodeName, + PrivateIP: node.PrivateIP, + Subnets: node.Subnets, + }) + } + if src.Status.ActiveEndpoints == nil { + klog.Infof("convert from v1beta1 to v1alpha1 for %s", dst.Name) + return nil + } + if len(src.Status.ActiveEndpoints) < 1 { + dst.Status.ActiveEndpoint = nil + } else { + dst.Status.ActiveEndpoint = &Endpoint{ + NodeName: src.Status.ActiveEndpoints[0].NodeName, + PublicIP: src.Status.ActiveEndpoints[0].PublicIP, + UnderNAT: src.Status.ActiveEndpoints[0].UnderNAT, + Config: src.Status.ActiveEndpoints[0].Config, + } + } + klog.Infof("convert from v1beta1 to v1alpha1 for %s", dst.Name) + return nil +} diff --git a/pkg/apis/raven/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/raven/v1alpha1/zz_generated.deepcopy.go index d2d42342df3..c736ea956f0 100644 --- a/pkg/apis/raven/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/raven/v1alpha1/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ limitations under the License. package v1alpha1 import ( - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/pkg/apis/raven/v1beta1/default.go b/pkg/apis/raven/v1beta1/default.go new file mode 100644 index 00000000000..cbbea9a915f --- /dev/null +++ b/pkg/apis/raven/v1beta1/default.go @@ -0,0 +1,43 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/openyurtio/openyurt/pkg/apis/raven" +) + +// SetDefaultsGateway set default values for Gateway. +func SetDefaultsGateway(obj *Gateway) { + // Set default value for Gateway + obj.Spec.NodeSelector = &metav1.LabelSelector{ + MatchLabels: map[string]string{ + raven.LabelCurrentGateway: obj.Name, + }, + } + for idx, val := range obj.Spec.Endpoints { + if val.Port == 0 { + switch val.Type { + case Proxy: + obj.Spec.Endpoints[idx].Port = DefaultProxyServerExposedPort + case Tunnel: + obj.Spec.Endpoints[idx].Port = DefaultTunnelServerExposedPort + } + } + } +} diff --git a/pkg/apis/raven/v1beta1/gateway_conversion.go b/pkg/apis/raven/v1beta1/gateway_conversion.go new file mode 100644 index 00000000000..4a87604e040 --- /dev/null +++ b/pkg/apis/raven/v1beta1/gateway_conversion.go @@ -0,0 +1,48 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +/* +Implementing the hub method is pretty easy -- we just have to add an empty +method called Hub() to serve as a +[marker](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/conversion?tab=doc#Hub). +*/ + +// NOTE !!!!!! @kadisi +// If this version is storageversion, you only need to uncommand this method + +// Hub marks this type as a conversion hub. +//func (*Gateway) Hub() {} + +// NOTE !!!!!!! @kadisi +// If this version is not storageversion, you need to implement the ConvertTo and ConvertFrom methods + +// need import "sigs.k8s.io/controller-runtime/pkg/conversion" +//func (src *Gateway) ConvertTo(dstRaw conversion.Hub) error { +// return nil +//} + +// NOTE !!!!!!! @kadisi +// If this version is not storageversion, you need to implement the ConvertTo and ConvertFrom methods + +// need import "sigs.k8s.io/controller-runtime/pkg/conversion" +//func (dst *Gateway) ConvertFrom(srcRaw conversion.Hub) error { +// return nil +//} + +// Hub marks this type as a conversion hub. +func (*Gateway) Hub() {} diff --git a/pkg/apis/raven/v1beta1/gateway_types.go b/pkg/apis/raven/v1beta1/gateway_types.go new file mode 100644 index 00000000000..a8d433c0a46 --- /dev/null +++ b/pkg/apis/raven/v1beta1/gateway_types.go @@ -0,0 +1,139 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// Event reason. +const ( + // EventActiveEndpointElected is the event indicating a new active endpoint is elected. + EventActiveEndpointElected = "ActiveEndpointElected" + // EventActiveEndpointLost is the event indicating the active endpoint is lost. + EventActiveEndpointLost = "ActiveEndpointLost" +) + +const ( + ExposeTypePublicIP = "PublicIP" + ExposeTypeLoadBalancer = "LoadBalancer" +) + +const ( + Proxy = "proxy" + Tunnel = "tunnel" + + DefaultProxyServerSecurePort = 10263 + DefaultProxyServerInsecurePort = 10264 + DefaultProxyServerExposedPort = 10262 + DefaultTunnelServerExposedPort = 4500 +) + +// ProxyConfiguration is the configuration for raven l7 proxy +type ProxyConfiguration struct { + // Replicas is the number of gateway active endpoints that enabled proxy + Replicas int `json:"Replicas"` + // ProxyHTTPPort is the proxy http port of the cross-domain request + ProxyHTTPPort string `json:"proxyHTTPPort,omitempty"` + // ProxyHTTPSPort is the proxy https port of the cross-domain request + ProxyHTTPSPort string `json:"proxyHTTPSPort,omitempty"` +} + +// TunnelConfiguration is the configuration for raven l3 tunnel +type TunnelConfiguration struct { + // Replicas is the number of gateway active endpoints that enabled tunnel + Replicas int `json:"Replicas"` +} + +// GatewaySpec defines the desired state of Gateway +type GatewaySpec struct { + // NodeSelector is a label query over nodes that managed by the gateway. + // The nodes in the same gateway should share same layer 3 network. + NodeSelector *metav1.LabelSelector `json:"nodeSelector,omitempty"` + // ProxyConfig determine the l7 proxy configuration + ProxyConfig ProxyConfiguration `json:"proxyConfig,omitempty"` + // TunnelConfig determine the l3 tunnel configuration + TunnelConfig TunnelConfiguration `json:"tunnelConfig,omitempty"` + // Endpoints are a list of available Endpoint. + Endpoints []Endpoint `json:"endpoints,omitempty"` + // ExposeType determines how the Gateway is exposed. + ExposeType string `json:"exposeType,omitempty"` +} + +// Endpoint stores all essential data for establishing the VPN tunnel and Proxy +type Endpoint struct { + // NodeName is the Node hosting this endpoint. + NodeName string `json:"nodeName"` + // Type is the service type of the node, proxy or tunnel + Type string `json:"type"` + // Port is the exposed port of the node + Port int `json:"port,omitempty"` + // UnderNAT indicates whether node is under NAT + UnderNAT bool `json:"underNAT,omitempty"` + // PublicIP is the exposed IP of the node + PublicIP string `json:"publicIP,omitempty"` + // Config is a map to record config for the raven agent of node + Config map[string]string `json:"config,omitempty"` +} + +// NodeInfo stores information of node managed by Gateway. +type NodeInfo struct { + // NodeName is the Node host name. + NodeName string `json:"nodeName"` + // PrivateIP is the node private ip address + PrivateIP string `json:"privateIP"` + // Subnets is the pod ip range of the node + Subnets []string `json:"subnets"` +} + +// GatewayStatus defines the observed state of Gateway +type GatewayStatus struct { + // Nodes contains all information of nodes managed by Gateway. + Nodes []NodeInfo `json:"nodes,omitempty"` + // ActiveEndpoints is the reference of the active endpoint. + ActiveEndpoints []*Endpoint `json:"activeEndpoints,omitempty"` +} + +// +genclient +// +k8s:openapi-gen=true +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster,path=gateways,shortName=gw,categories=all +// +kubebuilder:storageversion + +// Gateway is the Schema for the gateways API +type Gateway struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec GatewaySpec `json:"spec,omitempty"` + Status GatewayStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// GatewayList contains a list of Gateway +type GatewayList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Gateway `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Gateway{}, &GatewayList{}) +} diff --git a/pkg/apis/raven/v1beta1/groupversion_info.go b/pkg/apis/raven/v1beta1/groupversion_info.go new file mode 100644 index 00000000000..e35d10626d0 --- /dev/null +++ b/pkg/apis/raven/v1beta1/groupversion_info.go @@ -0,0 +1,44 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +// Package v1beta1 contains API Schema definitions for the raven v1beta1API group +// +kubebuilder:object:generate=true +// +groupName=raven.openyurt.io + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "raven.openyurt.io", Version: "v1beta1"} + + SchemeGroupVersion = GroupVersion + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) + +// Resource is required by pkg/client/listers/... +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} diff --git a/pkg/apis/raven/v1beta1/zz_generated.deepcopy.go b/pkg/apis/raven/v1beta1/zz_generated.deepcopy.go new file mode 100644 index 00000000000..99e1cbb33f2 --- /dev/null +++ b/pkg/apis/raven/v1beta1/zz_generated.deepcopy.go @@ -0,0 +1,220 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1beta1 + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Endpoint) DeepCopyInto(out *Endpoint) { + *out = *in + if in.Config != nil { + in, out := &in.Config, &out.Config + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Endpoint. +func (in *Endpoint) DeepCopy() *Endpoint { + if in == nil { + return nil + } + out := new(Endpoint) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Gateway) DeepCopyInto(out *Gateway) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Gateway. +func (in *Gateway) DeepCopy() *Gateway { + if in == nil { + return nil + } + out := new(Gateway) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Gateway) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GatewayList) DeepCopyInto(out *GatewayList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Gateway, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayList. +func (in *GatewayList) DeepCopy() *GatewayList { + if in == nil { + return nil + } + out := new(GatewayList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GatewayList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GatewaySpec) DeepCopyInto(out *GatewaySpec) { + *out = *in + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + out.ProxyConfig = in.ProxyConfig + out.TunnelConfig = in.TunnelConfig + if in.Endpoints != nil { + in, out := &in.Endpoints, &out.Endpoints + *out = make([]Endpoint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewaySpec. +func (in *GatewaySpec) DeepCopy() *GatewaySpec { + if in == nil { + return nil + } + out := new(GatewaySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GatewayStatus) DeepCopyInto(out *GatewayStatus) { + *out = *in + if in.Nodes != nil { + in, out := &in.Nodes, &out.Nodes + *out = make([]NodeInfo, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ActiveEndpoints != nil { + in, out := &in.ActiveEndpoints, &out.ActiveEndpoints + *out = make([]*Endpoint, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Endpoint) + (*in).DeepCopyInto(*out) + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayStatus. +func (in *GatewayStatus) DeepCopy() *GatewayStatus { + if in == nil { + return nil + } + out := new(GatewayStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeInfo) DeepCopyInto(out *NodeInfo) { + *out = *in + if in.Subnets != nil { + in, out := &in.Subnets, &out.Subnets + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeInfo. +func (in *NodeInfo) DeepCopy() *NodeInfo { + if in == nil { + return nil + } + out := new(NodeInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxyConfiguration) DeepCopyInto(out *ProxyConfiguration) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyConfiguration. +func (in *ProxyConfiguration) DeepCopy() *ProxyConfiguration { + if in == nil { + return nil + } + out := new(ProxyConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TunnelConfiguration) DeepCopyInto(out *TunnelConfiguration) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TunnelConfiguration. +func (in *TunnelConfiguration) DeepCopy() *TunnelConfiguration { + if in == nil { + return nil + } + out := new(TunnelConfiguration) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/controller/raven/common.go b/pkg/controller/raven/common.go index e1af9b1184d..f9325510320 100644 --- a/pkg/controller/raven/common.go +++ b/pkg/controller/raven/common.go @@ -21,5 +21,5 @@ var ( ) const ( - ControllerName = "ravenl3" + ControllerName = "gateway" ) diff --git a/pkg/webhook/gateway/v1alpha1/gateway_default.go b/pkg/webhook/gateway/v1alpha1/gateway_default.go index d7766e1e856..8a55524a79f 100644 --- a/pkg/webhook/gateway/v1alpha1/gateway_default.go +++ b/pkg/webhook/gateway/v1alpha1/gateway_default.go @@ -28,12 +28,12 @@ import ( // Default satisfies the defaulting webhook interface. func (webhook *GatewayHandler) Default(ctx context.Context, obj runtime.Object) error { - np, ok := obj.(*v1alpha1.Gateway) + gw, ok := obj.(*v1alpha1.Gateway) if !ok { return apierrors.NewBadRequest(fmt.Sprintf("expected a Gateway but got a %T", obj)) } - v1alpha1.SetDefaultsGateway(np) + v1alpha1.SetDefaultsGateway(gw) return nil } diff --git a/pkg/webhook/gateway/v1alpha1/gateway_handler.go b/pkg/webhook/gateway/v1alpha1/gateway_handler.go index 61cecb7c0e3..97573b424a4 100644 --- a/pkg/webhook/gateway/v1alpha1/gateway_handler.go +++ b/pkg/webhook/gateway/v1alpha1/gateway_handler.go @@ -44,9 +44,6 @@ func (webhook *GatewayHandler) SetupWebhookWithManager(mgr ctrl.Manager) (string Complete() } -// +kubebuilder:webhook:path=/validate-raven-openyurt-io-v1alpha1-gateway,mutating=false,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=raven.openyurt.io,resources=gateways,verbs=create;update,versions=v1alpha1,name=validate.raven.v1alpha1.gateway.openyurt.io -// +kubebuilder:webhook:path=/mutate-raven-openyurt-io-v1alpha1-gateway,mutating=true,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=raven.openyurt.io,resources=gateways,verbs=create;update,versions=v1alpha1,name=mutate.raven.v1alpha1.gateway.openyurt.io - // Cluster implements a validating and defaulting webhook for Cluster. type GatewayHandler struct { Client client.Client diff --git a/pkg/webhook/gateway/v1beta1/gateway_default.go b/pkg/webhook/gateway/v1beta1/gateway_default.go new file mode 100644 index 00000000000..c966416956c --- /dev/null +++ b/pkg/webhook/gateway/v1beta1/gateway_default.go @@ -0,0 +1,39 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" +) + +// Default satisfies the defaulting webhook interface. +func (webhook *GatewayHandler) Default(ctx context.Context, obj runtime.Object) error { + gw, ok := obj.(*v1beta1.Gateway) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a Gateway but got a %T", obj)) + } + + v1beta1.SetDefaultsGateway(gw) + + return nil +} diff --git a/pkg/webhook/gateway/v1beta1/gateway_handler.go b/pkg/webhook/gateway/v1beta1/gateway_handler.go new file mode 100644 index 00000000000..1cd33b5540a --- /dev/null +++ b/pkg/webhook/gateway/v1beta1/gateway_handler.go @@ -0,0 +1,56 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" + "github.com/openyurtio/openyurt/pkg/webhook/util" +) + +// SetupWebhookWithManager sets up Cluster webhooks. mutate path, validatepath, error +func (webhook *GatewayHandler) SetupWebhookWithManager(mgr ctrl.Manager) (string, string, error) { + // init + webhook.Client = mgr.GetClient() + + gvk, err := apiutil.GVKForObject(&v1beta1.Gateway{}, mgr.GetScheme()) + if err != nil { + return "", "", err + } + return util.GenerateMutatePath(gvk), + util.GenerateValidatePath(gvk), + ctrl.NewWebhookManagedBy(mgr). + For(&v1beta1.Gateway{}). + WithDefaulter(webhook). + WithValidator(webhook). + Complete() +} + +// +kubebuilder:webhook:path=/validate-raven-openyurt-io-v1beta1-gateway,mutating=false,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=raven.openyurt.io,resources=gateways,verbs=create;update,versions=v1beta1,name=validate.gateway.v1beta1.raven.openyurt.io +// +kubebuilder:webhook:path=/mutate-raven-openyurt-io-v1beta1-gateway,mutating=true,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=raven.openyurt.io,resources=gateways,verbs=create;update,versions=v1beta1,name=mutate.gateway.v1beta1.raven.openyurt.io + +// Cluster implements a validating and defaulting webhook for Cluster. +type GatewayHandler struct { + Client client.Client +} + +var _ webhook.CustomDefaulter = &GatewayHandler{} +var _ webhook.CustomValidator = &GatewayHandler{} diff --git a/pkg/webhook/gateway/v1beta1/gateway_validation.go b/pkg/webhook/gateway/v1beta1/gateway_validation.go new file mode 100644 index 00000000000..6c9008cb53c --- /dev/null +++ b/pkg/webhook/gateway/v1beta1/gateway_validation.go @@ -0,0 +1,128 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + "fmt" + "net" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" +) + +// ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. +func (webhook *GatewayHandler) ValidateCreate(ctx context.Context, obj runtime.Object) error { + gw, ok := obj.(*v1beta1.Gateway) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a Gateway but got a %T", obj)) + } + + return validate(gw) +} + +// ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type. +func (webhook *GatewayHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error { + newGw, ok := newObj.(*v1beta1.Gateway) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a Gateway but got a %T", newObj)) + } + oldGw, ok := oldObj.(*v1beta1.Gateway) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a Gateway} but got a %T", oldObj)) + } + + if newGw.GetName() != oldGw.GetName() { + return apierrors.NewBadRequest(fmt.Sprintf("gateway name can not change")) + } + if err := validate(newGw); err != nil { + return err + } + + return nil +} + +// ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type. +func (webhook *GatewayHandler) ValidateDelete(_ context.Context, obj runtime.Object) error { + return nil +} + +func validate(g *v1beta1.Gateway) error { + var errList field.ErrorList + + if g.Spec.ExposeType != "" { + if g.Spec.ExposeType != v1beta1.ExposeTypeLoadBalancer && g.Spec.ExposeType != v1beta1.ExposeTypePublicIP { + fldPath := field.NewPath("spec").Child("exposeType") + errList = append(errList, field.Invalid(fldPath, g.Spec.ExposeType, "the 'exposeType' field is irregularity")) + } + if g.Spec.ExposeType == v1beta1.ExposeTypeLoadBalancer || g.Spec.ExposeType == v1beta1.ExposeTypePublicIP { + for i, ep := range g.Spec.Endpoints { + if ep.UnderNAT { + fldPath := field.NewPath("spec").Child(fmt.Sprintf("endpoints[%d]", i)).Child("underNAT") + errList = append(errList, field.Invalid(fldPath, ep.UnderNAT, fmt.Sprintf("the 'underNAT' field for exposed gateway %s/%s must be false", g.Namespace, g.Name))) + } + } + } + } + + if len(g.Spec.Endpoints) != 0 { + underNAT := g.Spec.Endpoints[0].UnderNAT + for i, ep := range g.Spec.Endpoints { + if ep.UnderNAT != underNAT { + fldPath := field.NewPath("spec").Child(fmt.Sprintf("endpoints[%d]", i)).Child("underNAT") + errList = append(errList, field.Invalid(fldPath, ep.UnderNAT, "the 'underNAT' field in endpoints must be the same")) + } + if ep.PublicIP != "" { + if err := validateIP(ep.PublicIP); err != nil { + fldPath := field.NewPath("spec").Child(fmt.Sprintf("endpoints[%d]", i)).Child("publicIP") + errList = append(errList, field.Invalid(fldPath, ep.PublicIP, "the 'publicIP' field must be a validate IP address")) + } + } + if ep.Type != v1beta1.Tunnel && ep.Type != v1beta1.Proxy { + fldPath := field.NewPath("spec").Child(fmt.Sprintf("endpoints[%d]", i)).Child("type") + errList = append(errList, field.Invalid(fldPath, ep.Type, fmt.Sprintf("the 'type' field must be set %s or %s ", v1beta1.Tunnel, v1beta1.Proxy))) + } + if len(ep.NodeName) == 0 { + fldPath := field.NewPath("spec").Child(fmt.Sprintf("endpoints[%d]", i)).Child("nodeName") + errList = append(errList, field.Invalid(fldPath, ep.NodeName, "the 'nodeName' field must not be empty")) + } + } + } + + if errList != nil { + return apierrors.NewInvalid( + schema.GroupKind{Group: v1beta1.SchemeGroupVersion.Group, Kind: g.Kind}, + g.Name, errList) + } + + klog.Infof("Validate Gateway %s successfully ...", klog.KObj(g)) + + return nil +} + +func validateIP(ip string) error { + s := net.ParseIP(ip) + if s.To4() != nil || s.To16() != nil { + return nil + } + return fmt.Errorf("invalid ip address: %s", ip) +} diff --git a/pkg/webhook/server.go b/pkg/webhook/server.go index c22ffe2ef49..60278a30a2d 100644 --- a/pkg/webhook/server.go +++ b/pkg/webhook/server.go @@ -33,7 +33,7 @@ import ( "github.com/openyurtio/openyurt/pkg/controller/yurtappdaemon" "github.com/openyurtio/openyurt/pkg/controller/yurtappset" "github.com/openyurtio/openyurt/pkg/controller/yurtstaticset" - v1alpha1gateway "github.com/openyurtio/openyurt/pkg/webhook/gateway/v1alpha1" + v1beta1gateway "github.com/openyurtio/openyurt/pkg/webhook/gateway/v1beta1" v1node "github.com/openyurtio/openyurt/pkg/webhook/node/v1" v1alpha1nodepool "github.com/openyurtio/openyurt/pkg/webhook/nodepool/v1alpha1" v1beta1nodepool "github.com/openyurtio/openyurt/pkg/webhook/nodepool/v1beta1" @@ -73,7 +73,7 @@ func addControllerWebhook(name string, handler SetupWebhookWithManager) { } func init() { - addControllerWebhook(raven.ControllerName, &v1alpha1gateway.GatewayHandler{}) + addControllerWebhook(raven.ControllerName, &v1beta1gateway.GatewayHandler{}) addControllerWebhook(nodepool.ControllerName, &v1alpha1nodepool.NodePoolHandler{}) addControllerWebhook(nodepool.ControllerName, &v1beta1nodepool.NodePoolHandler{}) addControllerWebhook(yurtstaticset.ControllerName, &v1alpha1yurtstaticset.YurtStaticSetHandler{}) From 2ed7f0f8f699b4aa2011e333882ae9ec9337e050 Mon Sep 17 00:00:00 2001 From: Abyss <45425302+wangxye@users.noreply.github.com> Date: Mon, 14 Aug 2023 10:36:23 +0800 Subject: [PATCH 06/21] reclaim device, deviceprofile and deviceservice (#1647) Signed-off-by: wangxye <1031989637@qq.com> --- cmd/yurt-iot-dock/app/core.go | 49 ++++++++++++++++++- .../controllers/device_controller.go | 25 ++++++++++ .../controllers/deviceprofile_controller.go | 25 ++++++++++ .../controllers/deviceservice_controller.go | 25 ++++++++++ 4 files changed, 123 insertions(+), 1 deletion(-) diff --git a/cmd/yurt-iot-dock/app/core.go b/cmd/yurt-iot-dock/app/core.go index 39e30bcaf30..3d4aae34219 100644 --- a/cmd/yurt-iot-dock/app/core.go +++ b/cmd/yurt-iot-dock/app/core.go @@ -20,6 +20,8 @@ import ( "context" "fmt" "os" + "os/signal" + "syscall" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -30,6 +32,7 @@ import ( "k8s.io/klog/v2" "k8s.io/klog/v2/klogr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "github.com/openyurtio/openyurt/cmd/yurt-iot-dock/app/options" @@ -181,12 +184,56 @@ func Run(opts *options.YurtIoTDockOptions, stopCh <-chan struct{}) { } setupLog.Info("[run controllers] Starting manager, acting on " + fmt.Sprintf("[NodePool: %s, Namespace: %s]", opts.Nodepool, opts.Namespace)) - if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + if err := mgr.Start(SetupSignalHandler(mgr.GetClient(), opts)); err != nil { setupLog.Error(err, "failed to running manager") os.Exit(1) } } +func deleteCRsOnControllerShutdown(ctx context.Context, cli client.Client, opts *options.YurtIoTDockOptions) error { + setupLog.Info("[deleteCRsOnControllerShutdown] start delete device crd") + if err := controllers.DeleteDevicesOnControllerShutdown(ctx, cli, opts); err != nil { + setupLog.Error(err, "failed to shutdown device cr") + return err + } + + setupLog.Info("[deleteCRsOnControllerShutdown] start delete deviceprofile crd") + if err := controllers.DeleteDeviceProfilesOnControllerShutdown(ctx, cli, opts); err != nil { + setupLog.Error(err, "failed to shutdown deviceprofile cr") + return err + } + + setupLog.Info("[deleteCRsOnControllerShutdown] start delete deviceservice crd") + if err := controllers.DeleteDeviceServicesOnControllerShutdown(ctx, cli, opts); err != nil { + setupLog.Error(err, "failed to shutdown deviceservice cr") + return err + } + + return nil +} + +var onlyOneSignalHandler = make(chan struct{}) +var shutdownSignals = []os.Signal{syscall.SIGTERM} + +func SetupSignalHandler(client client.Client, opts *options.YurtIoTDockOptions) context.Context { + close(onlyOneSignalHandler) // panics when called twice + + ctx, cancel := context.WithCancel(context.Background()) + setupLog.Info("[SetupSignalHandler] shutdown controller with crd") + c := make(chan os.Signal, 2) + signal.Notify(c, shutdownSignals...) + go func() { + <-c + setupLog.Info("[SetupSignalHandler] shutdown signal concur") + deleteCRsOnControllerShutdown(ctx, client, opts) + cancel() + <-c + os.Exit(1) // second signal. Exit directly. + }() + + return ctx +} + func preflightCheck(mgr ctrl.Manager, opts *options.YurtIoTDockOptions) error { client, err := kubernetes.NewForConfig(mgr.GetConfig()) if err != nil { diff --git a/pkg/yurtiotdock/controllers/device_controller.go b/pkg/yurtiotdock/controllers/device_controller.go index dd61d5a6a96..662feb3632d 100644 --- a/pkg/yurtiotdock/controllers/device_controller.go +++ b/pkg/yurtiotdock/controllers/device_controller.go @@ -29,6 +29,7 @@ import ( "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/openyurtio/openyurt/cmd/yurt-iot-dock/app/options" iotv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" @@ -293,3 +294,27 @@ func (r *DeviceReconciler) reconcileDeviceProperties(d *iotv1alpha1.Device, devi } return newDeviceStatus, failedPropertyNames } + +func DeleteDevicesOnControllerShutdown(ctx context.Context, cli client.Client, opts *options.YurtIoTDockOptions) error { + var deviceList iotv1alpha1.DeviceList + if err := cli.List(ctx, &deviceList, client.InNamespace(opts.Namespace)); err != nil { + return err + } + klog.V(4).Infof("DeviceList, successfully get the list") + + for _, device := range deviceList.Items { + controllerutil.RemoveFinalizer(&device, iotv1alpha1.DeviceFinalizer) + if err := cli.Update(ctx, &device); err != nil { + klog.Errorf("DeviceName: %s, update device err:%v", device.GetName(), err) + continue + } + + if err := cli.Delete(ctx, &device); err != nil { + klog.Errorf("DeviceName: %s, update device err:%v", device.GetName(), err) + continue + } + } + klog.V(4).Infof("DeviceList, successfully delete the list") + + return nil +} diff --git a/pkg/yurtiotdock/controllers/deviceprofile_controller.go b/pkg/yurtiotdock/controllers/deviceprofile_controller.go index d174cc2ebb6..459f39156b3 100644 --- a/pkg/yurtiotdock/controllers/deviceprofile_controller.go +++ b/pkg/yurtiotdock/controllers/deviceprofile_controller.go @@ -27,6 +27,7 @@ import ( "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/openyurtio/openyurt/cmd/yurt-iot-dock/app/options" iotv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" @@ -163,3 +164,27 @@ func (r *DeviceProfileReconciler) reconcileCreateDeviceProfile(ctx context.Conte dp.Status.Synced = true return r.Status().Update(ctx, dp) } + +func DeleteDeviceProfilesOnControllerShutdown(ctx context.Context, cli client.Client, opts *options.YurtIoTDockOptions) error { + var deviceProfileList iotv1alpha1.DeviceProfileList + if err := cli.List(ctx, &deviceProfileList, client.InNamespace(opts.Namespace)); err != nil { + return err + } + klog.V(4).Infof("DeviceProfileList, successfully get the list") + + for _, deviceProfile := range deviceProfileList.Items { + controllerutil.RemoveFinalizer(&deviceProfile, iotv1alpha1.DeviceProfileFinalizer) + if err := cli.Update(ctx, &deviceProfile); err != nil { + klog.Errorf("deviceProfileName: %s, update deviceProfile err:%v", deviceProfile.GetName(), err) + continue + } + + if err := cli.Delete(ctx, &deviceProfile); err != nil { + klog.Errorf("deviceProfileName: %s, update deviceProfile err:%v", deviceProfile.GetName(), err) + continue + } + } + klog.V(4).Infof("DeviceProfileList, successfully delete the list") + + return nil +} diff --git a/pkg/yurtiotdock/controllers/deviceservice_controller.go b/pkg/yurtiotdock/controllers/deviceservice_controller.go index 2822558b5f0..b924d789c5c 100644 --- a/pkg/yurtiotdock/controllers/deviceservice_controller.go +++ b/pkg/yurtiotdock/controllers/deviceservice_controller.go @@ -28,6 +28,7 @@ import ( "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/openyurtio/openyurt/cmd/yurt-iot-dock/app/options" iotv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" @@ -218,3 +219,27 @@ func (r *DeviceServiceReconciler) reconcileUpdateDeviceService(ctx context.Conte util.SetDeviceServiceCondition(deviceServiceStatus, util.NewDeviceServiceCondition(iotv1alpha1.DeviceServiceManagingCondition, corev1.ConditionTrue, "", "")) return nil } + +func DeleteDeviceServicesOnControllerShutdown(ctx context.Context, cli client.Client, opts *options.YurtIoTDockOptions) error { + var deviceServiceList iotv1alpha1.DeviceServiceList + if err := cli.List(ctx, &deviceServiceList, client.InNamespace(opts.Namespace)); err != nil { + return err + } + klog.V(4).Infof("DeviceServiceList, successfully get the list") + + for _, deviceService := range deviceServiceList.Items { + controllerutil.RemoveFinalizer(&deviceService, iotv1alpha1.DeviceServiceFinalizer) + if err := cli.Update(ctx, &deviceService); err != nil { + klog.Errorf("DeviceServiceName: %s, update deviceservice err:%v", deviceService.GetName(), err) + continue + } + + if err := cli.Delete(ctx, &deviceService); err != nil { + klog.Errorf("DeviceServiceName: %s, update deviceservice err:%v", deviceService.GetName(), err) + continue + } + } + klog.V(4).Infof("DeviceServiceList, successfully get the list") + + return nil +} From 3dbb153701f9de34bdb2b276f9196e8beb2d48ab Mon Sep 17 00:00:00 2001 From: rambohe Date: Tue, 15 Aug 2023 10:26:23 +0800 Subject: [PATCH 07/21] use dynamicinformerfactory to get nodepool and remove yurt-app-manager-api dependencies (#1652) --- cmd/yurthub/app/config/config.go | 100 +-- cmd/yurthub/app/start.go | 2 +- go.mod | 1 - go.sum | 9 - .../apps/well_known_labels_annotations.go | 1 + pkg/util/kubeconfig/kubeconfig.go | 17 - pkg/util/kubeconfig/kubeconfig_test.go | 2 - .../kubeadm/app/util/apiclient/idempotency.go | 32 +- pkg/yurtadm/cmd/join/join.go | 12 +- pkg/yurthub/filter/initializer/initializer.go | 18 +- .../filter/initializer/initializer_test.go | 21 +- pkg/yurthub/filter/manager/manager.go | 10 +- pkg/yurthub/filter/manager/manager_test.go | 19 +- .../filter/nodeportisolation/filter.go | 2 +- pkg/yurthub/filter/servicetopology/filter.go | 42 +- .../filter/servicetopology/filter_test.go | 591 +++++++++--------- 16 files changed, 423 insertions(+), 456 deletions(-) diff --git a/cmd/yurthub/app/config/config.go b/cmd/yurthub/app/config/config.go index 06529a69fc5..22255987b70 100644 --- a/cmd/yurthub/app/config/config.go +++ b/cmd/yurthub/app/config/config.go @@ -28,15 +28,15 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/apimachinery/pkg/watch" apiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/dynamiccertificates" apiserveroptions "k8s.io/apiserver/pkg/server/options" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/dynamic/dynamicinformer" "k8s.io/client-go/informers" coreinformers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - core "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" componentbaseconfig "k8s.io/component-base/config" @@ -47,18 +47,12 @@ import ( "github.com/openyurtio/openyurt/pkg/yurthub/cachemanager" "github.com/openyurtio/openyurt/pkg/yurthub/certificate" certificatemgr "github.com/openyurtio/openyurt/pkg/yurthub/certificate/manager" - "github.com/openyurtio/openyurt/pkg/yurthub/filter" "github.com/openyurtio/openyurt/pkg/yurthub/filter/manager" "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/meta" "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/serializer" "github.com/openyurtio/openyurt/pkg/yurthub/network" "github.com/openyurtio/openyurt/pkg/yurthub/storage/disk" "github.com/openyurtio/openyurt/pkg/yurthub/util" - yurtcorev1alpha1 "github.com/openyurtio/yurt-app-manager-api/pkg/yurtappmanager/apis/apps/v1alpha1" - yurtclientset "github.com/openyurtio/yurt-app-manager-api/pkg/yurtappmanager/client/clientset/versioned" - "github.com/openyurtio/yurt-app-manager-api/pkg/yurtappmanager/client/clientset/versioned/fake" - yurtinformers "github.com/openyurtio/yurt-app-manager-api/pkg/yurtappmanager/client/informers/externalversions" - yurtv1alpha1 "github.com/openyurtio/yurt-app-manager-api/pkg/yurtappmanager/client/informers/externalversions/apps/v1alpha1" ) // YurtHubConfiguration represents configuration of yurthub @@ -77,7 +71,7 @@ type YurtHubConfiguration struct { SerializerManager *serializer.SerializerManager RESTMapperManager *meta.RESTMapperManager SharedFactory informers.SharedInformerFactory - YurtSharedFactory yurtinformers.SharedInformerFactory + NodePoolInformerFactory dynamicinformer.DynamicSharedInformerFactory WorkingMode util.WorkingMode KubeletHealthGracePeriod time.Duration FilterManager *manager.Manager @@ -132,13 +126,13 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) { } workingMode := util.WorkingMode(options.WorkingMode) - proxiedClient, sharedFactory, yurtSharedFactory, err := createClientAndSharedInformers(fmt.Sprintf("http://%s:%d", options.YurtHubProxyHost, options.YurtHubProxyPort), options.EnableNodePool) + proxiedClient, sharedFactory, dynamicSharedFactory, err := createClientAndSharedInformers(fmt.Sprintf("http://%s:%d", options.YurtHubProxyHost, options.YurtHubProxyPort), options.NodePoolName) if err != nil { return nil, err } tenantNs := util.ParseTenantNsFromOrgs(options.YurtHubCertOrganizations) - registerInformers(options, sharedFactory, yurtSharedFactory, workingMode, tenantNs) - filterManager, err := manager.NewFilterManager(options, sharedFactory, yurtSharedFactory, proxiedClient, serializerManager, us[0].Host) + registerInformers(options, sharedFactory, workingMode, tenantNs) + filterManager, err := manager.NewFilterManager(options, sharedFactory, dynamicSharedFactory, proxiedClient, serializerManager, us[0].Host) if err != nil { klog.Errorf("could not create filter manager, %v", err) return nil, err @@ -160,7 +154,7 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) { SerializerManager: serializerManager, RESTMapperManager: restMapperManager, SharedFactory: sharedFactory, - YurtSharedFactory: yurtSharedFactory, + NodePoolInformerFactory: dynamicSharedFactory, KubeletHealthGracePeriod: options.KubeletHealthGracePeriod, FilterManager: filterManager, MinRequestTimeout: options.MinRequestTimeout, @@ -241,9 +235,8 @@ func parseRemoteServers(serverAddr string) ([]*url.URL, error) { } // createClientAndSharedInformers create kubeclient and sharedInformers from the given proxyAddr. -func createClientAndSharedInformers(proxyAddr string, enableNodePool bool) (kubernetes.Interface, informers.SharedInformerFactory, yurtinformers.SharedInformerFactory, error) { +func createClientAndSharedInformers(proxyAddr string, nodePoolName string) (kubernetes.Interface, informers.SharedInformerFactory, dynamicinformer.DynamicSharedInformerFactory, error) { var kubeConfig *rest.Config - var yurtClient yurtclientset.Interface var err error kubeConfig, err = clientcmd.BuildConfigFromFlags(proxyAddr, "") if err != nil { @@ -255,53 +248,27 @@ func createClientAndSharedInformers(proxyAddr string, enableNodePool bool) (kube return nil, nil, nil, err } - fakeYurtClient := &fake.Clientset{} - fakeWatch := watch.NewFake() - fakeYurtClient.AddWatchReactor("nodepools", core.DefaultWatchReactor(fakeWatch, nil)) - // init yurtClient by fake client - yurtClient = fakeYurtClient - if enableNodePool { - yurtClient, err = yurtclientset.NewForConfig(kubeConfig) - if err != nil { - return nil, nil, nil, err - } + dynamicClient, err := dynamic.NewForConfig(kubeConfig) + if err != nil { + return nil, nil, nil, err } - return client, informers.NewSharedInformerFactory(client, 24*time.Hour), - yurtinformers.NewSharedInformerFactory(yurtClient, 24*time.Hour), nil + dynamicInformerFactory := dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, 24*time.Hour) + if len(nodePoolName) != 0 { + dynamicInformerFactory = dynamicinformer.NewFilteredDynamicSharedInformerFactory(dynamicClient, 24*time.Hour, metav1.NamespaceAll, func(options *metav1.ListOptions) { + options.FieldSelector = fields.Set{"metadata.name": nodePoolName}.String() + }) + } + + return client, informers.NewSharedInformerFactory(client, 24*time.Hour), dynamicInformerFactory, nil } -// registerInformers reconstruct node/nodePool/configmap informers +// registerInformers reconstruct configmap/secret/pod informers func registerInformers(options *options.YurtHubOptions, informerFactory informers.SharedInformerFactory, - yurtInformerFactory yurtinformers.SharedInformerFactory, workingMode util.WorkingMode, tenantNs string) { - // skip construct node/nodePool informers if service topology filter disabled - serviceTopologyFilterEnabled := isServiceTopologyFilterEnabled(options) - if serviceTopologyFilterEnabled { - if workingMode == util.WorkingModeCloud { - newNodeInformer := func(client kubernetes.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { - tweakListOptions := func(ops *metav1.ListOptions) { - ops.FieldSelector = fields.Set{"metadata.name": options.NodeName}.String() - } - return coreinformers.NewFilteredNodeInformer(client, resyncPeriod, nil, tweakListOptions) - } - informerFactory.InformerFor(&corev1.Node{}, newNodeInformer) - } - - if len(options.NodePoolName) != 0 { - newNodePoolInformer := func(client yurtclientset.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { - tweakListOptions := func(ops *metav1.ListOptions) { - ops.FieldSelector = fields.Set{"metadata.name": options.NodePoolName}.String() - } - return yurtv1alpha1.NewFilteredNodePoolInformer(client, resyncPeriod, nil, tweakListOptions) - } - - yurtInformerFactory.InformerFor(&yurtcorev1alpha1.NodePool{}, newNodePoolInformer) - } - } - + // configmap informer is used by Yurthub filter approver newConfigmapInformer := func(client kubernetes.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { tweakListOptions := func(options *metav1.ListOptions) { options.FieldSelector = fields.Set{"metadata.name": util.YurthubConfigMapName}.String() @@ -310,6 +277,7 @@ func registerInformers(options *options.YurtHubOptions, } informerFactory.InformerFor(&corev1.ConfigMap{}, newConfigmapInformer) + // secret informer is used by Tenant manager, this feature is not enabled in general. if tenantNs != "" { newSecretInformer := func(client kubernetes.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { return coreinformers.NewFilteredSecretInformer(client, tenantNs, resyncPeriod, nil, nil) @@ -317,6 +285,7 @@ func registerInformers(options *options.YurtHubOptions, informerFactory.InformerFor(&corev1.Secret{}, newSecretInformer) } + // pod informer is used by OTA updater on cloud working mode if workingMode == util.WorkingModeCloud { newPodInformer := func(client kubernetes.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { listOptions := func(ops *metav1.ListOptions) { @@ -328,29 +297,6 @@ func registerInformers(options *options.YurtHubOptions, } } -// isServiceTopologyFilterEnabled is used to verify the service topology filter should be enabled or not. -func isServiceTopologyFilterEnabled(options *options.YurtHubOptions) bool { - if !options.EnableResourceFilter { - return false - } - - for _, filterName := range options.DisabledResourceFilters { - if filterName == filter.ServiceTopologyFilterName { - return false - } - } - - if options.WorkingMode == string(util.WorkingModeCloud) { - for i := range filter.DisabledInCloudMode { - if filter.DisabledInCloudMode[i] == filter.ServiceTopologyFilterName { - return false - } - } - } - - return true -} - func prepareServerServing(options *options.YurtHubOptions, certMgr certificate.YurtCertificateManager, cfg *YurtHubConfiguration) error { if err := (&apiserveroptions.DeprecatedInsecureServingOptions{ BindAddress: net.ParseIP(options.YurtHubHost), diff --git a/cmd/yurthub/app/start.go b/cmd/yurthub/app/start.go index ba6b166ede8..14e101a2245 100644 --- a/cmd/yurthub/app/start.go +++ b/cmd/yurthub/app/start.go @@ -176,7 +176,7 @@ func Run(ctx context.Context, cfg *config.YurtHubConfiguration) error { // Start the informer factory if all informers have been registered cfg.SharedFactory.Start(ctx.Done()) - cfg.YurtSharedFactory.Start(ctx.Done()) + cfg.NodePoolInformerFactory.Start(ctx.Done()) klog.Infof("%d. new reverse proxy handler for remote servers", trace) yurtProxyHandler, err := proxy.NewYurtReverseProxyHandler( diff --git a/go.mod b/go.mod index 53dcd2e2ad0..b26335a009b 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,6 @@ require ( github.com/onsi/ginkgo/v2 v2.1.4 github.com/onsi/gomega v1.19.0 github.com/opencontainers/selinux v1.11.0 - github.com/openyurtio/yurt-app-manager-api v0.6.0 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 github.com/projectcalico/api v0.0.0-20230222223746-44aa60c2201f diff --git a/go.sum b/go.sum index c77bd68f7d3..bf7f5a02e50 100644 --- a/go.sum +++ b/go.sum @@ -232,7 +232,6 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM= @@ -381,7 +380,6 @@ github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -524,7 +522,6 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= @@ -532,7 +529,6 @@ github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= @@ -557,8 +553,6 @@ github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:Ff github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU= github.com/openyurtio/apiserver-network-proxy v0.1.0 h1:uJI6LeAHmkQL0zV1+NIbgRsx2ayzsPfMA2bd1gROypc= github.com/openyurtio/apiserver-network-proxy v0.1.0/go.mod h1:X5Au3jBNIgYL2uK0IHeNGnZqlUlVSCFQhi/npPgkKRg= -github.com/openyurtio/yurt-app-manager-api v0.6.0 h1:GoayIUkdITBufJirU94dvyknFFG4On1T7XcDvsqCWaQ= -github.com/openyurtio/yurt-app-manager-api v0.6.0/go.mod h1:Ql/n89HmezW7s0d2Cyq9P3hl2MEvvjjv3xxPkLVzz10= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= @@ -1188,7 +1182,6 @@ k8s.io/csi-translation-lib v0.22.3/go.mod h1:YkdI+scWhZJQeA26iNg9XrKO3LhLz6dAcRK k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-aggregator v0.22.3/go.mod h1:TIpLq1HvR/S4y75i3y+4q9ik3ZvgyaDz72CBfDS0A6E= @@ -1208,7 +1201,6 @@ k8s.io/pod-security-admission v0.22.3/go.mod h1:xtkf/UhVWICokQLSDvD+8plfGkTQW4VT k8s.io/sample-apiserver v0.22.3/go.mod h1:HuEOdD/pT5R7gKNr2REb62uabZaJuFZyY3wUd86nFCA= k8s.io/system-validators v1.5.0/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210527160623-6fdb442a123b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b h1:wxEMGetGMur3J1xuGLQY7GEQYg9bZxKn3tKo5k/eYcs= @@ -1224,7 +1216,6 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22 h1:fmRfl9WJ4ApJn7LxNuED4m0t18qivVQOxP6aAYG9J6c= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-runtime v0.9.0/go.mod h1:TgkfvrhhEw3PlI0BRL/5xM+89y3/yc0ZDfdbTl84si8= sigs.k8s.io/controller-runtime v0.10.3 h1:s5Ttmw/B4AuIbwrXD3sfBkXwnPMMWrqpVj4WRt1dano= sigs.k8s.io/controller-runtime v0.10.3/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= sigs.k8s.io/kustomize/api v0.8.11 h1:LzQzlq6Z023b+mBtc6v72N2mSHYmN8x7ssgbf/hv0H8= diff --git a/pkg/apis/apps/well_known_labels_annotations.go b/pkg/apis/apps/well_known_labels_annotations.go index a712a7b6676..7118efd71d4 100644 --- a/pkg/apis/apps/well_known_labels_annotations.go +++ b/pkg/apis/apps/well_known_labels_annotations.go @@ -40,6 +40,7 @@ const ( // LabelCurrentNodePool indicates which nodepool the node is currently // belonging to LabelCurrentNodePool = "apps.openyurt.io/nodepool" + NodePoolLabel = "apps.openyurt.io/nodepool" AnnotationPrevAttrs = "nodepool.openyurt.io/previous-attributes" diff --git a/pkg/util/kubeconfig/kubeconfig.go b/pkg/util/kubeconfig/kubeconfig.go index ca713622390..9625d2d3351 100644 --- a/pkg/util/kubeconfig/kubeconfig.go +++ b/pkg/util/kubeconfig/kubeconfig.go @@ -24,8 +24,6 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - - yurtclientset "github.com/openyurtio/yurt-app-manager-api/pkg/yurtappmanager/client/clientset/versioned" ) // CreateBasic creates a basic, general KubeConfig object that then can be extended @@ -127,18 +125,3 @@ func GetAuthInfoFromKubeConfig(config *clientcmdapi.Config) *clientcmdapi.AuthIn } return nil } - -// ToYurtClientSet converts a KubeConfig object to a yurtClient -func ToYurtClientSet(config *clientcmdapi.Config) (yurtclientset.Interface, error) { - overrides := clientcmd.ConfigOverrides{Timeout: "10s"} - clientConfig, err := clientcmd.NewDefaultClientConfig(*config, &overrides).ClientConfig() - if err != nil { - return nil, errors.Wrap(err, "failed to create yurt client configuration from kubeconfig") - } - - client, err := yurtclientset.NewForConfig(clientConfig) - if err != nil { - return nil, errors.Wrap(err, "failed to create yurt client") - } - return client, nil -} diff --git a/pkg/util/kubeconfig/kubeconfig_test.go b/pkg/util/kubeconfig/kubeconfig_test.go index 11d1a075e65..b1cce66144a 100644 --- a/pkg/util/kubeconfig/kubeconfig_test.go +++ b/pkg/util/kubeconfig/kubeconfig_test.go @@ -183,8 +183,6 @@ func TestWriteKubeconfigToDisk(t *testing.T) { newFile, ) } - client, err := ToYurtClientSet(c) - t.Log(client, err) }) } } diff --git a/pkg/util/kubernetes/kubeadm/app/util/apiclient/idempotency.go b/pkg/util/kubernetes/kubeadm/app/util/apiclient/idempotency.go index e8d85e2372e..4fecdde5bb7 100644 --- a/pkg/util/kubernetes/kubeadm/app/util/apiclient/idempotency.go +++ b/pkg/util/kubernetes/kubeadm/app/util/apiclient/idempotency.go @@ -24,13 +24,17 @@ import ( rbac "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" clientsetretry "k8s.io/client-go/util/retry" + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/constants" - nodepoolv1alpha1 "github.com/openyurtio/yurt-app-manager-api/pkg/yurtappmanager/apis/apps/v1alpha1" - yurtclientset "github.com/openyurtio/yurt-app-manager-api/pkg/yurtappmanager/client/clientset/versioned" ) // ConfigMapMutator is a function that mutates the given ConfigMap and optionally returns an error @@ -134,12 +138,24 @@ func GetConfigMapWithRetry(client clientset.Interface, namespace, name string) ( return nil, lastError } -func GetNodePoolInfoWithRetry(client yurtclientset.Interface, name string) (*nodepoolv1alpha1.NodePool, error) { - var np *nodepoolv1alpha1.NodePool +func GetNodePoolInfoWithRetry(cfg *clientcmdapi.Config, name string) (*v1beta1.NodePool, error) { + gvr := v1beta1.GroupVersion.WithResource("nodepools") + + clientConfig := clientcmd.NewDefaultClientConfig(*cfg, &clientcmd.ConfigOverrides{}) + restConfig, err := clientConfig.ClientConfig() + if err != nil { + return nil, err + } + dynamicClient, err := dynamic.NewForConfig(restConfig) + if err != nil { + return nil, err + } + + var obj *unstructured.Unstructured var lastError error - err := wait.ExponentialBackoff(clientsetretry.DefaultBackoff, func() (bool, error) { + err = wait.ExponentialBackoff(clientsetretry.DefaultBackoff, func() (bool, error) { var err error - np, err = client.AppsV1alpha1().NodePools().Get(context.TODO(), name, metav1.GetOptions{}) + obj, err = dynamicClient.Resource(gvr).Get(context.TODO(), name, metav1.GetOptions{}) if err == nil { return true, nil } @@ -150,6 +166,10 @@ func GetNodePoolInfoWithRetry(client yurtclientset.Interface, name string) (*nod return false, nil }) if err == nil { + np := new(v1beta1.NodePool) + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), np); err != nil { + return nil, err + } return np, nil } return nil, lastError diff --git a/pkg/yurtadm/cmd/join/join.go b/pkg/yurtadm/cmd/join/join.go index ee81077a6be..1406c75b88d 100644 --- a/pkg/yurtadm/cmd/join/join.go +++ b/pkg/yurtadm/cmd/join/join.go @@ -30,6 +30,7 @@ import ( clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "k8s.io/klog/v2" + "github.com/openyurtio/openyurt/pkg/apis/apps" "github.com/openyurtio/openyurt/pkg/controller/yurtstaticset/util" kubeconfigutil "github.com/openyurtio/openyurt/pkg/util/kubeconfig" "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/util/apiclient" @@ -39,7 +40,6 @@ import ( "github.com/openyurtio/openyurt/pkg/yurtadm/util/edgenode" yurtadmutil "github.com/openyurtio/openyurt/pkg/yurtadm/util/kubernetes" "github.com/openyurtio/openyurt/pkg/yurtadm/util/yurthub" - nodepoolv1alpha1 "github.com/openyurtio/yurt-app-manager-api/pkg/yurtappmanager/apis/apps/v1alpha1" ) type joinOptions struct { @@ -345,19 +345,13 @@ func newJoinData(args []string, opt *joinOptions) (*joinData, error) { // check whether specified nodePool exists if len(opt.nodePoolName) != 0 { - yurtClient, err := kubeconfigutil.ToYurtClientSet(cfg) - if err != nil { - klog.Errorf("failed to create yurt client, %v", err) - return nil, err - } - - np, err := apiclient.GetNodePoolInfoWithRetry(yurtClient, opt.nodePoolName) + np, err := apiclient.GetNodePoolInfoWithRetry(cfg, opt.nodePoolName) if err != nil || np == nil { // the specified nodePool not exist, return return nil, errors.Errorf("when --nodepool-name is specified, the specified nodePool should be exist.") } // add nodePool label for node by kubelet - data.nodeLabels[nodepoolv1alpha1.LabelDesiredNodePool] = opt.nodePoolName + data.nodeLabels[apps.NodePoolLabel] = opt.nodePoolName } // check static pods has value and yurtstaticset is already exist diff --git a/pkg/yurthub/filter/initializer/initializer.go b/pkg/yurthub/filter/initializer/initializer.go index 87d0d982185..6049372c4ad 100644 --- a/pkg/yurthub/filter/initializer/initializer.go +++ b/pkg/yurthub/filter/initializer/initializer.go @@ -17,11 +17,11 @@ limitations under the License. package initializer import ( + "k8s.io/client-go/dynamic/dynamicinformer" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "github.com/openyurtio/openyurt/pkg/yurthub/filter" - yurtinformers "github.com/openyurtio/yurt-app-manager-api/pkg/yurtappmanager/client/informers/externalversions" ) // WantsSharedInformerFactory is an interface for setting SharedInformerFactory @@ -29,9 +29,9 @@ type WantsSharedInformerFactory interface { SetSharedInformerFactory(factory informers.SharedInformerFactory) error } -// WantsYurtSharedInformerFactory is an interface for setting Yurt-App-Manager SharedInformerFactory -type WantsYurtSharedInformerFactory interface { - SetYurtSharedInformerFactory(yurtFactory yurtinformers.SharedInformerFactory) error +// WantsNodePoolInformerFactory is an interface for setting NodePool CRD SharedInformerFactory +type WantsNodePoolInformerFactory interface { + SetNodePoolInformerFactory(factory dynamicinformer.DynamicSharedInformerFactory) error } // WantsNodeName is an interface for setting node name @@ -58,7 +58,7 @@ type WantsKubeClient interface { // genericFilterInitializer is responsible for initializing generic filter type genericFilterInitializer struct { factory informers.SharedInformerFactory - yurtFactory yurtinformers.SharedInformerFactory + nodePoolFactory dynamicinformer.DynamicSharedInformerFactory nodeName string nodePoolName string masterServiceHost string @@ -68,12 +68,12 @@ type genericFilterInitializer struct { // New creates an filterInitializer object func New(factory informers.SharedInformerFactory, - yurtFactory yurtinformers.SharedInformerFactory, + nodePoolFactory dynamicinformer.DynamicSharedInformerFactory, kubeClient kubernetes.Interface, nodeName, nodePoolName, masterServiceHost, masterServicePort string) filter.Initializer { return &genericFilterInitializer{ factory: factory, - yurtFactory: yurtFactory, + nodePoolFactory: nodePoolFactory, nodeName: nodeName, nodePoolName: nodePoolName, masterServiceHost: masterServiceHost, @@ -112,8 +112,8 @@ func (fi *genericFilterInitializer) Initialize(ins filter.ObjectFilter) error { } } - if wants, ok := ins.(WantsYurtSharedInformerFactory); ok { - if err := wants.SetYurtSharedInformerFactory(fi.yurtFactory); err != nil { + if wants, ok := ins.(WantsNodePoolInformerFactory); ok { + if err := wants.SetNodePoolInformerFactory(fi.nodePoolFactory); err != nil { return err } } diff --git a/pkg/yurthub/filter/initializer/initializer_test.go b/pkg/yurthub/filter/initializer/initializer_test.go index efda901fb1d..74c1d26f69c 100644 --- a/pkg/yurthub/filter/initializer/initializer_test.go +++ b/pkg/yurthub/filter/initializer/initializer_test.go @@ -24,6 +24,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/dynamic/dynamicinformer" + dynamicfake "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" @@ -33,21 +35,22 @@ import ( "github.com/openyurtio/openyurt/pkg/yurthub/filter/masterservice" "github.com/openyurtio/openyurt/pkg/yurthub/filter/nodeportisolation" "github.com/openyurtio/openyurt/pkg/yurthub/filter/servicetopology" - yurtfake "github.com/openyurtio/yurt-app-manager-api/pkg/yurtappmanager/client/clientset/versioned/fake" - yurtinformers "github.com/openyurtio/yurt-app-manager-api/pkg/yurtappmanager/client/informers/externalversions" ) func TestNew(t *testing.T) { fakeClient := &fake.Clientset{} - fakeYurtClient := &yurtfake.Clientset{} sharedFactory := informers.NewSharedInformerFactory(fakeClient, 24*time.Hour) - yurtSharedFactory := yurtinformers.NewSharedInformerFactory(fakeYurtClient, 24*time.Hour) + + scheme := runtime.NewScheme() + fakeDynamicClient := dynamicfake.NewSimpleDynamicClient(scheme) + nodePoolFactory := dynamicinformer.NewDynamicSharedInformerFactory(fakeDynamicClient, 24*time.Hour) + nodeName := "foo" nodePoolName := "foo-pool" masterServiceHost := "127.0.0.1" masterServicePort := "8080" - obj := New(sharedFactory, yurtSharedFactory, fakeClient, nodeName, nodePoolName, masterServiceHost, masterServicePort) + obj := New(sharedFactory, nodePoolFactory, fakeClient, nodeName, nodePoolName, masterServiceHost, masterServicePort) _, ok := obj.(filter.Initializer) if !ok { t.Errorf("expect a filter Initializer object, but got %v", reflect.TypeOf(obj)) @@ -85,15 +88,17 @@ func TestInitialize(t *testing.T) { }, } fakeClient := &fake.Clientset{} - fakeYurtClient := &yurtfake.Clientset{} sharedFactory := informers.NewSharedInformerFactory(fakeClient, 24*time.Hour) - yurtSharedFactory := yurtinformers.NewSharedInformerFactory(fakeYurtClient, 24*time.Hour) + + scheme := runtime.NewScheme() + fakeDynamicClient := dynamicfake.NewSimpleDynamicClient(scheme) + nodePoolFactory := dynamicinformer.NewDynamicSharedInformerFactory(fakeDynamicClient, 24*time.Hour) nodeName := "foo" nodePoolName := "foo-pool" masterServiceHost := "127.0.0.1" masterServicePort := "8080" - obj := New(sharedFactory, yurtSharedFactory, fakeClient, nodeName, nodePoolName, masterServiceHost, masterServicePort) + obj := New(sharedFactory, nodePoolFactory, fakeClient, nodeName, nodePoolName, masterServiceHost, masterServicePort) for k, tc := range testcases { t.Run(k, func(t *testing.T) { diff --git a/pkg/yurthub/filter/manager/manager.go b/pkg/yurthub/filter/manager/manager.go index bc3d5f5dd91..b50dc3037b8 100644 --- a/pkg/yurthub/filter/manager/manager.go +++ b/pkg/yurthub/filter/manager/manager.go @@ -22,6 +22,7 @@ import ( "strconv" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/dynamic/dynamicinformer" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" @@ -35,7 +36,6 @@ import ( "github.com/openyurtio/openyurt/pkg/yurthub/filter/servicetopology" "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/serializer" "github.com/openyurtio/openyurt/pkg/yurthub/util" - yurtinformers "github.com/openyurtio/yurt-app-manager-api/pkg/yurtappmanager/client/informers/externalversions" ) type Manager struct { @@ -46,7 +46,7 @@ type Manager struct { func NewFilterManager(options *options.YurtHubOptions, sharedFactory informers.SharedInformerFactory, - yurtSharedFactory yurtinformers.SharedInformerFactory, + nodePoolFactory dynamicinformer.DynamicSharedInformerFactory, proxiedClient kubernetes.Interface, serializerManager *serializer.SerializerManager, apiserverAddr string) (*Manager, error) { @@ -70,7 +70,7 @@ func NewFilterManager(options *options.YurtHubOptions, } } - objFilters, err := createObjectFilters(filters, sharedFactory, yurtSharedFactory, proxiedClient, options.NodeName, options.NodePoolName, mutatedMasterServiceHost, mutatedMasterServicePort) + objFilters, err := createObjectFilters(filters, sharedFactory, nodePoolFactory, proxiedClient, options.NodeName, options.NodePoolName, mutatedMasterServiceHost, mutatedMasterServicePort) if err != nil { return nil, err } @@ -113,14 +113,14 @@ func (m *Manager) FindResponseFilter(req *http.Request) (filter.ResponseFilter, // createObjectFilters return all object filters that initializations completed. func createObjectFilters(filters *filter.Filters, sharedFactory informers.SharedInformerFactory, - yurtSharedFactory yurtinformers.SharedInformerFactory, + nodePoolFactory dynamicinformer.DynamicSharedInformerFactory, proxiedClient kubernetes.Interface, nodeName, nodePoolName, mutatedMasterServiceHost, mutatedMasterServicePort string) ([]filter.ObjectFilter, error) { if filters == nil { return nil, nil } - genericInitializer := initializer.New(sharedFactory, yurtSharedFactory, proxiedClient, nodeName, nodePoolName, mutatedMasterServiceHost, mutatedMasterServicePort) + genericInitializer := initializer.New(sharedFactory, nodePoolFactory, proxiedClient, nodeName, nodePoolName, mutatedMasterServiceHost, mutatedMasterServicePort) initializerChain := filter.Initializers{} initializerChain = append(initializerChain, genericInitializer) return filters.NewFromFilters(initializerChain) diff --git a/pkg/yurthub/filter/manager/manager_test.go b/pkg/yurthub/filter/manager/manager_test.go index e702b7cd1b9..9c4e6d31159 100644 --- a/pkg/yurthub/filter/manager/manager_test.go +++ b/pkg/yurthub/filter/manager/manager_test.go @@ -23,23 +23,27 @@ import ( "testing" "time" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/endpoints/filters" "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/client-go/dynamic/dynamicinformer" + dynamicfake "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" "github.com/openyurtio/openyurt/cmd/yurthub/app/options" + "github.com/openyurtio/openyurt/pkg/apis" "github.com/openyurtio/openyurt/pkg/yurthub/filter" "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/serializer" "github.com/openyurtio/openyurt/pkg/yurthub/proxy/util" - yurtfake "github.com/openyurtio/yurt-app-manager-api/pkg/yurtappmanager/client/clientset/versioned/fake" - yurtinformers "github.com/openyurtio/yurt-app-manager-api/pkg/yurtappmanager/client/informers/externalversions" ) func TestFindResponseFilter(t *testing.T) { fakeClient := &fake.Clientset{} - fakeYurtClient := &yurtfake.Clientset{} + scheme := runtime.NewScheme() + apis.AddToScheme(scheme) + fakeDynamicClient := dynamicfake.NewSimpleDynamicClient(scheme) serializerManager := serializer.NewSerializerManager() apiserverAddr := "127.0.0.1:6443" @@ -128,18 +132,19 @@ func TestFindResponseFilter(t *testing.T) { } options.DisabledResourceFilters = append(options.DisabledResourceFilters, tt.disabledResourceFilters...) - sharedFactory, yurtSharedFactory := informers.NewSharedInformerFactory(fakeClient, 24*time.Hour), - yurtinformers.NewSharedInformerFactory(fakeYurtClient, 24*time.Hour) + sharedFactory, nodePoolFactory := informers.NewSharedInformerFactory(fakeClient, 24*time.Hour), + dynamicinformer.NewDynamicSharedInformerFactory(fakeDynamicClient, 24*time.Hour) + stopper := make(chan struct{}) defer close(stopper) - mgr, _ := NewFilterManager(options, sharedFactory, yurtSharedFactory, fakeClient, serializerManager, apiserverAddr) + mgr, _ := NewFilterManager(options, sharedFactory, nodePoolFactory, fakeClient, serializerManager, apiserverAddr) if tt.mgrIsNil && mgr == nil { return } sharedFactory.Start(stopper) - yurtSharedFactory.Start(stopper) + nodePoolFactory.Start(stopper) req, err := http.NewRequest(tt.verb, tt.path, nil) if err != nil { diff --git a/pkg/yurthub/filter/nodeportisolation/filter.go b/pkg/yurthub/filter/nodeportisolation/filter.go index d21171f0f20..8ab3f428943 100644 --- a/pkg/yurthub/filter/nodeportisolation/filter.go +++ b/pkg/yurthub/filter/nodeportisolation/filter.go @@ -130,7 +130,7 @@ func (nif *nodePortIsolationFilter) resolveNodePoolName() string { klog.Warningf("skip isolateNodePortService filter, failed to get node(%s), %v", nif.nodeName, err) return nif.nodePoolName } - nif.nodePoolName = node.Labels[apps.LabelDesiredNodePool] + nif.nodePoolName = node.Labels[apps.NodePoolLabel] return nif.nodePoolName } diff --git a/pkg/yurthub/filter/servicetopology/filter.go b/pkg/yurthub/filter/servicetopology/filter.go index 209c1f47610..4e4cf7af76a 100644 --- a/pkg/yurthub/filter/servicetopology/filter.go +++ b/pkg/yurthub/filter/servicetopology/filter.go @@ -23,18 +23,19 @@ import ( discovery "k8s.io/api/discovery/v1" discoveryV1beta1 "k8s.io/api/discovery/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/dynamic/dynamicinformer" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" + "github.com/openyurtio/openyurt/pkg/apis/apps" + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" "github.com/openyurtio/openyurt/pkg/yurthub/filter" - nodepoolv1alpha1 "github.com/openyurtio/yurt-app-manager-api/pkg/yurtappmanager/apis/apps/v1alpha1" - yurtinformers "github.com/openyurtio/yurt-app-manager-api/pkg/yurtappmanager/client/informers/externalversions" - appslisters "github.com/openyurtio/yurt-app-manager-api/pkg/yurtappmanager/client/listers/apps/v1alpha1" ) const ( @@ -62,7 +63,7 @@ func NewServiceTopologyFilter() (filter.ObjectFilter, error) { type serviceTopologyFilter struct { serviceLister listers.ServiceLister serviceSynced cache.InformerSynced - nodePoolLister appslisters.NodePoolLister + nodePoolLister cache.GenericLister nodePoolSynced cache.InformerSynced nodePoolName string nodeName string @@ -87,9 +88,10 @@ func (stf *serviceTopologyFilter) SetSharedInformerFactory(factory informers.Sha return nil } -func (stf *serviceTopologyFilter) SetYurtSharedInformerFactory(yurtFactory yurtinformers.SharedInformerFactory) error { - stf.nodePoolLister = yurtFactory.Apps().V1alpha1().NodePools().Lister() - stf.nodePoolSynced = yurtFactory.Apps().V1alpha1().NodePools().Informer().HasSynced +func (stf *serviceTopologyFilter) SetNodePoolInformerFactory(dynamicInformerFactory dynamicinformer.DynamicSharedInformerFactory) error { + gvr := v1beta1.GroupVersion.WithResource("nodepools") + stf.nodePoolLister = dynamicInformerFactory.ForResource(gvr).Lister() + stf.nodePoolSynced = dynamicInformerFactory.ForResource(gvr).Informer().HasSynced return nil } @@ -120,7 +122,7 @@ func (stf *serviceTopologyFilter) resolveNodePoolName() string { klog.Warningf("failed to get node(%s) in serviceTopologyFilter filter, %v", stf.nodeName, err) return stf.nodePoolName } - stf.nodePoolName = node.Labels[nodepoolv1alpha1.LabelDesiredNodePool] + stf.nodePoolName = node.Labels[apps.NodePoolLabel] return stf.nodePoolName } @@ -228,11 +230,25 @@ func (stf *serviceTopologyFilter) nodePoolTopologyHandler(obj runtime.Object) ru return stf.nodeTopologyHandler(obj) } - nodePool, err := stf.nodePoolLister.Get(nodePoolName) + runtimeObj, err := stf.nodePoolLister.Get(nodePoolName) if err != nil { klog.Warningf("serviceTopologyFilterHandler: failed to get nodepool %s, err: %v", nodePoolName, err) return obj } + var nodePool *v1beta1.NodePool + switch poolObj := runtimeObj.(type) { + case *v1beta1.NodePool: + nodePool = poolObj + case *unstructured.Unstructured: + nodePool = new(v1beta1.NodePool) + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(poolObj.UnstructuredContent(), nodePool); err != nil { + klog.Warningf("object(%#+v) is not a v1beta1.NodePool", poolObj) + return obj + } + default: + klog.Warningf("object(%#+v) is not a unknown type", poolObj) + return obj + } switch v := obj.(type) { case *discoveryV1beta1.EndpointSlice: @@ -247,7 +263,7 @@ func (stf *serviceTopologyFilter) nodePoolTopologyHandler(obj runtime.Object) ru } // reassembleV1beta1EndpointSlice will discard endpoints that are not on the same node/nodePool for v1beta1.EndpointSlice -func reassembleV1beta1EndpointSlice(endpointSlice *discoveryV1beta1.EndpointSlice, nodeName string, nodePool *nodepoolv1alpha1.NodePool) *discoveryV1beta1.EndpointSlice { +func reassembleV1beta1EndpointSlice(endpointSlice *discoveryV1beta1.EndpointSlice, nodeName string, nodePool *v1beta1.NodePool) *discoveryV1beta1.EndpointSlice { if len(nodeName) != 0 && nodePool != nil { klog.Warningf("reassembleV1beta1EndpointSlice: nodeName(%s) and nodePool can not be set at the same time", nodeName) return endpointSlice @@ -274,7 +290,7 @@ func reassembleV1beta1EndpointSlice(endpointSlice *discoveryV1beta1.EndpointSlic } // reassembleEndpointSlice will discard endpoints that are not on the same node/nodePool for v1.EndpointSlice -func reassembleEndpointSlice(endpointSlice *discovery.EndpointSlice, nodeName string, nodePool *nodepoolv1alpha1.NodePool) *discovery.EndpointSlice { +func reassembleEndpointSlice(endpointSlice *discovery.EndpointSlice, nodeName string, nodePool *v1beta1.NodePool) *discovery.EndpointSlice { if len(nodeName) != 0 && nodePool != nil { klog.Warningf("reassembleEndpointSlice: nodeName(%s) and nodePool can not be set at the same time", nodeName) return endpointSlice @@ -301,7 +317,7 @@ func reassembleEndpointSlice(endpointSlice *discovery.EndpointSlice, nodeName st } // reassembleEndpoints will discard subset that are not on the same node/nodePool for v1.Endpoints -func reassembleEndpoints(endpoints *v1.Endpoints, nodeName string, nodePool *nodepoolv1alpha1.NodePool) *v1.Endpoints { +func reassembleEndpoints(endpoints *v1.Endpoints, nodeName string, nodePool *v1beta1.NodePool) *v1.Endpoints { if len(nodeName) != 0 && nodePool != nil { klog.Warningf("reassembleEndpoints: nodeName(%s) and nodePool can not be set at the same time", nodeName) return endpoints @@ -329,7 +345,7 @@ func reassembleEndpoints(endpoints *v1.Endpoints, nodeName string, nodePool *nod return endpoints } -func filterValidEndpointsAddr(addresses []v1.EndpointAddress, nodeName string, nodePool *nodepoolv1alpha1.NodePool) []v1.EndpointAddress { +func filterValidEndpointsAddr(addresses []v1.EndpointAddress, nodeName string, nodePool *v1beta1.NodePool) []v1.EndpointAddress { var newEpAddresses []v1.EndpointAddress for i := range addresses { if addresses[i].NodeName == nil { diff --git a/pkg/yurthub/filter/servicetopology/filter_test.go b/pkg/yurthub/filter/servicetopology/filter_test.go index 3e234d92abf..01538a86b22 100644 --- a/pkg/yurthub/filter/servicetopology/filter_test.go +++ b/pkg/yurthub/filter/servicetopology/filter_test.go @@ -26,15 +26,18 @@ import ( discoveryV1beta1 "k8s.io/api/discovery/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/dynamic/dynamicinformer" + "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/informers" k8sfake "k8s.io/client-go/kubernetes/fake" + "github.com/openyurtio/openyurt/pkg/apis" + "github.com/openyurtio/openyurt/pkg/apis/apps" + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" "github.com/openyurtio/openyurt/pkg/util" "github.com/openyurtio/openyurt/pkg/yurthub/filter" - nodepoolv1alpha1 "github.com/openyurtio/yurt-app-manager-api/pkg/yurtappmanager/apis/apps/v1alpha1" - yurtfake "github.com/openyurtio/yurt-app-manager-api/pkg/yurtappmanager/client/clientset/versioned/fake" - yurtinformers "github.com/openyurtio/yurt-app-manager-api/pkg/yurtappmanager/client/informers/externalversions" ) func TestName(t *testing.T) { @@ -63,6 +66,11 @@ func TestSupportedResourceAndVerbs(t *testing.T) { } func TestFilter(t *testing.T) { + scheme := runtime.NewScheme() + apis.AddToScheme(scheme) + gvrToListKind := map[schema.GroupVersionResource]string{ + {Group: "apps.openyurt.io", Version: "v1beta1", Resource: "nodepools"}: "NodePoolList", + } currentNodeName := "node1" nodeName2 := "node2" nodeName3 := "node3" @@ -72,7 +80,7 @@ func TestFilter(t *testing.T) { nodeName string responseObject runtime.Object kubeClient *k8sfake.Clientset - yurtClient *yurtfake.Clientset + yurtClient *fake.FakeDynamicClient expectObject runtime.Object }{ "v1beta1.EndpointSliceList: topologyKeys is kubernetes.io/hostname": { @@ -129,7 +137,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: currentNodeName, Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -137,7 +145,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -145,7 +153,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -159,29 +167,29 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ currentNodeName, "node3", }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", }, @@ -274,7 +282,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: currentNodeName, Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -282,7 +290,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -290,7 +298,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -304,29 +312,29 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ currentNodeName, "node3", }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", }, @@ -426,7 +434,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: currentNodeName, Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -434,7 +442,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -442,7 +450,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -456,29 +464,29 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ currentNodeName, "node3", }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", }, @@ -578,7 +586,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: currentNodeName, Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -586,7 +594,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -594,7 +602,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -606,29 +614,29 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ currentNodeName, "node3", }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", }, @@ -742,7 +750,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -750,7 +758,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -764,28 +772,28 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node3", }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", }, @@ -877,7 +885,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: currentNodeName, Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -885,7 +893,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -893,7 +901,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -907,29 +915,29 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ currentNodeName, "node3", }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", }, @@ -1003,7 +1011,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: currentNodeName, Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -1011,7 +1019,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -1019,7 +1027,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -1033,29 +1041,29 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", "node3", }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ currentNodeName, }, @@ -1126,7 +1134,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: currentNodeName, Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -1134,7 +1142,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -1142,7 +1150,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -1156,29 +1164,29 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", "node3", }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ currentNodeName, }, @@ -1275,7 +1283,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: currentNodeName, Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -1283,7 +1291,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -1291,7 +1299,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -1305,29 +1313,29 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ currentNodeName, "node3", }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", }, @@ -1407,7 +1415,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: currentNodeName, Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -1415,7 +1423,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -1423,7 +1431,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -1437,29 +1445,29 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ currentNodeName, "node3", }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", }, @@ -1545,7 +1553,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: currentNodeName, Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -1553,7 +1561,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -1561,7 +1569,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -1575,29 +1583,29 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ currentNodeName, "node3", }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", }, @@ -1683,7 +1691,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: currentNodeName, Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -1691,7 +1699,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -1699,7 +1707,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -1711,29 +1719,29 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ currentNodeName, "node3", }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", }, @@ -1831,7 +1839,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -1839,7 +1847,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -1853,28 +1861,28 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node3", }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", }, @@ -1942,7 +1950,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: currentNodeName, Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -1950,7 +1958,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -1958,7 +1966,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -1972,28 +1980,28 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ currentNodeName, }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", "node3", @@ -2048,7 +2056,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: currentNodeName, Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -2056,7 +2064,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -2064,7 +2072,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -2078,28 +2086,28 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ currentNodeName, }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", "node3", @@ -2163,7 +2171,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: currentNodeName, Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -2171,7 +2179,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -2179,7 +2187,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -2193,29 +2201,29 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ currentNodeName, "node3", }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", }, @@ -2293,7 +2301,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: currentNodeName, Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -2301,7 +2309,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -2309,7 +2317,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -2323,29 +2331,29 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ currentNodeName, "node3", }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", }, @@ -2427,7 +2435,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: currentNodeName, Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -2435,7 +2443,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -2443,7 +2451,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -2457,29 +2465,29 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ currentNodeName, "node3", }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", }, @@ -2561,7 +2569,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: currentNodeName, Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -2569,7 +2577,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -2577,7 +2585,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -2589,29 +2597,29 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ currentNodeName, "node3", }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", }, @@ -2703,7 +2711,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -2711,7 +2719,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -2725,28 +2733,28 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node3", }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", }, @@ -2816,7 +2824,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: currentNodeName, Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -2824,7 +2832,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -2832,7 +2840,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -2846,28 +2854,28 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ currentNodeName, }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", "node3", @@ -2924,7 +2932,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: currentNodeName, Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -2932,7 +2940,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -2940,7 +2948,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -2954,28 +2962,28 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ currentNodeName, }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", "node3", @@ -3032,7 +3040,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: currentNodeName, Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -3040,7 +3048,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -3048,7 +3056,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -3062,28 +3070,28 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ currentNodeName, }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", "node3", @@ -3132,7 +3140,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: currentNodeName, Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "hangzhou", + apps.NodePoolLabel: "hangzhou", }, }, }, @@ -3140,7 +3148,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node2", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -3148,7 +3156,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "node3", Labels: map[string]string{ - nodepoolv1alpha1.LabelDesiredNodePool: "shanghai", + apps.NodePoolLabel: "shanghai", }, }, }, @@ -3162,28 +3170,28 @@ func TestFilter(t *testing.T) { }, }, ), - yurtClient: yurtfake.NewSimpleClientset( - &nodepoolv1alpha1.NodePool{ + yurtClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "hangzhou", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ currentNodeName, }, }, }, - &nodepoolv1alpha1.NodePool{ + &v1beta1.NodePool{ ObjectMeta: metav1.ObjectMeta{ Name: "shanghai", }, - Spec: nodepoolv1alpha1.NodePoolSpec{ - Type: nodepoolv1alpha1.Edge, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, }, - Status: nodepoolv1alpha1.NodePoolStatus{ + Status: v1beta1.NodePoolStatus{ Nodes: []string{ "node2", "node3", @@ -3213,8 +3221,9 @@ func TestFilter(t *testing.T) { factory.Start(stopper) factory.WaitForCacheSync(stopper) - yurtFactory := yurtinformers.NewSharedInformerFactory(tt.yurtClient, 24*time.Hour) - nodePoolInformer := yurtFactory.Apps().V1alpha1().NodePools() + gvr := v1beta1.GroupVersion.WithResource("nodepools") + yurtFactory := dynamicinformer.NewDynamicSharedInformerFactory(tt.yurtClient, 24*time.Hour) + nodePoolInformer := yurtFactory.ForResource(gvr) nodePoolLister := nodePoolInformer.Lister() nodePoolSynced := nodePoolInformer.Informer().HasSynced From 9d0a63885f693f013a3c8be2024c7e52f35ed474 Mon Sep 17 00:00:00 2001 From: dsy3502 Date: Tue, 15 Aug 2023 10:59:23 +0800 Subject: [PATCH 08/21] replace str by const (#1645) --- pkg/yurttunnel/trafficforward/dns/dns.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/yurttunnel/trafficforward/dns/dns.go b/pkg/yurttunnel/trafficforward/dns/dns.go index e0a7970e37e..b9e833e4d89 100644 --- a/pkg/yurttunnel/trafficforward/dns/dns.go +++ b/pkg/yurttunnel/trafficforward/dns/dns.go @@ -53,7 +53,8 @@ const ( maxRetries = 15 minSyncPeriod = 30 - dnatPortPrefix = "dnat-" + dnatPortPrefix = "dnat-" + dnsControllerName = "tunnel-dns-controller" ) var ( @@ -141,7 +142,7 @@ func (dnsctl *coreDNSRecordController) Run(stopCh <-chan struct{}) { if err != nil { klog.Fatalf("failed to get hostname, %v", err) } - rl, err := resourcelock.New("leases", metav1.NamespaceSystem, "tunnel-dns-controller", + rl, err := resourcelock.New("leases", metav1.NamespaceSystem, dnsControllerName, dnsctl.kubeClient.CoreV1(), dnsctl.kubeClient.CoordinationV1(), resourcelock.ResourceLockConfig{ @@ -165,7 +166,7 @@ func (dnsctl *coreDNSRecordController) Run(stopCh <-chan struct{}) { }, }, WatchDog: electionChecker, - Name: "tunnel-dns-controller", + Name: dnsControllerName, }) panic("unreachable") } @@ -177,7 +178,7 @@ func (dnsctl *coreDNSRecordController) run(stopCh <-chan struct{}) { klog.Infof("starting tunnel dns controller") defer klog.Infof("shutting down tunnel dns controller") - if !cache.WaitForNamedCacheSync("tunnel-dns-controller", stopCh, + if !cache.WaitForNamedCacheSync(dnsControllerName, stopCh, dnsctl.nodeListerSynced, dnsctl.svcInformerSynced, dnsctl.cmInformerSynced) { return } From d837588c0e19fb8fdbbe60ff38d074cbc40226b3 Mon Sep 17 00:00:00 2001 From: Zhen Zhao <413621396@qq.com> Date: Tue, 15 Aug 2023 11:01:23 +0800 Subject: [PATCH 09/21] remove node-servant unused command (#1632) --- cmd/yurt-node-servant/node-servant.go | 4 +- .../preflight-convert/preflight.go | 69 --- pkg/node-servant/constant.go | 84 ---- pkg/node-servant/job.go | 10 - pkg/node-servant/preflight-convert/options.go | 126 ----- .../preflight-convert/preflight.go | 50 -- pkg/node-servant/preflight/checks.go | 436 ------------------ pkg/node-servant/preflight/constants.go | 27 -- pkg/node-servant/preflight/interface.go | 32 -- 9 files changed, 1 insertion(+), 837 deletions(-) delete mode 100644 cmd/yurt-node-servant/preflight-convert/preflight.go delete mode 100644 pkg/node-servant/preflight-convert/options.go delete mode 100644 pkg/node-servant/preflight-convert/preflight.go delete mode 100644 pkg/node-servant/preflight/checks.go delete mode 100644 pkg/node-servant/preflight/constants.go delete mode 100644 pkg/node-servant/preflight/interface.go diff --git a/cmd/yurt-node-servant/node-servant.go b/cmd/yurt-node-servant/node-servant.go index 001cb63ea59..3ffbdeed692 100644 --- a/cmd/yurt-node-servant/node-servant.go +++ b/cmd/yurt-node-servant/node-servant.go @@ -26,7 +26,6 @@ import ( "github.com/openyurtio/openyurt/cmd/yurt-node-servant/config" "github.com/openyurtio/openyurt/cmd/yurt-node-servant/convert" - preflightconvert "github.com/openyurtio/openyurt/cmd/yurt-node-servant/preflight-convert" "github.com/openyurtio/openyurt/cmd/yurt-node-servant/revert" upgrade "github.com/openyurtio/openyurt/cmd/yurt-node-servant/static-pod-upgrade" "github.com/openyurtio/openyurt/pkg/projectinfo" @@ -42,13 +41,12 @@ func main() { version := fmt.Sprintf("%#v", projectinfo.Get()) rootCmd := &cobra.Command{ Use: "node-servant", - Short: "node-servant do preflight-convert/convert/revert specific node", + Short: "node-servant do convert/revert specific node", Version: version, } rootCmd.PersistentFlags().String("kubeconfig", "", "The path to the kubeconfig file") rootCmd.AddCommand(convert.NewConvertCmd()) rootCmd.AddCommand(revert.NewRevertCmd()) - rootCmd.AddCommand(preflightconvert.NewxPreflightConvertCmd()) rootCmd.AddCommand(config.NewConfigCmd()) rootCmd.AddCommand(upgrade.NewUpgradeCmd()) diff --git a/cmd/yurt-node-servant/preflight-convert/preflight.go b/cmd/yurt-node-servant/preflight-convert/preflight.go deleted file mode 100644 index 1db8c0dcec7..00000000000 --- a/cmd/yurt-node-servant/preflight-convert/preflight.go +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright 2021 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package preflight_convert - -import ( - "os" - - "github.com/spf13/cobra" - "k8s.io/klog/v2" - - preflightconvert "github.com/openyurtio/openyurt/pkg/node-servant/preflight-convert" -) - -const ( - latestYurtHubImage = "openyurt/yurthub:latest" - latestYurtTunnelAgentImage = "openyurt/yurt-tunnel-agent:latest" -) - -// NewxPreflightConvertCmd generates a new preflight-convert check command -func NewxPreflightConvertCmd() *cobra.Command { - o := preflightconvert.NewPreflightConvertOptions() - cmd := &cobra.Command{ - Use: "preflight-convert", - Short: "", - Run: func(cmd *cobra.Command, args []string) { - if err := o.Complete(cmd.Flags()); err != nil { - klog.Errorf("Fail to complete the preflight-convert option: %s", err) - os.Exit(1) - } - preflighter := preflightconvert.NewPreflighterWithOptions(o) - if err := preflighter.Do(); err != nil { - klog.Errorf("Fail to run pre-flight checks: %s", err) - os.Exit(1) - } - klog.Info("convert pre-flight checks success") - }, - Args: cobra.NoArgs, - } - setFlags(cmd) - - return cmd -} - -func setFlags(cmd *cobra.Command) { - cmd.Flags().StringP("kubeadm-conf-path", "k", "", - "The path to kubelet service conf that is used by kubelet component to join the cluster on the work node."+ - "Support multiple values, will search in order until get the file.(e.g -k kbcfg1,kbcfg2)", - ) - cmd.Flags().String("yurthub-image", latestYurtHubImage, "The yurthub image.") - cmd.Flags().String("yurt-tunnel-agent-image", latestYurtTunnelAgentImage, "The yurt-tunnel-agent image.") - cmd.Flags().BoolP("deploy-yurttunnel", "t", false, "If set, yurt-tunnel-agent will be deployed.") - cmd.Flags().String("ignore-preflight-errors", "", "A list of checks whose errors will be shown as warnings. "+ - "Example: 'isprivilegeduser,imagepull'.Value 'all' ignores errors from all checks.", - ) -} diff --git a/pkg/node-servant/constant.go b/pkg/node-servant/constant.go index 5b268e9cfbc..b91d0b4ed0b 100644 --- a/pkg/node-servant/constant.go +++ b/pkg/node-servant/constant.go @@ -23,12 +23,6 @@ const ( // RevertJobNameBase is the prefix of the revert ServantJob name RevertJobNameBase = "node-servant-revert" - //ConvertPreflightJobNameBase is the prefix of the preflight-convert ServantJob name - ConvertPreflightJobNameBase = "node-servant-preflight-convert" - - // ConfigControlPlaneJobNameBase is the prefix of the config control-plane ServantJob name - ConfigControlPlaneJobNameBase = "config-control-plane" - // ConvertServantJobTemplate defines the node convert servant job in yaml format ConvertServantJobTemplate = ` apiVersion: batch/v1 @@ -114,83 +108,5 @@ spec: - name: KUBELET_SVC value: {{.kubeadm_conf_path}} {{end}} -` - // ConvertPreflightJobTemplate defines the node convert preflight checks servant job in yaml format - ConvertPreflightJobTemplate = ` -apiVersion: batch/v1 -kind: Job -metadata: - name: {{.jobName}} - namespace: kube-system -spec: - template: - spec: - hostPID: true - hostNetwork: true - restartPolicy: OnFailure - nodeName: {{.nodeName}} - volumes: - - name: host-root - hostPath: - path: / - type: Directory - containers: - - name: node-servant - image: {{.node_servant_image}} - imagePullPolicy: IfNotPresent - command: - - /bin/sh - - -c - args: - - "/usr/local/bin/entry.sh preflight-convert {{if .ignore_preflight_errors}}--ignore-preflight-errors {{.ignore_preflight_errors}} {{end}}" - securityContext: - privileged: true - volumeMounts: - - mountPath: /openyurt - name: host-root - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - {{if .kubeadm_conf_path }} - - name: KUBELET_SVC - value: {{.kubeadm_conf_path}} - {{end}} -` - - // ConfigControlPlaneJobTemplate defines the node-servant config control-plane for configuring kube-apiserver and kube-controller-manager - ConfigControlPlaneJobTemplate = ` -apiVersion: batch/v1 -kind: Job -metadata: - name: {{.jobName}} - namespace: kube-system -spec: - template: - spec: - hostPID: true - hostNetwork: true - restartPolicy: OnFailure - nodeName: {{.nodeName}} - volumes: - - name: host-root - hostPath: - path: / - type: Directory - containers: - - name: node-servant - image: {{.node_servant_image}} - imagePullPolicy: IfNotPresent - command: - - /bin/sh - - -c - args: - - "/usr/local/bin/entry.sh config control-plane" - securityContext: - privileged: true - volumeMounts: - - mountPath: /openyurt - name: host-root ` ) diff --git a/pkg/node-servant/job.go b/pkg/node-servant/job.go index 36f44034da7..b14d86fac80 100644 --- a/pkg/node-servant/job.go +++ b/pkg/node-servant/job.go @@ -46,12 +46,6 @@ func RenderNodeServantJob(action string, renderCtx map[string]string, nodeName s case "revert": servantJobTemplate = RevertServantJobTemplate jobBaseName = RevertJobNameBase - case "preflight-convert": - servantJobTemplate = ConvertPreflightJobTemplate - jobBaseName = ConvertPreflightJobNameBase - case "config-control-plane": - servantJobTemplate = ConfigControlPlaneJobTemplate - jobBaseName = ConfigControlPlaneJobNameBase } tmplCtx["jobName"] = jobBaseName + "-" + nodeName @@ -95,10 +89,6 @@ func validate(action string, tmplCtx map[string]string, nodeName string) error { case "revert": keysMustHave := []string{"node_servant_image"} return checkKeys(keysMustHave, tmplCtx) - case "preflight-convert", "config-control-plane": - keysMustHave := []string{"node_servant_image"} - return checkKeys(keysMustHave, tmplCtx) - default: return fmt.Errorf("action invalied: %s ", action) } diff --git a/pkg/node-servant/preflight-convert/options.go b/pkg/node-servant/preflight-convert/options.go deleted file mode 100644 index 8c02e2df5da..00000000000 --- a/pkg/node-servant/preflight-convert/options.go +++ /dev/null @@ -1,126 +0,0 @@ -/* -Copyright 2021 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package preflight_convert - -import ( - "strings" - - "github.com/spf13/pflag" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/sets" - - "github.com/openyurtio/openyurt/pkg/node-servant/components" -) - -const ( - kubeAdmFlagsEnvFile = "/var/lib/kubelet/kubeadm-flags.env" -) - -// Options has the information that required by preflight-convert operation -type Options struct { - KubeadmConfPaths []string - YurthubImage string - YurttunnelAgentImage string - DeployTunnel bool - IgnorePreflightErrors sets.String - - KubeAdmFlagsEnvFile string - ImagePullPolicy v1.PullPolicy - CRISocket string -} - -func (o *Options) GetCRISocket() string { - return o.CRISocket -} - -func (o *Options) GetImageList() []string { - imgs := []string{} - - imgs = append(imgs, o.YurthubImage) - if o.DeployTunnel { - imgs = append(imgs, o.YurttunnelAgentImage) - } - return imgs -} - -func (o *Options) GetImagePullPolicy() v1.PullPolicy { - return o.ImagePullPolicy -} - -func (o *Options) GetKubeadmConfPaths() []string { - return o.KubeadmConfPaths -} - -func (o *Options) GetKubeAdmFlagsEnvFile() string { - return o.KubeAdmFlagsEnvFile -} - -// NewPreflightConvertOptions creates a new Options -func NewPreflightConvertOptions() *Options { - return &Options{ - KubeadmConfPaths: components.GetDefaultKubeadmConfPath(), - IgnorePreflightErrors: sets.NewString(), - KubeAdmFlagsEnvFile: kubeAdmFlagsEnvFile, - ImagePullPolicy: v1.PullIfNotPresent, - } -} - -// Complete completes all the required options. -func (o *Options) Complete(flags *pflag.FlagSet) error { - - kubeadmConfPaths, err := flags.GetString("kubeadm-conf-path") - if err != nil { - return err - } - if kubeadmConfPaths != "" { - o.KubeadmConfPaths = strings.Split(kubeadmConfPaths, ",") - } - - yurthubImage, err := flags.GetString("yurthub-image") - if err != nil { - return err - } - o.YurthubImage = yurthubImage - - yurttunnelAgentImage, err := flags.GetString("yurt-tunnel-agent-image") - if err != nil { - return err - } - o.YurttunnelAgentImage = yurttunnelAgentImage - - dt, err := flags.GetBool("deploy-yurttunnel") - if err != nil { - return err - } - o.DeployTunnel = dt - - ipStr, err := flags.GetString("ignore-preflight-errors") - if err != nil { - return err - } - if ipStr != "" { - ipStr = strings.ToLower(ipStr) - o.IgnorePreflightErrors = sets.NewString(strings.Split(ipStr, ",")...) - } - - CRISocket, err := components.DetectCRISocket() - if err != nil { - return err - } - o.CRISocket = CRISocket - return nil -} diff --git a/pkg/node-servant/preflight-convert/preflight.go b/pkg/node-servant/preflight-convert/preflight.go deleted file mode 100644 index d404e40fc5e..00000000000 --- a/pkg/node-servant/preflight-convert/preflight.go +++ /dev/null @@ -1,50 +0,0 @@ -/* -Copyright 2021 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package preflight_convert - -import ( - "k8s.io/klog/v2" - - "github.com/openyurtio/openyurt/pkg/node-servant/preflight" -) - -// ConvertPreflighter do the preflight-convert-convert job -type ConvertPreflighter struct { - Options -} - -// NewPreflighterWithOptions create nodePreflighter -func NewPreflighterWithOptions(o *Options) *ConvertPreflighter { - return &ConvertPreflighter{ - *o, - } -} - -func (n *ConvertPreflighter) Do() error { - klog.Infof("[preflight-convert] Running node-servant pre-flight checks") - if err := preflight.RunConvertNodeChecks(n, n.IgnorePreflightErrors, n.DeployTunnel); err != nil { - return err - } - - klog.Infof("[preflight-convert] Pulling images required for converting a Kubernetes cluster to an OpenYurt cluster") - klog.Infof("[preflight-convert] This might take a minute or two, depending on the speed of your internet connection") - if err := preflight.RunPullImagesCheck(n, n.IgnorePreflightErrors); err != nil { - return err - } - - return nil -} diff --git a/pkg/node-servant/preflight/checks.go b/pkg/node-servant/preflight/checks.go deleted file mode 100644 index 22eb1fe5cec..00000000000 --- a/pkg/node-servant/preflight/checks.go +++ /dev/null @@ -1,436 +0,0 @@ -/* -Copyright 2021 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package preflight - -import ( - "bytes" - "fmt" - "io" - "net" - "os" - "strings" - "sync" - "time" - - "github.com/pkg/errors" - batchv1 "k8s.io/api/batch/v1" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - utilsexec "k8s.io/utils/exec" - - nodeutil "github.com/openyurtio/openyurt/pkg/controller/util/node" - "github.com/openyurtio/openyurt/pkg/node-servant/components" - "github.com/openyurtio/openyurt/pkg/projectinfo" - kubeutil "github.com/openyurtio/openyurt/pkg/yurtadm/util/kubernetes" -) - -// Error defines struct for communicating error messages generated by preflight-convert-convert checks -type Error struct { - Msg string -} - -// Error implements the standard error interface -func (e *Error) Error() string { - return fmt.Sprintf("[preflight] Some fatal errors occurred:\n%s%s", e.Msg, "[preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...`\n") -} - -// Preflight identifies this error as a preflight-convert-convert error -func (e *Error) Preflight() bool { - return true -} - -// Checker validates the state of the system to ensure kubeadm will be -// successful as often as possible. -type Checker interface { - Check() (warnings, errorList []error) - Name() string -} - -// IsPrivilegedUserCheck verifies user is privileged (linux - root). -// The check under windows environment has not been implemented yet. -type IsPrivilegedUserCheck struct{} - -func (IsPrivilegedUserCheck) Name() string { - return "IsPrivilegedUser" -} - -// Check validates if an user has elevated (root) privileges. -func (ipuc IsPrivilegedUserCheck) Check() (warnings, errorList []error) { - if os.Getuid() != 0 { - return nil, []error{errors.New("user is not running as root")} - } - - return nil, nil -} - -// NodeReadyCheck checks the nodes status whether is ready. -type NodeReadyCheck struct { - NodeLst *v1.NodeList -} - -func (NodeReadyCheck) Name() string { - return "NodeReady" -} - -func (nrc NodeReadyCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("validating node status") - var notReadyNodeNames []string - for _, node := range nrc.NodeLst.Items { - if !isNodeReady(&node.Status) { - notReadyNodeNames = append(notReadyNodeNames, node.Name) - } - } - if len(notReadyNodeNames) != 0 { - return nil, []error{errors.Errorf("the status of nodes: %s is not 'Ready'", notReadyNodeNames)} - } - return nil, nil -} - -// NodeEdgeWorkerLabelCheck checks whether the node contains edgeWorker label which represents an OpenYurt node. -type NodeEdgeWorkerLabelCheck struct { - NodeLst *v1.NodeList -} - -func (NodeEdgeWorkerLabelCheck) Name() string { - return "NodeEdgeWorkerLabel" -} - -func (nlc NodeEdgeWorkerLabelCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("validating node edgeworker label") - var hasLabelNodeNames []string - for _, node := range nlc.NodeLst.Items { - if _, ok := node.Labels[projectinfo.GetEdgeWorkerLabelKey()]; ok { - hasLabelNodeNames = append(hasLabelNodeNames, node.Name) - } - } - if len(hasLabelNodeNames) != 0 { - return nil, []error{errors.Errorf("the nodes %s has already been labeled as a OpenYurt node", hasLabelNodeNames)} - } - return nil, nil -} - -// JobExistCheck checks whether the jobs with a specific prefix exist. -type JobExistCheck struct { - JobLst *batchv1.JobList - Prefix string - Label string -} - -func (jc JobExistCheck) Name() string { - if jc.Label != "" { - return jc.Label - } - return fmt.Sprintf("JobExist-%s", jc.Prefix) -} - -func (jc JobExistCheck) Check() (warnings, errorList []error) { - klog.V(1).Infoln("validating convert jobs") - var invalidJobNames []string - for _, job := range jc.JobLst.Items { - if strings.HasPrefix(job.Name, jc.Prefix) { - invalidJobNames = append(invalidJobNames, job.Name) - } - } - if len(invalidJobNames) != 0 { - return nil, []error{errors.Errorf("jobs %s has prefix %s, may conflict with the conversion job name", invalidJobNames, jc.Prefix)} - } - return nil, nil -} - -// NodeServantJobCheck create jobs to do preflight checks. -// After the job is successfully executed, it will be deleted. -// The failed job will not be deleted, and the user needs to delete it manually. -type NodeServantJobCheck struct { - cliSet *kubernetes.Clientset - jobLst []*batchv1.Job - - waitServantJobTimeout time.Duration - checkServantJobPeriod time.Duration -} - -func (NodeServantJobCheck) Name() string { - return "NodeServantJob" -} - -func (nc NodeServantJobCheck) Check() (warnings []error, errorList []error) { - var wg sync.WaitGroup - - res := make(chan error, len(nc.jobLst)) - for _, job := range nc.jobLst { - wg.Add(1) - entity := *job - go func() { - defer wg.Done() - if err := kubeutil.RunJobAndCleanup(nc.cliSet, &entity, - nc.waitServantJobTimeout, nc.checkServantJobPeriod, false); err != nil { - msg := fmt.Errorf("fail to run servant job(%s): %w\n", entity.GetName(), err) - res <- msg - } else { - klog.V(1).Infof("servant job(%s) has succeeded\n", entity.GetName()) - } - }() - } - wg.Wait() - close(res) - for m := range res { - errorList = append(errorList, m) - } - return nil, errorList -} - -// FileExistingCheck checks that the given file already exist. -type FileExistingCheck struct { - Path string - Label string -} - -// Name returns label for individual FileExistingChecks. If not known, will return based on path. -func (fac FileExistingCheck) Name() string { - if fac.Label != "" { - return fac.Label - } - return fmt.Sprintf("FileExisting-%s", strings.Replace(fac.Path, "/", "-", -1)) -} - -func (fac FileExistingCheck) Check() (warnings, errorList []error) { - klog.V(1).Infof("validating the existence of file %s", fac.Path) - - if _, err := os.Stat(fac.Path); err != nil { - return nil, []error{errors.Errorf("%s doesn't exist", fac.Path)} - } - return nil, nil -} - -// FileAtLeastOneExistingCheck checks if at least one file exists in the file list. -// After a file is found, the remaining files will not be checked. -type FileAtLeastOneExistingCheck struct { - Paths []string - Label string -} - -func (foc FileAtLeastOneExistingCheck) Name() string { - if foc.Label != "" { - return foc.Label - } - return fmt.Sprintf("FileAtLeastOneExistingCheck-%s", foc.Paths[0]) -} - -func (foc FileAtLeastOneExistingCheck) Check() (warnings, errorList []error) { - klog.V(1).Infof("validating if at least one file exists in the file list: %s", foc.Paths) - for _, path := range foc.Paths { - if _, err := os.Stat(path); err == nil { - return nil, nil - } - } - return nil, []error{errors.Errorf("no file in list %s exists", foc.Paths)} - -} - -// DirExistingCheck checks if the given directory either exist, or is not empty. -type DirExistingCheck struct { - Path string - Label string -} - -// Name returns label for individual DirExistingChecks. If not known, will return based on path. -func (dac DirExistingCheck) Name() string { - if dac.Label != "" { - return dac.Label - } - return fmt.Sprintf("DirExisting-%s", strings.Replace(dac.Path, "/", "-", -1)) -} - -// Check validates if a directory exists or does not empty. -func (dac DirExistingCheck) Check() (warnings, errorList []error) { - klog.V(1).Infof("validating the existence of directory %s", dac.Path) - - if _, err := os.Stat(dac.Path); os.IsNotExist(err) { - return nil, []error{errors.Errorf("%s doesn't exist", dac.Path)} - } - - f, err := os.Open(dac.Path) - if err != nil { - return nil, []error{errors.Wrapf(err, "unable to check if %s is empty", dac.Path)} - } - defer f.Close() - - _, err = f.Readdirnames(1) - if err == io.EOF { - return nil, []error{errors.Errorf("%s is empty", dac.Path)} - } - return nil, nil -} - -// PortOpenCheck ensures the given port is available for use. -type PortOpenCheck struct { - port int - label string -} - -func (poc PortOpenCheck) Name() string { - if poc.label != "" { - return poc.label - } - return fmt.Sprintf("Port-%d", poc.port) -} - -func (poc PortOpenCheck) Check() (warnings, errorList []error) { - klog.V(1).Infof("validating availability of port %d", poc.port) - - ln, err := net.Listen("tcp", fmt.Sprintf(":%d", poc.port)) - if err != nil { - errorList = []error{errors.Errorf("Port %d is in use", poc.port)} - } - if ln != nil { - if err = ln.Close(); err != nil { - warnings = append(warnings, errors.Errorf("when closing port %d, encountered %v", poc.port, err)) - } - } - return warnings, errorList -} - -// ImagePullCheck will pull container images used by node-servant -type ImagePullCheck struct { - //runtime utilruntime.ContainerRuntime - runtime components.ContainerRuntimeForImage - imageList []string - imagePullPolicy v1.PullPolicy -} - -func (ImagePullCheck) Name() string { - return "ImagePull" -} - -func (ipc ImagePullCheck) Check() (warnings, errorList []error) { - policy := ipc.imagePullPolicy - klog.V(1).Infof("using image pull policy: %s", policy) - for _, image := range ipc.imageList { - switch policy { - case v1.PullNever: - klog.V(1).Infof("skipping pull of image: %s", image) - continue - case v1.PullIfNotPresent: - ret, err := ipc.runtime.ImageExists(image) - if ret && err == nil { - klog.V(1).Infof("image exists: %s", image) - continue - } - if err != nil { - errorList = append(errorList, errors.Wrapf(err, "failed to check if image %s exists", image)) - } - fallthrough // Proceed with pulling the image if it does not exist - case v1.PullAlways: - klog.V(1).Infof("pulling: %s", image) - if err := ipc.runtime.PullImage(image); err != nil { - errorList = append(errorList, errors.Wrapf(err, "failed to pull image %s", image)) - } - default: - // If the policy is unknown return early with an error - errorList = append(errorList, errors.Errorf("unsupported pull policy %q", policy)) - return warnings, errorList - } - } - return warnings, errorList -} - -func RunConvertNodeChecks(o KubePathOperator, ignorePreflightErrors sets.String, deployTunnel bool) error { - // First, check if we're root separately from the other preflight-convert-convert checks and fail fast - if err := RunRootCheckOnly(ignorePreflightErrors); err != nil { - return err - } - - checks := []Checker{ - FileAtLeastOneExistingCheck{Paths: o.GetKubeadmConfPaths(), Label: "KubeadmConfig"}, - FileExistingCheck{Path: o.GetKubeAdmFlagsEnvFile(), Label: "KubeAdmFlagsEnv"}, - DirExistingCheck{Path: KubernetesDir}, - DirExistingCheck{Path: KubeletPkiDir}, - PortOpenCheck{port: YurtHubProxySecurePort}, - PortOpenCheck{port: YurtHubProxyPort}, - PortOpenCheck{port: YurtHubPort}, - } - - if deployTunnel { - checks = append(checks, PortOpenCheck{port: YurttunnelAgentPort}) - } - return RunChecks(checks, os.Stderr, ignorePreflightErrors) - -} - -// RunRootCheckOnly initializes checks slice of structs and call RunChecks -func RunRootCheckOnly(ignorePreflightErrors sets.String) error { - checks := []Checker{ - IsPrivilegedUserCheck{}, - } - - return RunChecks(checks, os.Stderr, ignorePreflightErrors) -} - -// RunPullImagesCheck will pull images convert needs if they are not found on the system -func RunPullImagesCheck(o ImageOperator, ignorePreflightErrors sets.String) error { - containerRuntime, err := components.NewContainerRuntimeForImage(utilsexec.New(), o.GetCRISocket()) - if err != nil { - return err - } - - checks := []Checker{ - ImagePullCheck{runtime: containerRuntime, imageList: o.GetImageList(), imagePullPolicy: o.GetImagePullPolicy()}, - } - return RunChecks(checks, os.Stderr, ignorePreflightErrors) -} - -// RunChecks runs each check, displays it's warnings/errors, and once all -// are processed will exit if any errors occurred. -func RunChecks(checks []Checker, ww io.Writer, ignorePreflightErrors sets.String) error { - var errsBuffer bytes.Buffer - - for _, c := range checks { - name := c.Name() - warnings, errs := c.Check() - - if setHasItemOrAll(ignorePreflightErrors, name) { - // Decrease severity of errors to warnings for this check - warnings = append(warnings, errs...) - errs = []error{} - } - - for _, w := range warnings { - io.WriteString(ww, fmt.Sprintf("\t[WARNING %s]: %v\n", name, w)) - } - for _, i := range errs { - errsBuffer.WriteString(fmt.Sprintf("\t[ERROR %s]: %v\n", name, i.Error())) - } - } - if errsBuffer.Len() > 0 { - return &Error{Msg: errsBuffer.String()} - } - return nil -} - -// setHasItemOrAll is helper function that return true if item is present in the set (case insensitive) or special key 'all' is present -func setHasItemOrAll(s sets.String, item string) bool { - if s.Has("all") || s.Has(strings.ToLower(item)) { - return true - } - return false -} - -func isNodeReady(status *v1.NodeStatus) bool { - _, condition := nodeutil.GetNodeCondition(status, v1.NodeReady) - return condition != nil && condition.Status == v1.ConditionTrue -} diff --git a/pkg/node-servant/preflight/constants.go b/pkg/node-servant/preflight/constants.go deleted file mode 100644 index 7b3c2cf8299..00000000000 --- a/pkg/node-servant/preflight/constants.go +++ /dev/null @@ -1,27 +0,0 @@ -/* -Copyright 2021 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package preflight - -const ( - KubernetesDir = "/etc/kubernetes" - KubeletPkiDir = "/var/lib/kubelet/pki" - - YurtHubProxySecurePort = 10268 - YurtHubProxyPort = 10261 - YurtHubPort = 10267 - YurttunnelAgentPort = 10266 -) diff --git a/pkg/node-servant/preflight/interface.go b/pkg/node-servant/preflight/interface.go deleted file mode 100644 index 9581449a7fe..00000000000 --- a/pkg/node-servant/preflight/interface.go +++ /dev/null @@ -1,32 +0,0 @@ -/* -Copyright 2021 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package preflight - -import ( - v1 "k8s.io/api/core/v1" -) - -type ImageOperator interface { - GetCRISocket() string - GetImageList() []string - GetImagePullPolicy() v1.PullPolicy -} - -type KubePathOperator interface { - GetKubeadmConfPaths() []string - GetKubeAdmFlagsEnvFile() string -} From 8c91be5f797db1394931223c4856f71afbbcc725 Mon Sep 17 00:00:00 2001 From: dsy3502 Date: Tue, 15 Aug 2023 11:16:23 +0800 Subject: [PATCH 10/21] replace string var by const (#1635) --- .../yurtappdaemon/v1alpha1/yurtappdaemon_validation.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_validation.go b/pkg/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_validation.go index 991eef78a7e..fca4cd2ed70 100644 --- a/pkg/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_validation.go +++ b/pkg/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_validation.go @@ -39,6 +39,10 @@ import ( "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" ) +const ( + YurtAppDaemonKind = "YurtAppDaemon" +) + // ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. func (webhook *YurtAppDaemonHandler) ValidateCreate(ctx context.Context, obj runtime.Object) error { daemon, ok := obj.(*v1alpha1.YurtAppDaemon) @@ -47,7 +51,7 @@ func (webhook *YurtAppDaemonHandler) ValidateCreate(ctx context.Context, obj run } if allErrs := validateYurtAppDaemon(webhook.Client, daemon); len(allErrs) > 0 { - return apierrors.NewInvalid(v1alpha1.GroupVersion.WithKind("YurtAppDaemon").GroupKind(), daemon.Name, allErrs) + return apierrors.NewInvalid(v1alpha1.GroupVersion.WithKind(YurtAppDaemonKind).GroupKind(), daemon.Name, allErrs) } return nil @@ -67,7 +71,7 @@ func (webhook *YurtAppDaemonHandler) ValidateUpdate(ctx context.Context, oldObj, validationErrorList := validateYurtAppDaemon(webhook.Client, newDaemon) updateErrorList := ValidateYurtAppDaemonUpdate(newDaemon, oldDaemon) if allErrs := append(validationErrorList, updateErrorList...); len(allErrs) > 0 { - return apierrors.NewInvalid(v1alpha1.GroupVersion.WithKind("YurtAppDaemon").GroupKind(), newDaemon.Name, allErrs) + return apierrors.NewInvalid(v1alpha1.GroupVersion.WithKind(YurtAppDaemonKind).GroupKind(), newDaemon.Name, allErrs) } return nil } From 01a49853612176f2abf0481cfeed52900d218d8e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 11:46:23 +0800 Subject: [PATCH 11/21] build(deps): bump google.golang.org/grpc from 1.56.2 to 1.57.0 (#1634) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.56.2 to 1.57.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.56.2...v1.57.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 ++++-- go.sum | 12 ++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index b26335a009b..39e28a77e40 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( go.etcd.io/etcd/client/v3 v3.5.0 golang.org/x/net v0.9.0 golang.org/x/sys v0.10.0 - google.golang.org/grpc v1.56.2 + google.golang.org/grpc v1.57.0 gopkg.in/cheggaaa/pb.v1 v1.0.28 gopkg.in/square/go-jose.v2 v2.6.0 gopkg.in/yaml.v3 v3.0.1 @@ -155,7 +155,9 @@ require ( golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.66.2 // indirect diff --git a/go.sum b/go.sum index bf7f5a02e50..40c5893a6c5 100644 --- a/go.sum +++ b/go.sum @@ -1070,8 +1070,12 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 h1:9NWlQfY2ePejTmfwUH1OWwmznFa+0kKcHGPDvcPza9M= +google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 h1:m8v1xLLLzMe1m5P+gCTF8nJB9epwZQUBERm20Oy1poQ= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1087,8 +1091,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI= -google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From f4c8d287c8a91873e0e053c4109df9b845408077 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 12:52:23 +0800 Subject: [PATCH 12/21] build(deps): bump golang.org/x/net from 0.9.0 to 0.14.0 (#1643) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.9.0 to 0.14.0. - [Commits](https://github.com/golang/net/compare/v0.9.0...v0.14.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 39e28a77e40..7671b2da1fa 100644 --- a/go.mod +++ b/go.mod @@ -26,8 +26,8 @@ require ( go.etcd.io/etcd/api/v3 v3.5.0 go.etcd.io/etcd/client/pkg/v3 v3.5.0 go.etcd.io/etcd/client/v3 v3.5.0 - golang.org/x/net v0.9.0 - golang.org/x/sys v0.10.0 + golang.org/x/net v0.14.0 + golang.org/x/sys v0.11.0 google.golang.org/grpc v1.57.0 gopkg.in/cheggaaa/pb.v1 v1.0.28 gopkg.in/square/go-jose.v2 v2.6.0 @@ -147,11 +147,11 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.5.0 // indirect + golang.org/x/crypto v0.12.0 // indirect golang.org/x/oauth2 v0.7.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/term v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/term v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index 40c5893a6c5..b46474bda68 100644 --- a/go.sum +++ b/go.sum @@ -776,8 +776,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= -golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -858,8 +858,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -944,13 +944,13 @@ golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -960,8 +960,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From dbd2eaaf4ab6c35707b9dd43be76e01ebeaeb7e7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Aug 2023 12:54:23 +0800 Subject: [PATCH 13/21] build(deps): bump github.com/go-resty/resty/v2 from 2.4.0 to 2.7.0 (#1644) Bumps [github.com/go-resty/resty/v2](https://github.com/go-resty/resty) from 2.4.0 to 2.7.0. - [Release notes](https://github.com/go-resty/resty/releases) - [Commits](https://github.com/go-resty/resty/compare/v2.4.0...v2.7.0) --- updated-dependencies: - dependency-name: github.com/go-resty/resty/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7671b2da1fa..b2c36bffecc 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/aliyun/alibaba-cloud-sdk-go v1.62.156 github.com/davecgh/go-spew v1.1.1 github.com/edgexfoundry/go-mod-core-contracts/v2 v2.3.0 - github.com/go-resty/resty/v2 v2.4.0 + github.com/go-resty/resty/v2 v2.7.0 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 diff --git a/go.sum b/go.sum index b46474bda68..91b67aef842 100644 --- a/go.sum +++ b/go.sum @@ -256,8 +256,8 @@ github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/j github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ= github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= -github.com/go-resty/resty/v2 v2.4.0 h1:s6TItTLejEI+2mn98oijC5w/Rk2YU+OA6x0mnZN6r6k= -github.com/go-resty/resty/v2 v2.4.0/go.mod h1:B88+xCTEwvfD94NOuE6GS1wMlnoKNY8eEiNizfNwOwA= +github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= +github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -857,6 +857,7 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= From 4f6bcdcee207e21aa115470ae6675dee83073938 Mon Sep 17 00:00:00 2001 From: rambohe Date: Tue, 15 Aug 2023 16:42:23 +0800 Subject: [PATCH 14/21] improve nodepool as following: (#1651) 1. add spec.HostNetwork field 2. remove spec.Selector field 3. improve node labels for nodepool(apps.openyurt.io/desired-nodepool label is removed) 4. support node conversion between v1alpha1 and v1beta1 version Signed-off-by: rambohe-ch --- Makefile | 20 +- .../crds/apps.openyurt.io_nodepools.yaml | 509 +++++------ .../yurt-manager-auto-generated.yaml | 26 +- cmd/yurt-manager/app/manager.go | 2 +- .../app/options/nodepoolcontroller.go | 6 +- go.mod | 2 +- hack/make-rules/kustomize_to_chart.sh | 8 + pkg/apis/apps/v1alpha1/nodepool_conversion.go | 49 +- pkg/apis/apps/v1alpha1/nodepool_types.go | 3 +- pkg/apis/apps/v1beta1/default.go | 44 - pkg/apis/apps/v1beta1/nodepool_conversion.go | 49 +- pkg/apis/apps/v1beta1/nodepool_types.go | 9 +- .../apps/v1beta1/zz_generated.deepcopy.go | 10 +- .../apps/well_known_labels_annotations.go | 13 +- pkg/controller/nodepool/config/types.go | 2 +- .../nodepool/nodepool_controller.go | 143 ++- .../nodepool/nodepool_controller_test.go | 297 ++++++ .../nodepool/nodepool_enqueue_handlers.go | 90 +- .../nodepool_enqueue_handlers_test.go | 319 +++++++ pkg/controller/nodepool/util.go | 206 +---- pkg/controller/nodepool/util_test.go | 859 ++++++++++++++++++ pkg/webhook/node/v1/node_default.go | 61 ++ pkg/webhook/node/v1/node_default_test.go | 125 +++ pkg/webhook/node/v1/node_handler.go | 9 +- pkg/webhook/node/v1/node_validation.go | 32 +- pkg/webhook/node/v1/node_validation_test.go | 156 ++++ .../nodepool/v1alpha1/nodepool_default.go | 50 - .../nodepool/v1alpha1/nodepool_handler.go | 51 -- .../nodepool/v1alpha1/nodepool_validation.go | 143 --- .../nodepool/v1beta1/nodepool_default.go | 17 +- .../nodepool/v1beta1/nodepool_default_test.go | 110 +++ .../nodepool/v1beta1/nodepool_handler.go | 4 +- .../nodepool/v1beta1/nodepool_validation.go | 23 +- .../v1beta1/nodepool_validation_test.go | 285 ++++++ pkg/webhook/server.go | 7 +- .../util/controller/webhook_controller.go | 95 +- pkg/yurthub/network/dummyif_test.go | 2 + 37 files changed, 2841 insertions(+), 995 deletions(-) delete mode 100644 pkg/apis/apps/v1beta1/default.go create mode 100644 pkg/controller/nodepool/nodepool_controller_test.go create mode 100644 pkg/controller/nodepool/nodepool_enqueue_handlers_test.go create mode 100644 pkg/controller/nodepool/util_test.go create mode 100644 pkg/webhook/node/v1/node_default.go create mode 100644 pkg/webhook/node/v1/node_default_test.go create mode 100644 pkg/webhook/node/v1/node_validation_test.go delete mode 100644 pkg/webhook/nodepool/v1alpha1/nodepool_default.go delete mode 100644 pkg/webhook/nodepool/v1alpha1/nodepool_handler.go delete mode 100644 pkg/webhook/nodepool/v1alpha1/nodepool_validation.go create mode 100644 pkg/webhook/nodepool/v1beta1/nodepool_default_test.go create mode 100644 pkg/webhook/nodepool/v1beta1/nodepool_validation_test.go diff --git a/Makefile b/Makefile index fefb7fe0ca0..bae9f1a4a1f 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,9 @@ KUSTOMIZE ?= $(LOCALBIN)/kustomize KUBECTL_VERSION ?= v1.22.3 KUBECTL ?= $(LOCALBIN)/kubectl +YQ_VERSION := 4.13.2 +YQ := $(shell command -v $(LOCALBIN)/yq 2> /dev/null) + .PHONY: clean all build test all: test build @@ -72,8 +75,8 @@ build: # Run test test: - go test -v -short ./pkg/... ./cmd/... -coverprofile cover.out - go test -v -coverpkg=./pkg/yurttunnel/... -coverprofile=yurttunnel-cover.out ./test/integration/yurttunnel_test.go + go test -v ./pkg/... ./cmd/... -coverprofile cover.out + go test -v -coverpkg=./pkg/yurttunnel/... -coverprofile=yurttunnel-cover.out ./test/integration/yurttunnel_test.go clean: -rm -Rf _output @@ -178,7 +181,7 @@ generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and # hack/make-rule/generate_openapi.sh // TODO by kadisi $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./pkg/apis/..." -manifests: kustomize kubectl generate ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. +manifests: kustomize kubectl yq generate ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. rm -rf $(BUILD_KUSTOMIZE) $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=role webhook paths="./pkg/..." output:crd:artifacts:config=$(BUILD_KUSTOMIZE)/auto_generate/crd output:rbac:artifacts:config=$(BUILD_KUSTOMIZE)/auto_generate/rbac output:webhook:artifacts:config=$(BUILD_KUSTOMIZE)/auto_generate/webhook hack/make-rules/kustomize_to_chart.sh --crd $(BUILD_KUSTOMIZE)/auto_generate/crd --webhook $(BUILD_KUSTOMIZE)/auto_generate/webhook --rbac $(BUILD_KUSTOMIZE)/auto_generate/rbac --output $(BUILD_KUSTOMIZE)/kustomize --chartDir charts/yurt-manager @@ -219,6 +222,16 @@ $(KUSTOMIZE): $(LOCALBIN) fi test -s $(LOCALBIN)/kustomize || { curl -Ss $(KUSTOMIZE_INSTALL_SCRIPT) | bash -s -- $(subst v,,$(KUSTOMIZE_VERSION)) $(LOCALBIN); } +.PHONY: yq +yq: +ifndef YQ + @echo "Installing yq..." + test -s $(LOCALBIN)/yq || curl -k -L https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_$(shell go env GOOS)_$(shell go env GOARCH) -o $(LOCALBIN)/yq + chmod +x $(LOCALBIN)/yq +else + @echo "yq is already installed" +endif + # go-get-tool will 'go get' any package $2 and install it to $1. PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) define go-get-tool @@ -233,7 +246,6 @@ rm -rf $$TMP_DIR ;\ } endef - fmt: go fmt ./... find . -name '*.go' | grep -Ev 'vendor|thrift_gen' | xargs goimports -w diff --git a/charts/yurt-manager/crds/apps.openyurt.io_nodepools.yaml b/charts/yurt-manager/crds/apps.openyurt.io_nodepools.yaml index b7260163869..c9e0fc89c59 100644 --- a/charts/yurt-manager/crds/apps.openyurt.io_nodepools.yaml +++ b/charts/yurt-manager/crds/apps.openyurt.io_nodepools.yaml @@ -9,308 +9,241 @@ spec: group: apps.openyurt.io names: categories: - - all + - all kind: NodePool listKind: NodePoolList plural: nodepools shortNames: - - np + - np singular: nodepool scope: Cluster versions: - - additionalPrinterColumns: - - description: The type of nodepool - jsonPath: .spec.type - name: Type - type: string - - description: The number of ready nodes in the pool - jsonPath: .status.readyNodeNum - name: ReadyNodes - type: integer - - jsonPath: .status.unreadyNodeNum - name: NotReadyNodes - type: integer - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - deprecated: true - deprecationWarning: apps.openyurt.io/v1alpha1 NodePool is deprecated in v1.0.0+, - unavailable in v1.2.0+; use apps.openyurt.io/v1beta1 NodePool - name: v1alpha1 - schema: - openAPIV3Schema: - description: NodePool is the Schema for the nodepools API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NodePoolSpec defines the desired state of NodePool - properties: - annotations: - additionalProperties: - type: string - description: 'If specified, the Annotations will be added to all nodes. - NOTE: existing labels with samy keys on the nodes will be overwritten.' - type: object - labels: - additionalProperties: - type: string - description: 'If specified, the Labels will be added to all nodes. - NOTE: existing labels with samy keys on the nodes will be overwritten.' - type: object - selector: - description: A label query over nodes to consider for adding to the - pool - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic - merge patch. - items: + - additionalPrinterColumns: + - description: The type of nodepool + jsonPath: .spec.type + name: Type + type: string + - description: The number of ready nodes in the pool + jsonPath: .status.readyNodeNum + name: ReadyNodes + type: integer + - jsonPath: .status.unreadyNodeNum + name: NotReadyNodes + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + deprecated: true + deprecationWarning: apps.openyurt.io/v1alpha1 NodePool is deprecated in v1.0.0+; use apps.openyurt.io/v1beta1 NodePool + name: v1alpha1 + schema: + openAPIV3Schema: + description: NodePool is the Schema for the nodepools API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodePoolSpec defines the desired state of NodePool + properties: + annotations: + additionalProperties: + type: string + description: 'If specified, the Annotations will be added to all nodes. NOTE: existing labels with samy keys on the nodes will be overwritten.' + type: object + labels: + additionalProperties: + type: string + description: 'If specified, the Labels will be added to all nodes. NOTE: existing labels with samy keys on the nodes will be overwritten.' + type: object + selector: + description: A label query over nodes to consider for adding to the pool + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. type: string - type: array - required: - - key - - operator + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. - type: object - type: object - taints: - description: If specified, the Taints will be added to all nodes. - items: - description: The node this Taint is attached to has the "effect" - on any pod that does not tolerate the Taint. - properties: - effect: - description: Required. The effect of the taint on pods that - do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule - and NoExecute. - type: string - key: - description: Required. The taint key to be applied to a node. - type: string - timeAdded: - description: TimeAdded represents the time at which the taint - was added. It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - required: - - effect - - key type: object - type: array - type: - description: The type of the NodePool - type: string - type: object - status: - description: NodePoolStatus defines the observed state of NodePool - properties: - nodes: - description: The list of nodes' names in the pool - items: - type: string - type: array - readyNodeNum: - description: Total number of ready nodes in the pool. - format: int32 - type: integer - unreadyNodeNum: - description: Total number of unready nodes in the pool. - format: int32 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} - - additionalPrinterColumns: - - description: The type of nodepool - jsonPath: .spec.type - name: Type - type: string - - description: The number of ready nodes in the pool - jsonPath: .status.readyNodeNum - name: ReadyNodes - type: integer - - jsonPath: .status.unreadyNodeNum - name: NotReadyNodes - type: integer - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: NodePool is the Schema for the nodepools API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NodePoolSpec defines the desired state of NodePool - properties: - annotations: - additionalProperties: - type: string - description: 'If specified, the Annotations will be added to all nodes. - NOTE: existing labels with samy keys on the nodes will be overwritten.' - type: object - labels: - additionalProperties: - type: string - description: 'If specified, the Labels will be added to all nodes. - NOTE: existing labels with samy keys on the nodes will be overwritten.' - type: object - selector: - description: A label query over nodes to consider for adding to the - pool - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: + taints: + description: If specified, the Taints will be added to all nodes. + items: + description: The node this Taint is attached to has the "effect" on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to a node. + type: string + timeAdded: + description: TimeAdded represents the time at which the taint was added. It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. type: object - type: object - taints: - description: If specified, the Taints will be added to all nodes. - items: - description: The node this Taint is attached to has the "effect" - on any pod that does not tolerate the Taint. - properties: - effect: - description: Required. The effect of the taint on pods that - do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule - and NoExecute. - type: string - key: - description: Required. The taint key to be applied to a node. - type: string - timeAdded: - description: TimeAdded represents the time at which the taint - was added. It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - required: - - effect - - key + type: array + type: + description: The type of the NodePool + type: string + type: object + status: + description: NodePoolStatus defines the observed state of NodePool + properties: + nodes: + description: The list of nodes' names in the pool + items: + type: string + type: array + readyNodeNum: + description: Total number of ready nodes in the pool. + format: int32 + type: integer + unreadyNodeNum: + description: Total number of unready nodes in the pool. + format: int32 + type: integer + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: The type of nodepool + jsonPath: .spec.type + name: Type + type: string + - description: The number of ready nodes in the pool + jsonPath: .status.readyNodeNum + name: ReadyNodes + type: integer + - jsonPath: .status.unreadyNodeNum + name: NotReadyNodes + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: NodePool is the Schema for the nodepools API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodePoolSpec defines the desired state of NodePool + properties: + annotations: + additionalProperties: + type: string + description: 'If specified, the Annotations will be added to all nodes. NOTE: existing labels with samy keys on the nodes will be overwritten.' + type: object + hostNetwork: + description: HostNetwork is used to specify that cni components(like flannel) will not be installed on the nodes of this NodePool. This means all pods on the nodes of this NodePool will use HostNetwork and share network namespace with host machine. + type: boolean + labels: + additionalProperties: + type: string + description: 'If specified, the Labels will be added to all nodes. NOTE: existing labels with samy keys on the nodes will be overwritten.' type: object - type: array - type: - description: The type of the NodePool - type: string - type: object - status: - description: NodePoolStatus defines the observed state of NodePool - properties: - nodes: - description: The list of nodes' names in the pool - items: + taints: + description: If specified, the Taints will be added to all nodes. + items: + description: The node this Taint is attached to has the "effect" on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to a node. + type: string + timeAdded: + description: TimeAdded represents the time at which the taint was added. It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + type: array + type: + description: The type of the NodePool type: string - type: array - readyNodeNum: - description: Total number of ready nodes in the pool. - format: int32 - type: integer - unreadyNodeNum: - description: Total number of unready nodes in the pool. - format: int32 - type: integer - type: object - type: object - served: true - storage: false - subresources: - status: {} + type: object + status: + description: NodePoolStatus defines the observed state of NodePool + properties: + nodes: + description: The list of nodes' names in the pool + items: + type: string + type: array + readyNodeNum: + description: Total number of ready nodes in the pool. + format: int32 + type: integer + unreadyNodeNum: + description: Total number of unready nodes in the pool. + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1alpha1 + clientConfig: + service: + namespace: kube-system + name: yurt-manager-webhook-service + path: /convert status: acceptedNames: kind: "" diff --git a/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml b/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml index 177e5500223..6faa1d2d0f1 100644 --- a/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml +++ b/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml @@ -126,8 +126,6 @@ rules: resources: - nodepools verbs: - - create - - delete - get - list - patch @@ -538,6 +536,27 @@ webhooks: sideEffects: None - admissionReviewVersions: - v1 + clientConfig: + service: + name: yurt-manager-webhook-service + namespace: {{ .Release.Namespace }} + path: /mutate-core-openyurt-io-v1-node + failurePolicy: Fail + name: mutate.core.v1.node.openyurt.io + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - nodes + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 clientConfig: service: name: yurt-manager-webhook-service @@ -552,7 +571,6 @@ webhooks: - v1beta1 operations: - CREATE - - UPDATE resources: - nodepools sideEffects: None @@ -668,7 +686,6 @@ webhooks: sideEffects: None - admissionReviewVersions: - v1 - - v1beta1 clientConfig: service: name: yurt-manager-webhook-service @@ -688,6 +705,7 @@ webhooks: sideEffects: None - admissionReviewVersions: - v1 + - v1beta1 clientConfig: service: name: yurt-manager-webhook-service diff --git a/cmd/yurt-manager/app/manager.go b/cmd/yurt-manager/app/manager.go index 25ee89c57ab..fd328f28674 100644 --- a/cmd/yurt-manager/app/manager.go +++ b/cmd/yurt-manager/app/manager.go @@ -194,7 +194,7 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error { // +kubebuilder:scaffold:builder setupLog.Info("initialize webhook") - if err := webhook.Initialize(ctx, c); err != nil { + if err := webhook.Initialize(ctx, c, mgr.GetConfig()); err != nil { setupLog.Error(err, "unable to initialize webhook") os.Exit(1) } diff --git a/cmd/yurt-manager/app/options/nodepoolcontroller.go b/cmd/yurt-manager/app/options/nodepoolcontroller.go index fa9c7238ee2..8760760ef17 100644 --- a/cmd/yurt-manager/app/options/nodepoolcontroller.go +++ b/cmd/yurt-manager/app/options/nodepoolcontroller.go @@ -29,7 +29,7 @@ type NodePoolControllerOptions struct { func NewNodePoolControllerOptions() *NodePoolControllerOptions { return &NodePoolControllerOptions{ &config.NodePoolControllerConfiguration{ - CreateDefaultPool: false, + EnableSyncNodePoolConfigurations: true, }, } } @@ -40,7 +40,7 @@ func (n *NodePoolControllerOptions) AddFlags(fs *pflag.FlagSet) { return } - fs.BoolVar(&n.CreateDefaultPool, "create-default-pool", n.CreateDefaultPool, "Create default cloud/edge pools if indicated.") + fs.BoolVar(&n.EnableSyncNodePoolConfigurations, "enable-sync-nodepool-configurations", n.EnableSyncNodePoolConfigurations, "enable to sync nodepool configurations(including labels, annotations, taints in spec) to nodes in the nodepool.") } // ApplyTo fills up nodepool config with options. @@ -48,7 +48,7 @@ func (o *NodePoolControllerOptions) ApplyTo(cfg *config.NodePoolControllerConfig if o == nil { return nil } - cfg.CreateDefaultPool = o.CreateDefaultPool + cfg.EnableSyncNodePoolConfigurations = o.EnableSyncNodePoolConfigurations return nil } diff --git a/go.mod b/go.mod index b2c36bffecc..4e81d05d27c 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( gopkg.in/square/go-jose.v2 v2.6.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.22.3 + k8s.io/apiextensions-apiserver v0.22.2 k8s.io/apimachinery v0.22.3 k8s.io/apiserver v0.22.3 k8s.io/cli-runtime v0.22.3 @@ -163,7 +164,6 @@ require ( gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/apiextensions-apiserver v0.22.2 // indirect k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect diff --git a/hack/make-rules/kustomize_to_chart.sh b/hack/make-rules/kustomize_to_chart.sh index f8e0f98eff8..07545cec742 100755 --- a/hack/make-rules/kustomize_to_chart.sh +++ b/hack/make-rules/kustomize_to_chart.sh @@ -39,6 +39,8 @@ YURT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)" SUFFIX="auto_generated" +Conversion_Files=("apps.openyurt.io_nodepools.yaml") + while [ $# -gt 0 ];do case $1 in --crd|-c) @@ -197,6 +199,12 @@ EOF mv ${crd_dir}/apiextensions.k8s.io_v1_customresourcedefinition_deviceservices.iot.openyurt.io.yaml ${crd_dir}/iot.openyurt.io_deviceservices.yaml mv ${crd_dir}/apiextensions.k8s.io_v1_customresourcedefinition_deviceprofiles.iot.openyurt.io.yaml ${crd_dir}/iot.openyurt.io_deviceprofiles.yaml + # add conversion for crds + for file in "${Conversion_Files[@]}" + do + ${YURT_ROOT}/bin/yq eval -i ".spec.conversion = {\"strategy\": \"Webhook\", \"webhook\": {\"conversionReviewVersions\": [\"v1beta1\", \"v1alpha1\"], \"clientConfig\": {\"service\": {\"namespace\": \"kube-system\", \"name\": \"yurt-manager-webhook-service\", \"path\": \"/convert\"}}}}" ${crd_dir}/$file + done + # rbac dir local rbac_kustomization_resources="" for file in ${YURT_ROOT}/${RBAC}/* diff --git a/pkg/apis/apps/v1alpha1/nodepool_conversion.go b/pkg/apis/apps/v1alpha1/nodepool_conversion.go index a34319a0332..6dca755fb0b 100644 --- a/pkg/apis/apps/v1alpha1/nodepool_conversion.go +++ b/pkg/apis/apps/v1alpha1/nodepool_conversion.go @@ -16,11 +16,46 @@ limitations under the License. package v1alpha1 -/* -Implementing the hub method is pretty easy -- we just have to add an empty -method called `Hub()` to serve as a -[marker](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/conversion?tab=doc#Hub). -*/ +import ( + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/conversion" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" +) + +func (src *NodePool) ConvertTo(dstRaw conversion.Hub) error { + dst := dstRaw.(*v1beta1.NodePool) + + dst.ObjectMeta = src.ObjectMeta + + dst.Spec.Type = v1beta1.NodePoolType(src.Spec.Type) + dst.Spec.Labels = src.Spec.Labels + dst.Spec.Annotations = src.Spec.Annotations + dst.Spec.Taints = src.Spec.Taints + + dst.Status.ReadyNodeNum = src.Status.ReadyNodeNum + dst.Status.UnreadyNodeNum = src.Status.UnreadyNodeNum + dst.Status.Nodes = src.Status.Nodes + + klog.Infof("convert from v1alpha1 to v1beta1 for nodepool %s", dst.Name) + + return nil +} + +func (dst *NodePool) ConvertFrom(srcRaw conversion.Hub) error { + src := srcRaw.(*v1beta1.NodePool) + + dst.ObjectMeta = src.ObjectMeta + + dst.Spec.Type = NodePoolType(src.Spec.Type) + dst.Spec.Labels = src.Spec.Labels + dst.Spec.Annotations = src.Spec.Annotations + dst.Spec.Taints = src.Spec.Taints + + dst.Status.ReadyNodeNum = src.Status.ReadyNodeNum + dst.Status.UnreadyNodeNum = src.Status.UnreadyNodeNum + dst.Status.Nodes = src.Status.Nodes -// Hub marks this type as a conversion hub. -func (*NodePool) Hub() {} + klog.Infof("convert from v1beta1 to v1alpha1 for nodepool %s", dst.Name) + return nil +} diff --git a/pkg/apis/apps/v1alpha1/nodepool_types.go b/pkg/apis/apps/v1alpha1/nodepool_types.go index 548c5d90249..b19e0c8c984 100644 --- a/pkg/apis/apps/v1alpha1/nodepool_types.go +++ b/pkg/apis/apps/v1alpha1/nodepool_types.go @@ -73,12 +73,11 @@ type NodePoolStatus struct { // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Cluster,path=nodepools,shortName=np,categories=all // +kubebuilder:subresource:status -// +kubebuilder:deprecatedversion:warning="apps.openyurt.io/v1alpha1 NodePool is deprecated in v1.0.0+, unavailable in v1.2.0+; use apps.openyurt.io/v1beta1 NodePool" +// +kubebuilder:deprecatedversion:warning="apps.openyurt.io/v1alpha1 NodePool is deprecated in v1.0.0+; use apps.openyurt.io/v1beta1 NodePool" // +kubebuilder:printcolumn:name="Type",type="string",JSONPath=".spec.type",description="The type of nodepool" // +kubebuilder:printcolumn:name="ReadyNodes",type="integer",JSONPath=".status.readyNodeNum",description="The number of ready nodes in the pool" // +kubebuilder:printcolumn:name="NotReadyNodes",type="integer",JSONPath=".status.unreadyNodeNum" // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" -// +kubebuilder:storageversion // NodePool is the Schema for the nodepools API type NodePool struct { diff --git a/pkg/apis/apps/v1beta1/default.go b/pkg/apis/apps/v1beta1/default.go deleted file mode 100644 index f52b64855c2..00000000000 --- a/pkg/apis/apps/v1beta1/default.go +++ /dev/null @@ -1,44 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1beta1 - -import ( - "strings" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/openyurtio/openyurt/pkg/apis/apps" -) - -// SetDefaultsNodePool set default values for NodePool. -func SetDefaultsNodePool(obj *NodePool) { - // example for set default value for NodePool - if obj.Annotations == nil { - obj.Annotations = make(map[string]string) - } - - obj.Spec.Selector = &metav1.LabelSelector{ - MatchLabels: map[string]string{apps.LabelCurrentNodePool: obj.Name}, - } - - // add NodePool.Spec.Type to NodePool labels - if obj.Labels == nil { - obj.Labels = make(map[string]string) - } - obj.Labels[apps.NodePoolTypeLabelKey] = strings.ToLower(string(obj.Spec.Type)) - -} diff --git a/pkg/apis/apps/v1beta1/nodepool_conversion.go b/pkg/apis/apps/v1beta1/nodepool_conversion.go index fb6887bc42c..be5f0b3bf36 100644 --- a/pkg/apis/apps/v1beta1/nodepool_conversion.go +++ b/pkg/apis/apps/v1beta1/nodepool_conversion.go @@ -16,46 +16,11 @@ limitations under the License. package v1beta1 -import ( - "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/conversion" - - "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" -) - -func (src *NodePool) ConvertTo(dstRaw conversion.Hub) error { - dst := dstRaw.(*v1alpha1.NodePool) - - dst.ObjectMeta = src.ObjectMeta - - dst.Spec.Type = v1alpha1.NodePoolType(src.Spec.Type) - dst.Spec.Selector = src.Spec.Selector - dst.Spec.Annotations = src.Spec.Annotations - dst.Spec.Taints = src.Spec.Taints - - dst.Status.ReadyNodeNum = src.Status.ReadyNodeNum - dst.Status.UnreadyNodeNum = src.Status.UnreadyNodeNum - dst.Status.Nodes = src.Status.Nodes - - klog.Infof("convert from v1beta1 to v1alpha1 for %s", dst.Name) - - return nil -} - -func (dst *NodePool) ConvertFrom(srcRaw conversion.Hub) error { - src := srcRaw.(*v1alpha1.NodePool) - - dst.ObjectMeta = src.ObjectMeta - - dst.Spec.Type = NodePoolType(src.Spec.Type) - dst.Spec.Selector = src.Spec.Selector - dst.Spec.Annotations = src.Spec.Annotations - dst.Spec.Taints = src.Spec.Taints - - dst.Status.ReadyNodeNum = src.Status.ReadyNodeNum - dst.Status.UnreadyNodeNum = src.Status.UnreadyNodeNum - dst.Status.Nodes = src.Status.Nodes +/* +Implementing the hub method is pretty easy -- we just have to add an empty +method called `Hub()` to serve as a +[marker](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/conversion?tab=doc#Hub). +*/ - klog.Infof("convert from v1alpha1 to v1beta1 for %s", dst.Name) - return nil -} +// Hub marks this type as a conversion hub. +func (*NodePool) Hub() {} diff --git a/pkg/apis/apps/v1beta1/nodepool_types.go b/pkg/apis/apps/v1beta1/nodepool_types.go index bd970f4c164..60ee2a08b6c 100644 --- a/pkg/apis/apps/v1beta1/nodepool_types.go +++ b/pkg/apis/apps/v1beta1/nodepool_types.go @@ -36,9 +36,11 @@ type NodePoolSpec struct { // +optional Type NodePoolType `json:"type,omitempty"` - // A label query over nodes to consider for adding to the pool - // +optional - Selector *metav1.LabelSelector `json:"selector,omitempty"` + // HostNetwork is used to specify that cni components(like flannel) + // will not be installed on the nodes of this NodePool. + // This means all pods on the nodes of this NodePool will use + // HostNetwork and share network namespace with host machine. + HostNetwork bool `json:"hostNetwork,omitempty"` // If specified, the Labels will be added to all nodes. // NOTE: existing labels with samy keys on the nodes will be overwritten. @@ -80,6 +82,7 @@ type NodePoolStatus struct { // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" // +kubebuilder:subresource:status // +genclient:nonNamespaced +// +kubebuilder:storageversion // NodePool is the Schema for the nodepools API type NodePool struct { diff --git a/pkg/apis/apps/v1beta1/zz_generated.deepcopy.go b/pkg/apis/apps/v1beta1/zz_generated.deepcopy.go index ebb32758258..aed2b7ae309 100644 --- a/pkg/apis/apps/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/apps/v1beta1/zz_generated.deepcopy.go @@ -22,8 +22,7 @@ limitations under the License. package v1beta1 import ( - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -89,11 +88,6 @@ func (in *NodePoolList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodePoolSpec) DeepCopyInto(out *NodePoolSpec) { *out = *in - if in.Selector != nil { - in, out := &in.Selector, &out.Selector - *out = new(v1.LabelSelector) - (*in).DeepCopyInto(*out) - } if in.Labels != nil { in, out := &in.Labels, &out.Labels *out = make(map[string]string, len(*in)) @@ -110,7 +104,7 @@ func (in *NodePoolSpec) DeepCopyInto(out *NodePoolSpec) { } if in.Taints != nil { in, out := &in.Taints, &out.Taints - *out = make([]corev1.Taint, len(*in)) + *out = make([]v1.Taint, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } diff --git a/pkg/apis/apps/well_known_labels_annotations.go b/pkg/apis/apps/well_known_labels_annotations.go index 7118efd71d4..56f7abb77e4 100644 --- a/pkg/apis/apps/well_known_labels_annotations.go +++ b/pkg/apis/apps/well_known_labels_annotations.go @@ -40,17 +40,12 @@ const ( // LabelCurrentNodePool indicates which nodepool the node is currently // belonging to LabelCurrentNodePool = "apps.openyurt.io/nodepool" - NodePoolLabel = "apps.openyurt.io/nodepool" AnnotationPrevAttrs = "nodepool.openyurt.io/previous-attributes" - // DefaultCloudNodePoolName defines the name of the default cloud nodepool - DefaultCloudNodePoolName = "default-nodepool" + NodePoolLabel = "apps.openyurt.io/nodepool" + NodePoolTypeLabel = "nodepool.openyurt.io/type" + NodePoolHostNetworkLabel = "nodepool.openyurt.io/hostnetwork" - // DefaultEdgeNodePoolName defines the name of the default edge nodepool - DefaultEdgeNodePoolName = "default-edge-nodepool" - - // ServiceTopologyKey is the toplogy key that will be attached to node, - // the value will be the name of the nodepool - ServiceTopologyKey = "topology.kubernetes.io/zone" + NodePoolChangedEvent = "NodePoolChanged" ) diff --git a/pkg/controller/nodepool/config/types.go b/pkg/controller/nodepool/config/types.go index baedb88cf67..4ce71aa87c9 100644 --- a/pkg/controller/nodepool/config/types.go +++ b/pkg/controller/nodepool/config/types.go @@ -18,5 +18,5 @@ package config // NodePoolControllerConfiguration contains elements describing NodePoolController. type NodePoolControllerConfiguration struct { - CreateDefaultPool bool + EnableSyncNodePoolConfigurations bool } diff --git a/pkg/controller/nodepool/nodepool_controller.go b/pkg/controller/nodepool/nodepool_controller.go index ca0965105f6..2ca6d37ea35 100644 --- a/pkg/controller/nodepool/nodepool_controller.go +++ b/pkg/controller/nodepool/nodepool_controller.go @@ -18,11 +18,10 @@ package nodepool import ( "context" - "errors" "fmt" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/client-go/tools/record" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" @@ -36,14 +35,12 @@ import ( "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" "github.com/openyurtio/openyurt/pkg/apis/apps" appsv1beta1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" - nodepoolconfig "github.com/openyurtio/openyurt/pkg/controller/nodepool/config" - utilclient "github.com/openyurtio/openyurt/pkg/util/client" - utildiscovery "github.com/openyurtio/openyurt/pkg/util/discovery" + poolconfig "github.com/openyurtio/openyurt/pkg/controller/nodepool/config" ) var ( concurrentReconciles = 3 - controllerKind = appsv1beta1.SchemeGroupVersion.WithKind("NodePool") + controllerResource = appsv1beta1.SchemeGroupVersion.WithResource("nodepools") ) const ( @@ -58,9 +55,19 @@ func Format(format string, args ...interface{}) string { // ReconcileNodePool reconciles a NodePool object type ReconcileNodePool struct { client.Client - scheme *runtime.Scheme - recorder record.EventRecorder - Configration nodepoolconfig.NodePoolControllerConfiguration + mapper meta.RESTMapper + recorder record.EventRecorder + cfg poolconfig.NodePoolControllerConfiguration +} + +func (r *ReconcileNodePool) InjectClient(c client.Client) error { + r.Client = c + return nil +} + +func (r *ReconcileNodePool) InjectMapper(mapper meta.RESTMapper) error { + r.mapper = mapper + return nil } var _ reconcile.Reconciler = &ReconcileNodePool{} @@ -68,71 +75,53 @@ var _ reconcile.Reconciler = &ReconcileNodePool{} // Add creates a new NodePool Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(c *config.CompletedConfig, mgr manager.Manager) error { - if !utildiscovery.DiscoverGVK(controllerKind) { - klog.Errorf(Format("DiscoverGVK error")) - return nil + klog.Infof("nodepool-controller add controller %s", controllerResource.String()) + r := &ReconcileNodePool{ + cfg: c.ComponentConfig.NodePoolController, + recorder: mgr.GetEventRecorderFor(ControllerName), } - klog.Infof("nodepool-controller add controller %s", controllerKind.String()) - return add(mgr, newReconciler(c, mgr)) -} - -type NodePoolRelatedAttributes struct { - Labels map[string]string `json:"labels,omitempty"` - Annotations map[string]string `json:"annotations,omitempty"` - Taints []corev1.Taint `json:"taints,omitempty"` -} - -// newReconciler returns a new reconcile.Reconciler -func newReconciler(c *config.CompletedConfig, mgr manager.Manager) reconcile.Reconciler { - return &ReconcileNodePool{ - Client: utilclient.NewClientFromManager(mgr, ControllerName), - scheme: mgr.GetScheme(), - recorder: mgr.GetEventRecorderFor(ControllerName), - Configration: c.ComponentConfig.NodePoolController, - } -} - -// add adds a new Controller to mgr with r as the reconcile.Reconciler -func add(mgr manager.Manager, r reconcile.Reconciler) error { // Create a new controller - c, err := controller.New(ControllerName, mgr, controller.Options{ + ctrl, err := controller.New(ControllerName, mgr, controller.Options{ Reconciler: r, MaxConcurrentReconciles: concurrentReconciles, }) if err != nil { return err } + if _, err := r.mapper.KindFor(controllerResource); err != nil { + klog.Infof("resource %s doesn't exist", controllerResource.String()) + return err + } + // Watch for changes to NodePool - err = c.Watch(&source.Kind{Type: &appsv1beta1.NodePool{}}, &handler.EnqueueRequestForObject{}) + err = ctrl.Watch(&source.Kind{Type: &appsv1beta1.NodePool{}}, &handler.EnqueueRequestForObject{}) if err != nil { return err } // Watch for changes to Node - err = c.Watch(&source.Kind{ - Type: &corev1.Node{}}, - &EnqueueNodePoolForNode{}) + err = ctrl.Watch(&source.Kind{Type: &corev1.Node{}}, &EnqueueNodePoolForNode{ + EnableSyncNodePoolConfigurations: r.cfg.EnableSyncNodePoolConfigurations, + Recorder: r.recorder, + }) if err != nil { return err } - npr, ok := r.(*ReconcileNodePool) - if !ok { - return errors.New(Format("fail to assert interface to NodePoolReconciler")) - } - - if npr.Configration.CreateDefaultPool { - // register a node controller with the underlying informer of the manager - go createDefaultNodePool(mgr.GetClient()) - } return nil + +} + +type NodePoolRelatedAttributes struct { + Labels map[string]string `json:"labels,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + Taints []corev1.Taint `json:"taints,omitempty"` } -// +kubebuilder:rbac:groups=apps.openyurt.io,resources=nodepools,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps.openyurt.io,resources=nodepools,verbs=get;list;watch;update;patch // +kubebuilder:rbac:groups=apps.openyurt.io,resources=nodepools/status,verbs=get;update;patch // +kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch;update;patch -// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete // Reconcile reads that state of the cluster for a NodePool object and makes changes based on the state read // and what is in the NodePool.Spec @@ -141,51 +130,30 @@ func (r *ReconcileNodePool) Reconcile(ctx context.Context, req reconcile.Request // Note !!!!!!!!!! // We strongly recommend use Format() to encapsulation because Format() can print logs by module // @kadisi - klog.Infof(Format("Reconcile NodePool %s/%s", req.Namespace, req.Name)) + klog.Infof(Format("Reconcile NodePool %s", req.Name)) var nodePool appsv1beta1.NodePool // try to reconcile the NodePool object if err := r.Get(ctx, req.NamespacedName, &nodePool); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } - klog.Infof(Format("NodePool %++v", nodePool)) - - var desiredNodeList corev1.NodeList - if err := r.List(ctx, &desiredNodeList, client.MatchingLabels(map[string]string{ - apps.LabelDesiredNodePool: nodePool.GetName(), - })); err != nil { - return ctrl.Result{}, client.IgnoreNotFound(err) - } + klog.V(5).Infof("NodePool %s: %#+v", nodePool.Name, nodePool) var currentNodeList corev1.NodeList if err := r.List(ctx, ¤tNodeList, client.MatchingLabels(map[string]string{ - apps.LabelCurrentNodePool: nodePool.GetName(), + apps.NodePoolLabel: nodePool.GetName(), })); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } - // 1. handle the event of removing node out of the pool - // nodes in currentNodeList but not in the desiredNodeList, will be - // removed from the pool - removedNodes := getRemovedNodes(¤tNodeList, &desiredNodeList) - for _, rNode := range removedNodes { - if err := removePoolRelatedAttrs(&rNode); err != nil { - return ctrl.Result{}, err - } - if err := r.Update(ctx, &rNode); err != nil { - return ctrl.Result{}, err - } - } - - // 2. handle the event of adding node to the pool and the event of - // updating node pool attributes var ( readyNode int32 notReadyNode int32 nodes []string ) - for _, node := range desiredNodeList.Items { + // sync nodepool configurations to nodes + for _, node := range currentNodeList.Items { // prepare nodepool status nodes = append(nodes, node.GetName()) if isNodeReady(node) { @@ -194,23 +162,28 @@ func (r *ReconcileNodePool) Reconcile(ctx context.Context, req reconcile.Request notReadyNode += 1 } - // update node status according to nodepool - updated, err := concilateNode(&node, nodePool) - if err != nil { - return ctrl.Result{}, err - } - if updated { - if err := r.Update(ctx, &node); err != nil { - klog.Errorf(Format("Update Node %s error %v", node.Name, err)) + // sync nodepool configurations into node + if r.cfg.EnableSyncNodePoolConfigurations { + updated, err := conciliateNode(&node, &nodePool) + if err != nil { return ctrl.Result{}, err } + if updated { + if err := r.Update(ctx, &node); err != nil { + klog.Errorf(Format("Update Node %s error %v", node.Name, err)) + return ctrl.Result{}, err + } + } } } - // 3. always update the node pool status if necessary + // always update the node pool status if necessary needUpdate := conciliateNodePoolStatus(readyNode, notReadyNode, nodes, &nodePool) if needUpdate { + klog.V(5).Infof("nodepool(%s): (%#+v) will be updated", nodePool.Name, nodePool) return ctrl.Result{}, r.Status().Update(ctx, &nodePool) + } else { + klog.V(5).Infof("nodepool(%#+v) don't need to be updated, ready=%d, notReady=%d, nodes=%v", nodePool, readyNode, notReadyNode, nodes) } return ctrl.Result{}, nil } diff --git a/pkg/controller/nodepool/nodepool_controller_test.go b/pkg/controller/nodepool/nodepool_controller_test.go new file mode 100644 index 00000000000..aa0d0c687fb --- /dev/null +++ b/pkg/controller/nodepool/nodepool_controller_test.go @@ -0,0 +1,297 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodepool + +import ( + "context" + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/openyurtio/openyurt/pkg/apis" + "github.com/openyurtio/openyurt/pkg/apis/apps" + appsv1beta1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" + poolconfig "github.com/openyurtio/openyurt/pkg/controller/nodepool/config" +) + +func prepareNodes() []client.Object { + nodes := []client.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{ + apps.NodePoolLabel: "hangzhou", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node2", + Labels: map[string]string{ + apps.NodePoolLabel: "hangzhou", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeNetworkUnavailable, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node3", + Labels: map[string]string{ + apps.NodePoolLabel: "beijing", + }, + }, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node4", + Labels: map[string]string{ + apps.NodePoolLabel: "beijing", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + } + return nodes +} + +func prepareNodePools() []client.Object { + pools := []client.Object{ + &appsv1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hangzhou", + }, + Spec: appsv1beta1.NodePoolSpec{ + Type: appsv1beta1.Edge, + Labels: map[string]string{ + "region": "hangzhou", + }, + }, + }, + &appsv1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "beijing", + }, + Spec: appsv1beta1.NodePoolSpec{ + Type: appsv1beta1.Edge, + Labels: map[string]string{ + "region": "beijing", + }, + }, + }, + &appsv1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "shanghai", + }, + Spec: appsv1beta1.NodePoolSpec{ + Type: appsv1beta1.Edge, + Labels: map[string]string{ + "region": "shanghai", + }, + }, + }, + } + return pools +} + +func TestReconcile(t *testing.T) { + nodes := prepareNodes() + pools := prepareNodePools() + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + t.Fatal("Fail to add kubernetes clint-go custom resource") + } + apis.AddToScheme(scheme) + + c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(pools...).WithObjects(nodes...).Build() + testcases := map[string]struct { + EnableSyncNodePoolConfigurations bool + pool string + wantedPool *appsv1beta1.NodePool + wantedNodes []corev1.Node + err error + }{ + "reconcile hangzhou pool": { + pool: "hangzhou", + wantedPool: &appsv1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hangzhou", + }, + Spec: appsv1beta1.NodePoolSpec{ + Type: appsv1beta1.Edge, + Labels: map[string]string{ + "region": "hangzhou", + }, + }, + Status: appsv1beta1.NodePoolStatus{ + ReadyNodeNum: 1, + UnreadyNodeNum: 1, + Nodes: []string{"node1", "node2"}, + }, + }, + }, + "reconcile beijing pool": { + EnableSyncNodePoolConfigurations: true, + pool: "beijing", + wantedPool: &appsv1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "beijing", + }, + Spec: appsv1beta1.NodePoolSpec{ + Type: appsv1beta1.Edge, + Labels: map[string]string{ + "region": "beijing", + }, + }, + Status: appsv1beta1.NodePoolStatus{ + ReadyNodeNum: 1, + UnreadyNodeNum: 1, + Nodes: []string{"node3", "node4"}, + }, + }, + wantedNodes: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node3", + Labels: map[string]string{ + apps.NodePoolLabel: "beijing", + "region": "beijing", + }, + Annotations: map[string]string{ + "nodepool.openyurt.io/previous-attributes": "{\"labels\":{\"region\":\"beijing\"}}", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node4", + Labels: map[string]string{ + apps.NodePoolLabel: "beijing", + "region": "beijing", + }, + Annotations: map[string]string{ + "nodepool.openyurt.io/previous-attributes": "{\"labels\":{\"region\":\"beijing\"}}", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + }, + }, + "reconcile shanghai pool without nodes": { + pool: "shanghai", + wantedPool: &appsv1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "shanghai", + }, + Spec: appsv1beta1.NodePoolSpec{ + Type: appsv1beta1.Edge, + Labels: map[string]string{ + "region": "shanghai", + }, + }, + Status: appsv1beta1.NodePoolStatus{ + ReadyNodeNum: 0, + UnreadyNodeNum: 0, + }, + }, + }, + } + + ctx := context.TODO() + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + r := &ReconcileNodePool{ + Client: c, + cfg: poolconfig.NodePoolControllerConfiguration{ + EnableSyncNodePoolConfigurations: tc.EnableSyncNodePoolConfigurations, + }, + } + req := reconcile.Request{NamespacedName: types.NamespacedName{Name: tc.pool}} + _, err := r.Reconcile(ctx, req) + if err != nil { + t.Errorf("Reconcile() error = %v", err) + return + } + + var wantedPool appsv1beta1.NodePool + if err := r.Get(ctx, req.NamespacedName, &wantedPool); err != nil { + t.Errorf("Reconcile() error = %v", err) + return + } + if !reflect.DeepEqual(wantedPool.Status, tc.wantedPool.Status) { + t.Errorf("expected %#+v, got %#+v", tc.wantedPool.Status, wantedPool.Status) + return + } + + if len(tc.wantedNodes) != 0 { + var currentNodeList corev1.NodeList + if err := r.List(ctx, ¤tNodeList, client.MatchingLabels(map[string]string{ + apps.NodePoolLabel: tc.pool, + })); err != nil { + t.Errorf("Reconcile() error = %v", err) + return + } + gotNodes := make([]corev1.Node, 0) + for i := range currentNodeList.Items { + node := currentNodeList.Items[i] + node.ObjectMeta.ResourceVersion = "" + gotNodes = append(gotNodes, node) + } + + if !reflect.DeepEqual(tc.wantedNodes, gotNodes) { + t.Errorf("expected %#+v, \ngot %#+v", tc.wantedNodes, gotNodes) + } + } + }) + } +} diff --git a/pkg/controller/nodepool/nodepool_enqueue_handlers.go b/pkg/controller/nodepool/nodepool_enqueue_handlers.go index b19151043f4..6fe91665d6b 100644 --- a/pkg/controller/nodepool/nodepool_enqueue_handlers.go +++ b/pkg/controller/nodepool/nodepool_enqueue_handlers.go @@ -17,17 +17,24 @@ limitations under the License. package nodepool import ( + "fmt" "reflect" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/openyurtio/openyurt/pkg/apis/apps" ) -type EnqueueNodePoolForNode struct{} +type EnqueueNodePoolForNode struct { + EnableSyncNodePoolConfigurations bool + Recorder record.EventRecorder +} // Create implements EventHandler func (e *EnqueueNodePoolForNode) Create(evt event.CreateEvent, @@ -39,7 +46,7 @@ func (e *EnqueueNodePoolForNode) Create(evt event.CreateEvent, } klog.V(5).Infof(Format("will enqueue nodepool as node(%s) has been created", node.GetName())) - if np, exist := node.Labels[apps.LabelDesiredNodePool]; exist { + if np := node.Labels[apps.NodePoolLabel]; len(np) != 0 { addNodePoolToWorkQueue(np, q) return } @@ -61,59 +68,44 @@ func (e *EnqueueNodePoolForNode) Update(evt event.UpdateEvent, evt.ObjectOld.GetName())) return } - klog.V(5).Infof(Format("Will enqueue nodepool as node(%s) has been updated", - newNode.GetName())) - newNp := newNode.Labels[apps.LabelDesiredNodePool] - oldNp := oldNode.Labels[apps.LabelCurrentNodePool] + newNp := newNode.Labels[apps.NodePoolLabel] + oldNp := oldNode.Labels[apps.NodePoolLabel] + + // check the NodePoolLabel of node if len(oldNp) == 0 && len(newNp) == 0 { - klog.V(4).Infof(Format("node(%s) does not belong to any nodepool", newNode.GetName())) return - } - - if newNp != oldNp { - if newNp == "" { - // remove node from old pool - klog.V(5).Infof(Format("Will enqueue old pool(%s) for node(%s)", - oldNp, newNode.GetName())) - addNodePoolToWorkQueue(oldNp, q) - return - } - - if oldNp == "" { - // add node to the new Pool - klog.V(5).Infof(Format("Will enqueue new pool(%s) for node(%s)", - newNp, newNode.GetName())) - addNodePoolToWorkQueue(newNp, q) - return - } - klog.V(5).Infof(Format("Will enqueue both new pool(%s) and"+ - " old pool(%s) for node(%s)", - newNp, oldNp, newNode.GetName())) - addNodePoolToWorkQueue(oldNp, q) + } else if len(oldNp) == 0 { + // add node to the new Pool + klog.V(4).Infof(Format("node(%s) is added into pool(%s)", newNode.Name, newNp)) addNodePoolToWorkQueue(newNp, q) return + } else if oldNp != newNp { + klog.Warningf("It is not allowed to change the NodePoolLabel of node, but pool of node(%s) is changed from %s to %s", newNode.Name, oldNp, newNp) + // emit a warning event + e.Recorder.Event(newNode.DeepCopy(), corev1.EventTypeWarning, apps.NodePoolChangedEvent, + fmt.Sprintf("It is not allowed to change the NodePoolLabel of node, but nodepool of node(%s) is changed from %s to %s", newNode.Name, oldNp, newNp)) + return } + // check node ready status if isNodeReady(*newNode) != isNodeReady(*oldNode) { - // if the newNode and oldNode status are different - klog.V(5).Infof(Format("Node phase has been changed,"+ + klog.V(4).Infof(Format("Node ready status has been changed,"+ " will enqueue pool(%s) for node(%s)", newNp, newNode.GetName())) addNodePoolToWorkQueue(newNp, q) return } - if !reflect.DeepEqual(newNode.Labels, oldNode.Labels) || - !reflect.DeepEqual(newNode.Annotations, oldNode.Annotations) || - !reflect.DeepEqual(newNode.Spec.Taints, oldNode.Spec.Taints) { - // if node's labels, annotations or taints are updated - // TODO only consider the pool realted attributes - klog.V(5).Infof(Format("Nodepool related attributes has been changed,"+ - " will enqueue pool(%s) for node(%s)", - newNp, newNode.GetName())) - addNodePoolToWorkQueue(newNp, q) + // check node's labels, annotations or taints are updated or not + if e.EnableSyncNodePoolConfigurations { + if !reflect.DeepEqual(newNode.Labels, oldNode.Labels) || + !reflect.DeepEqual(newNode.Annotations, oldNode.Annotations) || + !reflect.DeepEqual(newNode.Spec.Taints, oldNode.Spec.Taints) { + // TODO only consider the pool related attributes + klog.V(5).Infof(Format("NodePool related attributes has been changed,will enqueue pool(%s) for node(%s)", newNp, newNode.Name)) + addNodePoolToWorkQueue(newNp, q) + } } - } // Delete implements EventHandler @@ -125,13 +117,13 @@ func (e *EnqueueNodePoolForNode) Delete(evt event.DeleteEvent, return } - np := node.Labels[apps.LabelCurrentNodePool] - if np == "" { - klog.V(5).Infof(Format("Node(%s) doesn't belong to any pool", node.GetName())) + np := node.Labels[apps.NodePoolLabel] + if len(np) == 0 { + klog.V(4).Infof(Format("A orphan node(%s) is removed", node.Name)) return } // enqueue the nodepool that the node belongs to - klog.V(5).Infof(Format("Will enqueue pool(%s) as node(%s) has been deleted", + klog.V(4).Infof(Format("Will enqueue pool(%s) as node(%s) has been deleted", np, node.GetName())) addNodePoolToWorkQueue(np, q) } @@ -140,3 +132,11 @@ func (e *EnqueueNodePoolForNode) Delete(evt event.DeleteEvent, func (e *EnqueueNodePoolForNode) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) { } + +// addNodePoolToWorkQueue adds the nodepool the reconciler's workqueue +func addNodePoolToWorkQueue(npName string, + q workqueue.RateLimitingInterface) { + q.Add(reconcile.Request{ + NamespacedName: types.NamespacedName{Name: npName}, + }) +} diff --git a/pkg/controller/nodepool/nodepool_enqueue_handlers_test.go b/pkg/controller/nodepool/nodepool_enqueue_handlers_test.go new file mode 100644 index 00000000000..67ed2a6a706 --- /dev/null +++ b/pkg/controller/nodepool/nodepool_enqueue_handlers_test.go @@ -0,0 +1,319 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodepool + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/event" + + "github.com/openyurtio/openyurt/pkg/apis/apps" +) + +func TestCreate(t *testing.T) { + testcases := map[string]struct { + event event.CreateEvent + wantedNum int + }{ + "add a pod": { + event: event.CreateEvent{ + Object: &corev1.Pod{}, + }, + wantedNum: 0, + }, + "node doesn't belong to a pool": { + event: event.CreateEvent{ + Object: &corev1.Node{}, + }, + wantedNum: 0, + }, + "node belongs to a pool": { + event: event.CreateEvent{ + Object: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "foo", + }, + }, + }, + }, + wantedNum: 1, + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + handler := &EnqueueNodePoolForNode{} + q := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) + handler.Create(tc.event, q) + + if q.Len() != tc.wantedNum { + t.Errorf("Expected %d, got %d", tc.wantedNum, q.Len()) + } + }) + } +} + +func TestUpdate(t *testing.T) { + testcases := map[string]struct { + event event.UpdateEvent + wantedNum int + }{ + "invalid old object": { + event: event.UpdateEvent{ + ObjectOld: &corev1.Pod{}, + ObjectNew: &corev1.Node{}, + }, + wantedNum: 0, + }, + "invalid new object": { + event: event.UpdateEvent{ + ObjectOld: &corev1.Node{}, + ObjectNew: &corev1.Pod{}, + }, + wantedNum: 0, + }, + "update orphan node": { + event: event.UpdateEvent{ + ObjectOld: &corev1.Node{}, + ObjectNew: &corev1.Node{}, + }, + wantedNum: 0, + }, + "add a node into pool": { + event: event.UpdateEvent{ + ObjectOld: &corev1.Node{}, + ObjectNew: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "foo", + }, + }, + }, + }, + wantedNum: 1, + }, + "pool of node is changed": { + event: event.UpdateEvent{ + ObjectOld: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "foo", + }, + }, + }, + ObjectNew: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "bar", + }, + }, + }, + }, + wantedNum: 0, + }, + "pool of node is not changed": { + event: event.UpdateEvent{ + ObjectOld: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "foo", + }, + }, + }, + ObjectNew: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "foo", + }, + }, + }, + }, + wantedNum: 0, + }, + "node ready status is changed": { + event: event.UpdateEvent{ + ObjectOld: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "foo", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionFalse, + }, + }, + }, + }, + ObjectNew: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "foo", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + }, + wantedNum: 1, + }, + "node ready status is not changed": { + event: event.UpdateEvent{ + ObjectOld: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "foo", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + ObjectNew: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "foo", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + }, + wantedNum: 0, + }, + "node labels is changed": { + event: event.UpdateEvent{ + ObjectOld: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "foo", + "label1": "value1", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + ObjectNew: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "foo", + "label2": "value2", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + }, + wantedNum: 1, + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + handler := &EnqueueNodePoolForNode{ + EnableSyncNodePoolConfigurations: true, + Recorder: record.NewFakeRecorder(100), + } + q := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) + handler.Update(tc.event, q) + + if q.Len() != tc.wantedNum { + t.Errorf("Expected %d, got %d", tc.wantedNum, q.Len()) + } + }) + } +} + +func TestDelete(t *testing.T) { + testcases := map[string]struct { + event event.DeleteEvent + wantedNum int + }{ + "delete a pod": { + event: event.DeleteEvent{ + Object: &corev1.Pod{}, + }, + wantedNum: 0, + }, + "delete a orphan node": { + event: event.DeleteEvent{ + Object: &corev1.Node{}, + }, + wantedNum: 0, + }, + "delete a pool node": { + event: event.DeleteEvent{ + Object: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "foo", + }, + }, + }, + }, + wantedNum: 1, + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + handler := &EnqueueNodePoolForNode{} + q := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) + handler.Delete(tc.event, q) + + if q.Len() != tc.wantedNum { + t.Errorf("Expected %d, got %d", tc.wantedNum, q.Len()) + } + }) + } +} diff --git a/pkg/controller/nodepool/util.go b/pkg/controller/nodepool/util.go index fc8b69ad333..dd816cde1d8 100644 --- a/pkg/controller/nodepool/util.go +++ b/pkg/controller/nodepool/util.go @@ -17,174 +17,44 @@ limitations under the License. package nodepool import ( - "context" "encoding/json" "reflect" "sort" - "time" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/util/workqueue" - "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/openyurtio/openyurt/pkg/apis/apps" appsv1beta1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" nodeutil "github.com/openyurtio/openyurt/pkg/controller/util/node" ) -var timeSleep = time.Sleep - -// createNodePool creates an nodepool, it will retry 5 times if it fails -func createNodePool(c client.Client, name string, - poolType appsv1beta1.NodePoolType) bool { - for i := 0; i < 5; i++ { - np := appsv1beta1.NodePool{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Spec: appsv1beta1.NodePoolSpec{ - Type: poolType, - }, - } - err := c.Create(context.TODO(), &np) - if err == nil { - klog.V(4).Infof("the default nodepool(%s) is created", name) - return true - } - if apierrors.IsAlreadyExists(err) { - klog.V(4).Infof("the default nodepool(%s) already exist", name) - return false - } - klog.Errorf("fail to create the node pool(%s): %s", name, err) - timeSleep(2 * time.Second) - } - klog.V(4).Info("fail to create the default nodepool after trying for 5 times") - return false -} - -// createDefaultNodePool creates the default NodePool if not exist -func createDefaultNodePool(client client.Client) { - createNodePool(client, - apps.DefaultEdgeNodePoolName, appsv1beta1.Edge) - createNodePool(client, - apps.DefaultCloudNodePoolName, appsv1beta1.Cloud) -} - // conciliatePoolRelatedAttrs will update the node's attributes that related to // the nodepool -func concilateNode(node *corev1.Node, nodePool appsv1beta1.NodePool) (attrUpdated bool, err error) { +func conciliateNode(node *corev1.Node, nodePool *appsv1beta1.NodePool) (bool, error) { // update node attr - npra := NodePoolRelatedAttributes{ + newNpra := &NodePoolRelatedAttributes{ Labels: nodePool.Spec.Labels, Annotations: nodePool.Spec.Annotations, Taints: nodePool.Spec.Taints, } - if preAttrs, exist := node.Annotations[apps.AnnotationPrevAttrs]; !exist { - node.Labels = mergeMap(node.Labels, npra.Labels) - node.Annotations = mergeMap(node.Annotations, npra.Annotations) - for _, npt := range npra.Taints { - for i, nt := range node.Spec.Taints { - if npt.Effect == nt.Effect && npt.Key == nt.Key { - node.Spec.Taints = append(node.Spec.Taints[:i], node.Spec.Taints[i+1:]...) - break - } - } - node.Spec.Taints = append(node.Spec.Taints, npt) - } - if err := cachePrevPoolAttrs(node, npra); err != nil { - return attrUpdated, err - } - attrUpdated = true - } else { - var preNpra NodePoolRelatedAttributes - if err := json.Unmarshal([]byte(preAttrs), &preNpra); err != nil { - return attrUpdated, err - } - if !reflect.DeepEqual(preNpra, npra) { - // pool related attributes will be updated - conciliateLabels(node, preNpra.Labels, npra.Labels) - conciliateAnnotations(node, preNpra.Annotations, npra.Annotations) - conciliateTaints(node, preNpra.Taints, npra.Taints) - if err := cachePrevPoolAttrs(node, npra); err != nil { - return attrUpdated, err - } - attrUpdated = true - } - } - - // update ownerLabel - if node.Labels[apps.LabelCurrentNodePool] != nodePool.GetName() { - if len(node.Labels) == 0 { - node.Labels = make(map[string]string) - } - node.Labels[apps.LabelCurrentNodePool] = nodePool.GetName() - attrUpdated = true - } - return attrUpdated, nil -} - -// getRemovedNodes calculates removed nodes from current nodes and desired nodes -func getRemovedNodes(currentNodeList *corev1.NodeList, desiredNodeList *corev1.NodeList) []corev1.Node { - var removedNodes []corev1.Node - for _, mNode := range currentNodeList.Items { - var found bool - for _, dNode := range desiredNodeList.Items { - if mNode.GetName() == dNode.GetName() { - found = true - break - } - } - if !found { - removedNodes = append(removedNodes, mNode) - } - } - return removedNodes -} - -// removePoolRelatedAttrs removes attributes(label/annotation/taint) that -// relate to nodepool -func removePoolRelatedAttrs(node *corev1.Node) error { - var npra NodePoolRelatedAttributes - - if _, exist := node.Annotations[apps.AnnotationPrevAttrs]; !exist { - return nil - } - - if err := json.Unmarshal( - []byte(node.Annotations[apps.AnnotationPrevAttrs]), - &npra); err != nil { - return err - } - - for lk, lv := range npra.Labels { - if node.Labels[lk] == lv { - delete(node.Labels, lk) - } - } - - for ak, av := range npra.Annotations { - if node.Annotations[ak] == av { - delete(node.Annotations, ak) - } + oldNpra, err := decodePoolAttrs(node) + if err != nil { + return false, err } - for _, t := range npra.Taints { - if i, exist := containTaint(t, node.Spec.Taints); exist { - node.Spec.Taints = append( - node.Spec.Taints[:i], - node.Spec.Taints[i+1:]...) + if !areNodePoolRelatedAttributesEqual(oldNpra, newNpra) { + //klog.Infof("oldNpra: %#+v, \n newNpra: %#+v", oldNpra, newNpra) + conciliateLabels(node, oldNpra.Labels, newNpra.Labels) + conciliateAnnotations(node, oldNpra.Annotations, newNpra.Annotations) + conciliateTaints(node, oldNpra.Taints, newNpra.Taints) + if err := encodePoolAttrs(node, newNpra); err != nil { + return false, err } + return true, nil } - delete(node.Annotations, apps.AnnotationPrevAttrs) - delete(node.Labels, apps.LabelCurrentNodePool) - return nil + return false, nil } // conciliateLabels will update the node's label that related to the nodepool @@ -219,17 +89,18 @@ func conciliateAnnotations(node *corev1.Node, oldAnnos, newAnnos map[string]stri // conciliateLabels will update the node's taint that related to the nodepool func conciliateTaints(node *corev1.Node, oldTaints, newTaints []corev1.Taint) { - // 1. remove taints from the node if they have been removed from the - // node pool + // 1. remove old taints from the node for _, oldTaint := range oldTaints { if _, exist := containTaint(oldTaint, node.Spec.Taints); exist { node.Spec.Taints = removeTaint(oldTaint, node.Spec.Taints) } } - // 2. update the node taints based on the latest node pool taints + // 2. add new node taints based on the latest node pool taints for _, nt := range newTaints { - node.Spec.Taints = append(node.Spec.Taints, nt) + if _, exist := containTaint(nt, node.Spec.Taints); !exist { + node.Spec.Taints = append(node.Spec.Taints, nt) + } } } @@ -253,7 +124,7 @@ func conciliateNodePoolStatus( // update the node list on demand sort.Strings(nodes) sort.Strings(nodePool.Status.Nodes) - if !reflect.DeepEqual(nodes, nodePool.Status.Nodes) { + if !(len(nodes) == 0 && len(nodePool.Status.Nodes) == 0 || reflect.DeepEqual(nodes, nodePool.Status.Nodes)) { nodePool.Status.Nodes = nodes needUpdate = true } @@ -301,10 +172,10 @@ func removeTaint(taint corev1.Taint, taints []corev1.Taint) []corev1.Taint { return taints } -// cachePrevPoolAttrs caches the nodepool-related attributes to the +// encodePoolAttrs caches the nodepool-related attributes to the // node's annotation -func cachePrevPoolAttrs(node *corev1.Node, - npra NodePoolRelatedAttributes) error { +func encodePoolAttrs(node *corev1.Node, + npra *NodePoolRelatedAttributes) error { npraJson, err := json.Marshal(npra) if err != nil { return err @@ -316,10 +187,29 @@ func cachePrevPoolAttrs(node *corev1.Node, return nil } -// addNodePoolToWorkQueue adds the nodepool the reconciler's workqueue -func addNodePoolToWorkQueue(npName string, - q workqueue.RateLimitingInterface) { - q.Add(reconcile.Request{ - NamespacedName: types.NamespacedName{Name: npName}, - }) +// decodePoolAttrs resolves nodepool attributes from node annotation +func decodePoolAttrs(node *corev1.Node) (*NodePoolRelatedAttributes, error) { + var oldNpra NodePoolRelatedAttributes + if preAttrs, exist := node.Annotations[apps.AnnotationPrevAttrs]; !exist { + return &oldNpra, nil + } else { + if err := json.Unmarshal([]byte(preAttrs), &oldNpra); err != nil { + return &oldNpra, err + } + + return &oldNpra, nil + } +} + +// areNodePoolRelatedAttributesEqual is used for checking NodePoolRelatedAttributes is equal +func areNodePoolRelatedAttributesEqual(a, b *NodePoolRelatedAttributes) bool { + if a == nil || b == nil { + return a == b + } + + isLabelsEqual := (len(a.Labels) == 0 && len(b.Labels) == 0) || reflect.DeepEqual(a.Labels, b.Labels) + isAnnotationsEqual := (len(a.Annotations) == 0 && len(b.Annotations) == 0) || reflect.DeepEqual(a.Annotations, b.Annotations) + isTaintsEqual := (len(a.Taints) == 0 && len(b.Taints) == 0) || reflect.DeepEqual(a.Taints, b.Taints) + + return isLabelsEqual && isAnnotationsEqual && isTaintsEqual } diff --git a/pkg/controller/nodepool/util_test.go b/pkg/controller/nodepool/util_test.go new file mode 100644 index 00000000000..8afb6065506 --- /dev/null +++ b/pkg/controller/nodepool/util_test.go @@ -0,0 +1,859 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodepool + +import ( + "encoding/json" + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/openyurtio/openyurt/pkg/apis/apps" + appsv1beta1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" +) + +func TestConcilateNode(t *testing.T) { + testcases := map[string]struct { + initNpra *NodePoolRelatedAttributes + mockNode corev1.Node + pool appsv1beta1.NodePool + wantedNodeExcludeAttribute corev1.Node + updated bool + }{ + "node has no pool attributes": { + mockNode: corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + Annotations: map[string]string{ + "anno1": "value1", + }, + }, + Spec: corev1.NodeSpec{ + Taints: []corev1.Taint{ + { + Key: "key1", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + }, + }, + pool: appsv1beta1.NodePool{ + Spec: appsv1beta1.NodePoolSpec{ + Labels: map[string]string{ + "poollabel1": "value1", + "poollabel2": "value2", + }, + Annotations: map[string]string{ + "poolanno1": "value1", + "poolanno2": "value2", + }, + Taints: []corev1.Taint{ + { + Key: "poolkey1", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + }, + }, + wantedNodeExcludeAttribute: corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + "poollabel1": "value1", + "poollabel2": "value2", + }, + Annotations: map[string]string{ + "anno1": "value1", + "poolanno1": "value1", + "poolanno2": "value2", + }, + }, + Spec: corev1.NodeSpec{ + Taints: []corev1.Taint{ + { + Key: "key1", + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "poolkey1", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + }, + }, + updated: true, + }, + "node has some pool attributes": { + initNpra: &NodePoolRelatedAttributes{ + Labels: map[string]string{ + "label2": "value2", + }, + Annotations: map[string]string{ + "anno2": "value2", + "anno3": "value3", + }, + Taints: []corev1.Taint{ + { + Key: "key1", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + }, + mockNode: corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + Annotations: map[string]string{ + "anno1": "value1", + "anno2": "value2", + "anno3": "value3", + }, + }, + Spec: corev1.NodeSpec{ + Taints: []corev1.Taint{ + { + Key: "key1", + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "key2", + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "key3", + Effect: corev1.TaintEffectNoExecute, + }, + }, + }, + }, + pool: appsv1beta1.NodePool{ + Spec: appsv1beta1.NodePoolSpec{ + Labels: map[string]string{ + "poollabel1": "value1", + "poollabel2": "value2", + }, + Annotations: map[string]string{ + "poolanno1": "value1", + "poolanno2": "value2", + }, + Taints: []corev1.Taint{ + { + Key: "poolkey1", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + }, + }, + wantedNodeExcludeAttribute: corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "label1": "value1", + "poollabel1": "value1", + "poollabel2": "value2", + }, + Annotations: map[string]string{ + "anno1": "value1", + "poolanno1": "value1", + "poolanno2": "value2", + }, + }, + Spec: corev1.NodeSpec{ + Taints: []corev1.Taint{ + { + Key: "key2", + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "key3", + Effect: corev1.TaintEffectNoExecute, + }, + { + Key: "poolkey1", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + }, + }, + updated: true, + }, + "pool attributes is not changed": { + initNpra: &NodePoolRelatedAttributes{ + Labels: map[string]string{ + "label2": "value2", + }, + Annotations: map[string]string{ + "anno2": "value2", + "anno3": "value3", + }, + Taints: []corev1.Taint{ + { + Key: "key1", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + }, + mockNode: corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + Annotations: map[string]string{ + "anno1": "value1", + "anno2": "value2", + "anno3": "value3", + }, + }, + Spec: corev1.NodeSpec{ + Taints: []corev1.Taint{ + { + Key: "key1", + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "key2", + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "key3", + Effect: corev1.TaintEffectNoExecute, + }, + }, + }, + }, + pool: appsv1beta1.NodePool{ + Spec: appsv1beta1.NodePoolSpec{ + Labels: map[string]string{ + "label2": "value2", + }, + Annotations: map[string]string{ + "anno2": "value2", + "anno3": "value3", + }, + Taints: []corev1.Taint{ + { + Key: "key1", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + }, + }, + wantedNodeExcludeAttribute: corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + Annotations: map[string]string{ + "anno1": "value1", + "anno2": "value2", + "anno3": "value3", + }, + }, + Spec: corev1.NodeSpec{ + Taints: []corev1.Taint{ + { + Key: "key1", + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "key2", + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "key3", + Effect: corev1.TaintEffectNoExecute, + }, + }, + }, + }, + updated: false, + }, + "pool has some duplicated attributes": { + mockNode: corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + Annotations: map[string]string{ + "anno1": "value1", + "anno2": "value2", + }, + }, + Spec: corev1.NodeSpec{ + Taints: []corev1.Taint{ + { + Key: "key1", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + }, + }, + pool: appsv1beta1.NodePool{ + Spec: appsv1beta1.NodePoolSpec{ + Labels: map[string]string{ + "label2": "value2", + "poollabel2": "value2", + }, + Annotations: map[string]string{ + "anno1": "value1", + }, + Taints: []corev1.Taint{ + { + Key: "key1", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + }, + }, + wantedNodeExcludeAttribute: corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + "poollabel2": "value2", + }, + Annotations: map[string]string{ + "anno1": "value1", + "anno2": "value2", + }, + }, + Spec: corev1.NodeSpec{ + Taints: []corev1.Taint{ + { + Key: "key1", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + }, + }, + updated: true, + }, + "node and pool has no pool attributes": { + mockNode: corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + Annotations: map[string]string{ + "anno1": "value1", + }, + }, + Spec: corev1.NodeSpec{ + Taints: []corev1.Taint{ + { + Key: "key1", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + }, + }, + pool: appsv1beta1.NodePool{ + Spec: appsv1beta1.NodePoolSpec{ + Labels: map[string]string{}, + Annotations: map[string]string{}, + Taints: []corev1.Taint{}, + }, + }, + wantedNodeExcludeAttribute: corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + Annotations: map[string]string{ + "anno1": "value1", + }, + }, + Spec: corev1.NodeSpec{ + Taints: []corev1.Taint{ + { + Key: "key1", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + }, + }, + updated: false, + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + if tc.initNpra != nil { + if err := encodePoolAttrs(&tc.mockNode, tc.initNpra); err != nil { + t.Errorf("Expected no error, got %v", err) + } + } + + changed, err := conciliateNode(&tc.mockNode, &tc.pool) + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + + if tc.updated != changed { + t.Errorf("Expected %v, got %v, node %#+v", tc.updated, changed, tc.mockNode) + } + + wantedNpra := NodePoolRelatedAttributes{ + Labels: tc.pool.Spec.Labels, + Annotations: tc.pool.Spec.Annotations, + Taints: tc.pool.Spec.Taints, + } + npra, err := decodePoolAttrs(&tc.mockNode) + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + if !areNodePoolRelatedAttributesEqual(npra, &wantedNpra) { + t.Errorf("Expected %v, got %v", wantedNpra, *npra) + } + + delete(tc.mockNode.Annotations, apps.AnnotationPrevAttrs) + if !reflect.DeepEqual(tc.mockNode, tc.wantedNodeExcludeAttribute) { + t.Errorf("Expected %v, got %v", tc.wantedNodeExcludeAttribute, tc.mockNode) + } + }) + } +} + +func TestConciliateLabels(t *testing.T) { + mockNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + }, + } + + oldLabels := map[string]string{ + "label1": "value1", + } + newLabels := map[string]string{ + "label3": "value3", + "label4": "value4", + } + + wantNodeLabels := map[string]string{ + "label2": "value2", + "label3": "value3", + "label4": "value4", + } + + conciliateLabels(mockNode, oldLabels, newLabels) + if !reflect.DeepEqual(wantNodeLabels, mockNode.Labels) { + t.Errorf("Expected %v, got %v", wantNodeLabels, mockNode.Labels) + } +} + +func TestConciliateAnnotations(t *testing.T) { + mockNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "anno1": "value1", + "anno2": "value2", + }, + }, + } + + oldAnnos := map[string]string{ + "anno1": "value1", + } + newAnnos := map[string]string{ + "anno3": "value3", + "anno4": "value4", + } + + wantNodeAnnos := map[string]string{ + "anno2": "value2", + "anno3": "value3", + "anno4": "value4", + } + + conciliateAnnotations(mockNode, oldAnnos, newAnnos) + if !reflect.DeepEqual(wantNodeAnnos, mockNode.Annotations) { + t.Errorf("Expected %v, got %v", wantNodeAnnos, mockNode.Annotations) + } +} + +func TestConciliateTaints(t *testing.T) { + mockNode := &corev1.Node{ + Spec: corev1.NodeSpec{ + Taints: []corev1.Taint{ + { + Key: "key1", + Value: "value1", + Effect: corev1.TaintEffectNoExecute, + }, + { + Key: "key2", + Value: "value2", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + }, + } + + // Test case where oldTaint is present in the node and should be removed + oldTaints := []corev1.Taint{ + { + Key: "key1", + Value: "value1", + Effect: corev1.TaintEffectNoExecute, + }, + } + newTaints := []corev1.Taint{ + { + Key: "key3", + Value: "value3", + Effect: corev1.TaintEffectPreferNoSchedule, + }, + } + wantNodeTaints := []corev1.Taint{ + { + Key: "key2", + Value: "value2", + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "key3", + Value: "value3", + Effect: corev1.TaintEffectPreferNoSchedule, + }, + } + conciliateTaints(mockNode, oldTaints, newTaints) + + if !reflect.DeepEqual(wantNodeTaints, mockNode.Spec.Taints) { + t.Errorf("Expected %v, got %v", wantNodeTaints, mockNode.Spec.Taints) + } +} + +func TestConciliateNodePoolStatus(t *testing.T) { + testcases := map[string]struct { + readyNodes int32 + notReadyNodes int32 + nodes []string + pool *appsv1beta1.NodePool + needUpdated bool + }{ + "status is needed to update": { + readyNodes: 5, + notReadyNodes: 2, + nodes: []string{"foo", "bar", "cat", "zxxde"}, + pool: &appsv1beta1.NodePool{ + Status: appsv1beta1.NodePoolStatus{ + ReadyNodeNum: 2, + UnreadyNodeNum: 3, + Nodes: []string{"foo", "bar", "cat", "zxxde", "lucky"}, + }, + }, + needUpdated: true, + }, + "status is not updated": { + readyNodes: 2, + notReadyNodes: 2, + nodes: []string{"foo", "bar", "cat", "zxxde"}, + pool: &appsv1beta1.NodePool{ + Status: appsv1beta1.NodePoolStatus{ + ReadyNodeNum: 2, + UnreadyNodeNum: 2, + Nodes: []string{"foo", "bar", "cat", "zxxde"}, + }, + }, + needUpdated: false, + }, + "status is not updated when pool is empty": { + readyNodes: 0, + notReadyNodes: 0, + nodes: []string{}, + pool: &appsv1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Status: appsv1beta1.NodePoolStatus{ + ReadyNodeNum: 0, + UnreadyNodeNum: 0, + Nodes: []string{}, + }, + }, + needUpdated: false, + }, + "status is not updated when pool has no status": { + readyNodes: 0, + notReadyNodes: 0, + nodes: []string{}, + pool: &appsv1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + }, + needUpdated: false, + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + needUpdated := conciliateNodePoolStatus(tc.readyNodes, tc.notReadyNodes, tc.nodes, tc.pool) + if needUpdated != tc.needUpdated { + t.Errorf("Expected %v, got %v", tc.needUpdated, needUpdated) + } + }) + } +} + +func TestContainTaint(t *testing.T) { + mockTaints := []corev1.Taint{ + { + Key: "key1", + Value: "value1", + Effect: corev1.TaintEffectNoExecute, + }, + { + Key: "key2", + Value: "value2", + Effect: corev1.TaintEffectNoSchedule, + }, + } + testcases := map[string]struct { + inputTaint corev1.Taint + resultIndex int + isContained bool + }{ + "taint is contained": { + inputTaint: corev1.Taint{ + Key: "key1", + Value: "value1", + Effect: corev1.TaintEffectNoExecute, + }, + resultIndex: 0, + isContained: true, + }, + "taint is not contained": { + inputTaint: corev1.Taint{ + Key: "key3", + Value: "value3", + Effect: corev1.TaintEffectPreferNoSchedule, + }, + resultIndex: 0, + isContained: false, + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + gotIndex, gotBool := containTaint(tc.inputTaint, mockTaints) + if gotIndex != tc.resultIndex || gotBool != tc.isContained { + t.Errorf("Expected index %v and bool %v, got index %v and bool %v", tc.resultIndex, tc.isContained, gotIndex, gotBool) + } + }) + } +} + +func TestRemoveTaint(t *testing.T) { + mockTaints := []corev1.Taint{ + { + Key: "key1", + Value: "value1", + Effect: corev1.TaintEffectNoExecute, + }, + { + Key: "key2", + Value: "value2", + Effect: corev1.TaintEffectNoSchedule, + }, + } + + testcases := map[string]struct { + mockTaint corev1.Taint + wantTaints []corev1.Taint + }{ + "remove exist taint": { + mockTaint: corev1.Taint{ + Key: "key1", + Value: "value1", + Effect: corev1.TaintEffectNoExecute, + }, + wantTaints: []corev1.Taint{ + { + Key: "key2", + Value: "value2", + Effect: corev1.TaintEffectNoSchedule, + }, + }, + }, + "remove not exist taint": { + mockTaint: corev1.Taint{ + Key: "key3", + Value: "value3", + Effect: corev1.TaintEffectPreferNoSchedule, + }, + wantTaints: mockTaints, + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + gotTaints := removeTaint(tc.mockTaint, mockTaints) + if !reflect.DeepEqual(tc.wantTaints, gotTaints) { + t.Errorf("Expected %v, got %v", tc.wantTaints, gotTaints) + } + }) + } +} + +func TestEncodePoolAttrs(t *testing.T) { + testcases := map[string]struct { + mockNode *corev1.Node + mockNpra *NodePoolRelatedAttributes + }{ + "annotations is not set": { + mockNode: &corev1.Node{}, + mockNpra: &NodePoolRelatedAttributes{ + Labels: map[string]string{ + "foo": "bar", + }, + }, + }, + "annotations is set": { + mockNode: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "foo": "bar", + }, + }, + }, + mockNpra: &NodePoolRelatedAttributes{ + Labels: map[string]string{ + "foo": "bar", + }, + }, + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + gotErr := encodePoolAttrs(tc.mockNode, tc.mockNpra) + if gotErr != nil { + t.Errorf("Expected no error, got %v", gotErr) + } + + // Ensure that the NodePoolRelatedAttributes has been correctly stored in the node's annotations + gotNpra, err := decodePoolAttrs(tc.mockNode) + if err != nil || !reflect.DeepEqual(gotNpra, tc.mockNpra) { + t.Errorf("Expected %v, got %v", tc.mockNpra, gotNpra) + } + }) + } +} + +func TestDecodePoolAttrs(t *testing.T) { + wantNpra := &NodePoolRelatedAttributes{ + Labels: map[string]string{ + "foo": "bar", + }, + } + npraJson, err := json.Marshal(wantNpra) + if err != nil { + t.Errorf("failed to marshal npra") + } + + mockNode := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + apps.AnnotationPrevAttrs: string(npraJson), + }, + }, + } + gotNpra, gotErr := decodePoolAttrs(mockNode) + + if gotErr != nil { + t.Errorf("Expected no error, got %v", gotErr) + } + + if !reflect.DeepEqual(wantNpra, gotNpra) { + t.Errorf("Expected %v, got %v", wantNpra, gotNpra) + } +} + +func TestIsNodeReady(t *testing.T) { + tests := []struct { + name string + node corev1.Node + want bool + }{ + { + name: "NodeReady and ConditionTrue", + node: corev1.Node{ + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + want: true, + }, + { + name: "NodeReady but ConditionFalse", + node: corev1.Node{ + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionFalse, + }, + }, + }, + }, + want: false, + }, + { + name: "Node status not NodeReady", + node: corev1.Node{ + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeMemoryPressure, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isNodeReady(tt.node); got != tt.want { + t.Errorf("isNodeReady() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/webhook/node/v1/node_default.go b/pkg/webhook/node/v1/node_default.go new file mode 100644 index 00000000000..ed352367a67 --- /dev/null +++ b/pkg/webhook/node/v1/node_default.go @@ -0,0 +1,61 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "context" + "fmt" + "strings" + + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/openyurtio/openyurt/pkg/apis/apps" + appsv1beta1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" +) + +// Default satisfies the defaulting webhook interface. +func (webhook *NodeHandler) Default(ctx context.Context, obj runtime.Object, req admission.Request) error { + node, ok := obj.(*v1.Node) + if !ok { + return apierrors.NewBadRequest(fmt.Sprintf("expected a Node but got a %T", obj)) + } + + npName, ok := node.Labels[apps.NodePoolLabel] + if !ok { + return nil + } + + var np appsv1beta1.NodePool + if err := webhook.Client.Get(ctx, types.NamespacedName{Name: npName}, &np); err != nil { + return err + } + + // add NodePool.Spec.Type to node labels + if node.Labels == nil { + node.Labels = make(map[string]string) + } + node.Labels[apps.NodePoolTypeLabel] = strings.ToLower(string(np.Spec.Type)) + + if np.Spec.HostNetwork { + node.Labels[apps.NodePoolHostNetworkLabel] = "true" + } + return nil +} diff --git a/pkg/webhook/node/v1/node_default_test.go b/pkg/webhook/node/v1/node_default_test.go new file mode 100644 index 00000000000..76d0679bf3c --- /dev/null +++ b/pkg/webhook/node/v1/node_default_test.go @@ -0,0 +1,125 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "context" + "net/http" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/openyurtio/openyurt/pkg/apis" + "github.com/openyurtio/openyurt/pkg/apis/apps" + appsv1beta1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" +) + +func TestDefault(t *testing.T) { + testcases := map[string]struct { + node runtime.Object + pool *appsv1beta1.NodePool + errCode int + errMsg string + }{ + "it is not a node": { + node: &corev1.Pod{}, + errCode: http.StatusBadRequest, + }, + "it is a orphan node": { + node: &corev1.Node{}, + errCode: 0, + }, + "nodepool doesn't exist": { + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Labels: map[string]string{ + apps.NodePoolLabel: "hangzhou", + }, + }, + }, + errCode: 0, + errMsg: "not found", + }, + "add labels for node": { + node: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Labels: map[string]string{ + apps.NodePoolLabel: "shanghai", + }, + }, + }, + pool: &appsv1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "shanghai", + }, + Spec: appsv1beta1.NodePoolSpec{ + Type: appsv1beta1.Edge, + HostNetwork: true, + }, + }, + errCode: 0, + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + t.Fatal("Fail to add kubernetes clint-go custom resource") + } + apis.AddToScheme(scheme) + + var c client.Client + if tc.pool != nil { + c = fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(tc.pool).Build() + } else { + c = fakeclient.NewClientBuilder().WithScheme(scheme).Build() + } + + w := &NodeHandler{Client: c} + err := w.Default(context.TODO(), tc.node, admission.Request{}) + if err != nil { + if tc.errCode != 0 { + statusErr := err.(*errors.StatusError) + if tc.errCode != int(statusErr.Status().Code) { + t.Errorf("Expected error code %d, got %v", tc.errCode, err) + return + } + } + + if len(tc.errMsg) != 0 { + if !strings.Contains(err.Error(), tc.errMsg) { + t.Errorf("Expected error msg %s, got %s", tc.errMsg, err.Error()) + return + } + } + } else if tc.errCode != 0 && len(tc.errMsg) != 0 { + t.Errorf("Expected error code %d, errmsg %s, got %v", tc.errCode, tc.errMsg, err) + } + }) + } +} diff --git a/pkg/webhook/node/v1/node_handler.go b/pkg/webhook/node/v1/node_handler.go index bb107988c50..f6cbc5b6aa0 100644 --- a/pkg/webhook/node/v1/node_handler.go +++ b/pkg/webhook/node/v1/node_handler.go @@ -30,7 +30,7 @@ const ( WebhookName = "node" ) -// SetupWebhookWithManager sets up Cluster webhooks. mutate path, validatepath, error +// SetupWebhookWithManager sets up Cluster webhooks. mutate path, validate path, error func (webhook *NodeHandler) SetupWebhookWithManager(mgr ctrl.Manager) (string, string, error) { // init webhook.Client = mgr.GetClient() @@ -43,15 +43,18 @@ func (webhook *NodeHandler) SetupWebhookWithManager(mgr ctrl.Manager) (string, s util.GenerateValidatePath(gvk), builder.WebhookManagedBy(mgr). For(&v1.Node{}). + WithDefaulter(webhook). WithValidator(webhook). Complete() } -// +kubebuilder:webhook:path=/validate-core-openyurt-io-v1-node,mutating=false,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups="",resources=nodes,verbs=update,versions=v1,name=validate.core.v1.node.openyurt.io +// +kubebuilder:webhook:path=/validate-core-openyurt-io-v1-node,mutating=false,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1,groups="",resources=nodes,verbs=update,versions=v1,name=validate.core.v1.node.openyurt.io +// +kubebuilder:webhook:path=/mutate-core-openyurt-io-v1-node,mutating=true,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1,groups="",resources=nodes,verbs=create;update,versions=v1,name=mutate.core.v1.node.openyurt.io -// Cluster implements a validating and defaulting webhook for Cluster. +// NodeHandler implements a validating and defaulting webhook for Cluster. type NodeHandler struct { Client client.Client } +var _ builder.CustomDefaulter = &NodeHandler{} var _ builder.CustomValidator = &NodeHandler{} diff --git a/pkg/webhook/node/v1/node_validation.go b/pkg/webhook/node/v1/node_validation.go index cfc6a0cba61..7ee9f6c1e80 100644 --- a/pkg/webhook/node/v1/node_validation.go +++ b/pkg/webhook/node/v1/node_validation.go @@ -58,21 +58,31 @@ func (webhook *NodeHandler) ValidateDelete(_ context.Context, obj runtime.Object } func validateNodeUpdate(newNode, oldNode *v1.Node, req admission.Request) field.ErrorList { - oldNp := oldNode.Labels[apps.LabelDesiredNodePool] - newNp := newNode.Labels[apps.LabelDesiredNodePool] + oldNp := oldNode.Labels[apps.NodePoolLabel] + newNp := newNode.Labels[apps.NodePoolLabel] + oldNpType := oldNode.Labels[apps.NodePoolTypeLabel] + newNpType := newNode.Labels[apps.NodePoolTypeLabel] + oldNpHostNetwork := oldNode.Labels[apps.NodePoolHostNetworkLabel] + newNpHostNetwork := newNode.Labels[apps.NodePoolHostNetworkLabel] + + var errList field.ErrorList + // it is not allowed to change NodePoolLabel if it has been set + if len(oldNp) != 0 && oldNp != newNp { + errList = append(errList, field.Forbidden(field.NewPath("metadata").Child("labels").Child(apps.NodePoolLabel), "apps.openyurt.io/nodepool can not be changed")) + } - if len(oldNp) == 0 { - return nil + // it is not allowed to change NodePoolTypeLabel if it has been set + if len(oldNpType) != 0 && oldNpType != newNpType { + errList = append(errList, field.Forbidden(field.NewPath("metadata").Child("labels").Child(apps.NodePoolTypeLabel), "nodepool.openyurt.io/type can not be changed")) } - // can not change LabelDesiredNodePool if it has been set - if oldNp != newNp { - return field.ErrorList([]*field.Error{ - field.Forbidden( - field.NewPath("metadata").Child("labels").Child(apps.LabelDesiredNodePool), - "apps.openyurt.io/desired-nodepool can not be changed"), - }) + // it is not allowed to change NodePoolHostNetworkLabel if it has been set + if len(oldNpHostNetwork) != 0 && oldNpHostNetwork != newNpHostNetwork { + errList = append(errList, field.Forbidden(field.NewPath("metadata").Child("labels").Child(apps.NodePoolHostNetworkLabel), "nodepool.openyurt.io/hostnetwork can not be changed")) } + if len(errList) != 0 { + return errList + } return nil } diff --git a/pkg/webhook/node/v1/node_validation_test.go b/pkg/webhook/node/v1/node_validation_test.go new file mode 100644 index 00000000000..6aa88d3d0e8 --- /dev/null +++ b/pkg/webhook/node/v1/node_validation_test.go @@ -0,0 +1,156 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "context" + "net/http" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/openyurtio/openyurt/pkg/apis/apps" +) + +func TestValidateUpdate(t *testing.T) { + testcases := map[string]struct { + oldNode runtime.Object + newNode runtime.Object + errCode int + }{ + "old object is not a node": { + oldNode: &corev1.Pod{}, + newNode: &corev1.Node{}, + errCode: http.StatusBadRequest, + }, + "new object is not a node": { + oldNode: &corev1.Node{}, + newNode: &corev1.Pod{}, + errCode: http.StatusBadRequest, + }, + "node pool is changed": { + oldNode: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "hangzhou", + }, + }, + }, + newNode: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "shanghai", + }, + }, + }, + errCode: http.StatusUnprocessableEntity, + }, + "node pool type is changed": { + oldNode: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "hangzhou", + apps.NodePoolTypeLabel: "edge", + }, + }, + }, + newNode: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "hangzhou", + apps.NodePoolTypeLabel: "cloud", + }, + }, + }, + errCode: http.StatusUnprocessableEntity, + }, + "node pool host network is changed": { + oldNode: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "hangzhou", + apps.NodePoolTypeLabel: "edge", + apps.NodePoolHostNetworkLabel: "true", + }, + }, + }, + newNode: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "hangzhou", + apps.NodePoolTypeLabel: "edge", + apps.NodePoolHostNetworkLabel: "false", + }, + }, + }, + errCode: http.StatusUnprocessableEntity, + }, + "it is a normal node update": { + oldNode: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "hangzhou", + apps.NodePoolTypeLabel: "edge", + apps.NodePoolHostNetworkLabel: "true", + }, + }, + }, + newNode: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "hangzhou", + apps.NodePoolTypeLabel: "edge", + apps.NodePoolHostNetworkLabel: "true", + }, + }, + }, + errCode: 0, + }, + "it is a normal node update without init labels": { + oldNode: &corev1.Node{}, + newNode: &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + apps.NodePoolLabel: "hangzhou", + apps.NodePoolTypeLabel: "edge", + apps.NodePoolHostNetworkLabel: "true", + }, + }, + }, + errCode: 0, + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + h := &NodeHandler{} + err := h.ValidateUpdate(context.TODO(), tc.oldNode, tc.newNode, admission.Request{}) + if tc.errCode == 0 && err != nil { + t.Errorf("Expected error code %d, got %v", tc.errCode, err) + } else if tc.errCode != 0 { + statusErr := err.(*errors.StatusError) + if tc.errCode != int(statusErr.Status().Code) { + t.Errorf("Expected error code %d, got %v", tc.errCode, err) + } + } + }) + } +} diff --git a/pkg/webhook/nodepool/v1alpha1/nodepool_default.go b/pkg/webhook/nodepool/v1alpha1/nodepool_default.go deleted file mode 100644 index bc0887736f3..00000000000 --- a/pkg/webhook/nodepool/v1alpha1/nodepool_default.go +++ /dev/null @@ -1,50 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - "context" - "fmt" - "strings" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - - "github.com/openyurtio/openyurt/pkg/apis/apps" - "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" -) - -// Default satisfies the defaulting webhook interface. -func (webhook *NodePoolHandler) Default(ctx context.Context, obj runtime.Object) error { - np, ok := obj.(*v1alpha1.NodePool) - if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a NodePool but got a %T", obj)) - } - - np.Spec.Selector = &metav1.LabelSelector{ - MatchLabels: map[string]string{apps.LabelCurrentNodePool: np.Name}, - } - - // add NodePool.Spec.Type to NodePool labels - if np.Labels == nil { - np.Labels = make(map[string]string) - } - np.Labels[apps.NodePoolTypeLabelKey] = strings.ToLower(string(np.Spec.Type)) - - return nil -} diff --git a/pkg/webhook/nodepool/v1alpha1/nodepool_handler.go b/pkg/webhook/nodepool/v1alpha1/nodepool_handler.go deleted file mode 100644 index bad4db5540e..00000000000 --- a/pkg/webhook/nodepool/v1alpha1/nodepool_handler.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - "sigs.k8s.io/controller-runtime/pkg/webhook" - - appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/webhook/util" -) - -// SetupWebhookWithManager sets up Cluster webhooks. mutate path, validatepath, error -func (webhook *NodePoolHandler) SetupWebhookWithManager(mgr ctrl.Manager) (string, string, error) { - // init - webhook.Client = mgr.GetClient() - - gvk, err := apiutil.GVKForObject(&appsv1alpha1.NodePool{}, mgr.GetScheme()) - if err != nil { - return "", "", err - } - return util.GenerateMutatePath(gvk), - util.GenerateValidatePath(gvk), - ctrl.NewWebhookManagedBy(mgr). - For(&appsv1alpha1.NodePool{}). - Complete() -} - -// NodePoolHandler implements a validating and defaulting webhook for Cluster. -type NodePoolHandler struct { - Client client.Client -} - -var _ webhook.CustomDefaulter = &NodePoolHandler{} -var _ webhook.CustomValidator = &NodePoolHandler{} diff --git a/pkg/webhook/nodepool/v1alpha1/nodepool_validation.go b/pkg/webhook/nodepool/v1alpha1/nodepool_validation.go deleted file mode 100644 index 383e321844e..00000000000 --- a/pkg/webhook/nodepool/v1alpha1/nodepool_validation.go +++ /dev/null @@ -1,143 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - "context" - "errors" - "fmt" - - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - apivalidation "k8s.io/apimachinery/pkg/api/validation" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/validation/field" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/openyurtio/openyurt/pkg/apis/apps" - appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" -) - -// ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *NodePoolHandler) ValidateCreate(ctx context.Context, obj runtime.Object) error { - np, ok := obj.(*appsv1alpha1.NodePool) - if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a NodePool but got a %T", obj)) - } - - if allErrs := validateNodePoolSpec(&np.Spec); len(allErrs) > 0 { - return apierrors.NewInvalid(appsv1alpha1.GroupVersion.WithKind("NodePool").GroupKind(), np.Name, allErrs) - } - - return nil -} - -// ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *NodePoolHandler) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error { - newNp, ok := newObj.(*appsv1alpha1.NodePool) - if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a NodePool but got a %T", newObj)) - } - oldNp, ok := oldObj.(*appsv1alpha1.NodePool) - if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a NodePool but got a %T", oldObj)) - } - - if allErrs := validateNodePoolSpecUpdate(&newNp.Spec, &oldNp.Spec); len(allErrs) > 0 { - return apierrors.NewInvalid(appsv1alpha1.GroupVersion.WithKind("NodePool").GroupKind(), newNp.Name, allErrs) - } - - return nil -} - -// ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type. -func (webhook *NodePoolHandler) ValidateDelete(_ context.Context, obj runtime.Object) error { - np, ok := obj.(*appsv1alpha1.NodePool) - if !ok { - return apierrors.NewBadRequest(fmt.Sprintf("expected a NodePool but got a %T", obj)) - } - if allErrs := validateNodePoolDeletion(webhook.Client, np); len(allErrs) > 0 { - return apierrors.NewInvalid(appsv1alpha1.GroupVersion.WithKind("NodePool").GroupKind(), np.Name, allErrs) - } - - return nil -} - -// annotationValidator validates the NodePool.Spec.Annotations -var annotationValidator = func(annos map[string]string) error { - errs := apivalidation.ValidateAnnotations(annos, field.NewPath("field")) - if len(errs) > 0 { - return errors.New(errs.ToAggregate().Error()) - } - return nil -} - -func validateNodePoolSpecAnnotations(annotations map[string]string) field.ErrorList { - if err := annotationValidator(annotations); err != nil { - return field.ErrorList([]*field.Error{ - field.Invalid(field.NewPath("spec").Child("annotations"), - annotations, "invalid annotations")}) - } - return nil -} - -// validateNodePoolSpec validates the nodepool spec. -func validateNodePoolSpec(spec *appsv1alpha1.NodePoolSpec) field.ErrorList { - if allErrs := validateNodePoolSpecAnnotations(spec.Annotations); allErrs != nil { - return allErrs - } - return nil -} - -// validateNodePoolSpecUpdate tests if required fields in the NodePool spec are set. -func validateNodePoolSpecUpdate(spec, oldSpec *appsv1alpha1.NodePoolSpec) field.ErrorList { - if allErrs := validateNodePoolSpec(spec); allErrs != nil { - return allErrs - } - - if spec.Type != oldSpec.Type { - return field.ErrorList([]*field.Error{ - field.Invalid(field.NewPath("spec").Child("type"), - spec.Annotations, "pool type can't be changed")}) - } - return nil -} - -// validateNodePoolDeletion validate the nodepool deletion event, which prevents -// the default-nodepool from being deleted -func validateNodePoolDeletion(cli client.Client, np *appsv1alpha1.NodePool) field.ErrorList { - nodes := corev1.NodeList{} - - if np.Name == apps.DefaultCloudNodePoolName || np.Name == apps.DefaultEdgeNodePoolName { - return field.ErrorList([]*field.Error{ - field.Forbidden(field.NewPath("metadata").Child("name"), - fmt.Sprintf("default nodepool %s forbidden to delete", np.Name))}) - } - - if err := cli.List(context.TODO(), &nodes, - client.MatchingLabels(np.Spec.Selector.MatchLabels)); err != nil { - return field.ErrorList([]*field.Error{ - field.Forbidden(field.NewPath("metadata").Child("name"), - "fail to get nodes associated to the pool")}) - } - if len(nodes.Items) != 0 { - return field.ErrorList([]*field.Error{ - field.Forbidden(field.NewPath("metadata").Child("name"), - "cannot remove nonempty pool, please drain the pool before deleting")}) - } - return nil -} diff --git a/pkg/webhook/nodepool/v1beta1/nodepool_default.go b/pkg/webhook/nodepool/v1beta1/nodepool_default.go index bf05d2f0a65..cd3c081f439 100644 --- a/pkg/webhook/nodepool/v1beta1/nodepool_default.go +++ b/pkg/webhook/nodepool/v1beta1/nodepool_default.go @@ -19,13 +19,10 @@ package v1beta1 import ( "context" "fmt" - "strings" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "github.com/openyurtio/openyurt/pkg/apis/apps" "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" ) @@ -36,15 +33,17 @@ func (webhook *NodePoolHandler) Default(ctx context.Context, obj runtime.Object) return apierrors.NewBadRequest(fmt.Sprintf("expected a NodePool but got a %T", obj)) } - np.Spec.Selector = &metav1.LabelSelector{ - MatchLabels: map[string]string{apps.LabelCurrentNodePool: np.Name}, + // specify default type as Edge + if len(np.Spec.Type) == 0 { + np.Spec.Type = v1beta1.Edge } - // add NodePool.Spec.Type to NodePool labels - if np.Labels == nil { - np.Labels = make(map[string]string) + // init node pool status + np.Status = v1beta1.NodePoolStatus{ + ReadyNodeNum: 0, + UnreadyNodeNum: 0, + Nodes: make([]string, 0), } - np.Labels[apps.NodePoolTypeLabelKey] = strings.ToLower(string(np.Spec.Type)) return nil } diff --git a/pkg/webhook/nodepool/v1beta1/nodepool_default_test.go b/pkg/webhook/nodepool/v1beta1/nodepool_default_test.go new file mode 100644 index 00000000000..100bbe840a1 --- /dev/null +++ b/pkg/webhook/nodepool/v1beta1/nodepool_default_test.go @@ -0,0 +1,110 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" +) + +func TestDefault(t *testing.T) { + testcases := map[string]struct { + obj runtime.Object + errHappened bool + wantedNodePool *v1beta1.NodePool + }{ + "it is not a nodepool": { + obj: &corev1.Pod{}, + errHappened: true, + }, + "nodepool has no type": { + obj: &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: v1beta1.NodePoolSpec{ + HostNetwork: true, + }, + }, + wantedNodePool: &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: v1beta1.NodePoolSpec{ + HostNetwork: true, + Type: v1beta1.Edge, + }, + Status: v1beta1.NodePoolStatus{ + ReadyNodeNum: 0, + UnreadyNodeNum: 0, + Nodes: []string{}, + }, + }, + }, + "nodepool has pool type": { + obj: &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: v1beta1.NodePoolSpec{ + HostNetwork: true, + Type: v1beta1.Cloud, + }, + }, + wantedNodePool: &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: v1beta1.NodePoolSpec{ + HostNetwork: true, + Type: v1beta1.Cloud, + }, + Status: v1beta1.NodePoolStatus{ + ReadyNodeNum: 0, + UnreadyNodeNum: 0, + Nodes: []string{}, + }, + }, + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + h := NodePoolHandler{} + err := h.Default(context.TODO(), tc.obj) + if tc.errHappened { + if err == nil { + t.Errorf("expect error, got nil") + } + } else if err != nil { + t.Errorf("expect no error, but got %v", err) + } else { + currentNp := tc.obj.(*v1beta1.NodePool) + if !reflect.DeepEqual(currentNp, tc.wantedNodePool) { + t.Errorf("expect %#+v, got %#+v", tc.wantedNodePool, currentNp) + } + } + }) + } +} diff --git a/pkg/webhook/nodepool/v1beta1/nodepool_handler.go b/pkg/webhook/nodepool/v1beta1/nodepool_handler.go index de606d8988c..d1fa643d5f7 100644 --- a/pkg/webhook/nodepool/v1beta1/nodepool_handler.go +++ b/pkg/webhook/nodepool/v1beta1/nodepool_handler.go @@ -44,8 +44,8 @@ func (webhook *NodePoolHandler) SetupWebhookWithManager(mgr ctrl.Manager) (strin Complete() } -// +kubebuilder:webhook:verbs=create;update;delete,path=/validate-apps-openyurt-io-v1beta1-nodepool,mutating=false,failurePolicy=fail,groups=apps.openyurt.io,resources=nodepools,versions=v1beta1,name=v.v1beta1.nodepool.kb.io,sideEffects=None,admissionReviewVersions=v1 -// +kubebuilder:webhook:path=/mutate-apps-openyurt-io-v1beta1-nodepool,mutating=true,failurePolicy=fail,groups=apps.openyurt.io,resources=nodepools,verbs=create;update,versions=v1beta1,name=m.v1beta1.nodepool.kb.io,sideEffects=None,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/validate-apps-openyurt-io-v1beta1-nodepool,mutating=false,failurePolicy=fail,groups=apps.openyurt.io,resources=nodepools,verbs=create;update;delete,versions=v1beta1,name=v.v1beta1.nodepool.kb.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 +// +kubebuilder:webhook:path=/mutate-apps-openyurt-io-v1beta1-nodepool,mutating=true,failurePolicy=fail,groups=apps.openyurt.io,resources=nodepools,verbs=create,versions=v1beta1,name=m.v1beta1.nodepool.kb.io,sideEffects=None,admissionReviewVersions=v1;v1beta1 // NodePoolHandler implements a validating and defaulting webhook for Cluster. type NodePoolHandler struct { diff --git a/pkg/webhook/nodepool/v1beta1/nodepool_validation.go b/pkg/webhook/nodepool/v1beta1/nodepool_validation.go index a7475b87697..81f7959d1a6 100644 --- a/pkg/webhook/nodepool/v1beta1/nodepool_validation.go +++ b/pkg/webhook/nodepool/v1beta1/nodepool_validation.go @@ -100,6 +100,11 @@ func validateNodePoolSpec(spec *appsv1beta1.NodePoolSpec) field.ErrorList { if allErrs := validateNodePoolSpecAnnotations(spec.Annotations); allErrs != nil { return allErrs } + + // NodePool type should be Edge or Cloud + if spec.Type != appsv1beta1.Edge && spec.Type != appsv1beta1.Cloud { + return []*field.Error{field.Invalid(field.NewPath("spec").Child("type"), spec.Type, "pool type should be Edge or Cloud")} + } return nil } @@ -111,8 +116,13 @@ func validateNodePoolSpecUpdate(spec, oldSpec *appsv1beta1.NodePoolSpec) field.E if spec.Type != oldSpec.Type { return field.ErrorList([]*field.Error{ - field.Invalid(field.NewPath("spec").Child("type"), - spec.Annotations, "pool type can't be changed")}) + field.Invalid(field.NewPath("spec").Child("type"), spec.Type, "pool type can't be changed")}) + } + + if spec.HostNetwork != oldSpec.HostNetwork { + return field.ErrorList([]*field.Error{ + field.Invalid(field.NewPath("spec").Child("hostNetwork"), spec.HostNetwork, "pool hostNetwork can't be changed"), + }) } return nil } @@ -122,14 +132,7 @@ func validateNodePoolSpecUpdate(spec, oldSpec *appsv1beta1.NodePoolSpec) field.E func validateNodePoolDeletion(cli client.Client, np *appsv1beta1.NodePool) field.ErrorList { nodes := corev1.NodeList{} - if np.Name == apps.DefaultCloudNodePoolName || np.Name == apps.DefaultEdgeNodePoolName { - return field.ErrorList([]*field.Error{ - field.Forbidden(field.NewPath("metadata").Child("name"), - fmt.Sprintf("default nodepool %s forbidden to delete", np.Name))}) - } - - if err := cli.List(context.TODO(), &nodes, - client.MatchingLabels(np.Spec.Selector.MatchLabels)); err != nil { + if err := cli.List(context.TODO(), &nodes, client.MatchingLabels(map[string]string{apps.NodePoolLabel: np.Name})); err != nil { return field.ErrorList([]*field.Error{ field.Forbidden(field.NewPath("metadata").Child("name"), "fail to get nodes associated to the pool")}) diff --git a/pkg/webhook/nodepool/v1beta1/nodepool_validation_test.go b/pkg/webhook/nodepool/v1beta1/nodepool_validation_test.go new file mode 100644 index 00000000000..0305d703225 --- /dev/null +++ b/pkg/webhook/nodepool/v1beta1/nodepool_validation_test.go @@ -0,0 +1,285 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + "net/http" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/openyurtio/openyurt/pkg/apis" + "github.com/openyurtio/openyurt/pkg/apis/apps" + appsv1beta1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" +) + +func TestValidateCreate(t *testing.T) { + testcases := map[string]struct { + pool runtime.Object + errcode int + }{ + "it is a normal nodepool": { + pool: &appsv1beta1.NodePool{ + Spec: appsv1beta1.NodePoolSpec{ + Type: appsv1beta1.Edge, + }, + }, + errcode: 0, + }, + "it is not a nodepool": { + pool: &corev1.Node{}, + errcode: http.StatusBadRequest, + }, + "invalid annotation": { + pool: &appsv1beta1.NodePool{ + Spec: appsv1beta1.NodePoolSpec{ + Type: appsv1beta1.Edge, + Annotations: map[string]string{ + "-&#foo": "invalid annotation", + }, + }, + }, + errcode: http.StatusUnprocessableEntity, + }, + "invalid pool type": { + pool: &appsv1beta1.NodePool{ + Spec: appsv1beta1.NodePoolSpec{ + Type: "invalid type", + }, + }, + errcode: http.StatusUnprocessableEntity, + }, + } + + handler := &NodePoolHandler{} + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + err := handler.ValidateCreate(context.TODO(), tc.pool) + if tc.errcode == 0 && err != nil { + t.Errorf("Expected error code %d, got %v", tc.errcode, err) + } else if tc.errcode != 0 { + statusErr := err.(*errors.StatusError) + if tc.errcode != int(statusErr.Status().Code) { + t.Errorf("Expected error code %d, got %v", tc.errcode, err) + } + } + }) + } +} + +func TestValidateUpdate(t *testing.T) { + testcases := map[string]struct { + oldPool runtime.Object + newPool runtime.Object + errcode int + }{ + "update a normal nodepool": { + oldPool: &appsv1beta1.NodePool{ + Spec: appsv1beta1.NodePoolSpec{ + Type: appsv1beta1.Edge, + }, + }, + newPool: &appsv1beta1.NodePool{ + Spec: appsv1beta1.NodePoolSpec{ + Type: appsv1beta1.Edge, + Labels: map[string]string{ + "foo": "bar", + }, + }, + }, + errcode: 0, + }, + "oldPool is not a nodepool": { + oldPool: &corev1.Node{}, + newPool: &appsv1beta1.NodePool{}, + errcode: http.StatusBadRequest, + }, + "newPool is not a nodepool": { + oldPool: &appsv1beta1.NodePool{}, + newPool: &corev1.Node{}, + errcode: http.StatusBadRequest, + }, + "invalid pool type": { + oldPool: &appsv1beta1.NodePool{ + Spec: appsv1beta1.NodePoolSpec{ + Type: appsv1beta1.Edge, + }, + }, + newPool: &appsv1beta1.NodePool{ + Spec: appsv1beta1.NodePoolSpec{ + Type: "invalid type", + }, + }, + errcode: http.StatusUnprocessableEntity, + }, + "type is changed": { + oldPool: &appsv1beta1.NodePool{ + Spec: appsv1beta1.NodePoolSpec{ + Type: appsv1beta1.Edge, + }, + }, + newPool: &appsv1beta1.NodePool{ + Spec: appsv1beta1.NodePoolSpec{ + Type: appsv1beta1.Cloud, + }, + }, + errcode: http.StatusUnprocessableEntity, + }, + "host network is changed": { + oldPool: &appsv1beta1.NodePool{ + Spec: appsv1beta1.NodePoolSpec{ + Type: appsv1beta1.Edge, + HostNetwork: false, + }, + }, + newPool: &appsv1beta1.NodePool{ + Spec: appsv1beta1.NodePoolSpec{ + Type: appsv1beta1.Edge, + HostNetwork: true, + }, + }, + errcode: http.StatusUnprocessableEntity, + }, + } + + handler := &NodePoolHandler{} + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + err := handler.ValidateUpdate(context.TODO(), tc.oldPool, tc.newPool) + if tc.errcode == 0 && err != nil { + t.Errorf("Expected error code %d, got %v", tc.errcode, err) + } else if tc.errcode != 0 { + statusErr := err.(*errors.StatusError) + if tc.errcode != int(statusErr.Status().Code) { + t.Errorf("Expected error code %d, got %v", tc.errcode, err) + } + } + }) + } +} + +func prepareNodes() []client.Object { + nodes := []client.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{ + apps.NodePoolLabel: "hangzhou", + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + } + return nodes +} + +func prepareNodePools() []client.Object { + pools := []client.Object{ + &appsv1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hangzhou", + }, + Spec: appsv1beta1.NodePoolSpec{ + Type: appsv1beta1.Edge, + Labels: map[string]string{ + "region": "hangzhou", + }, + }, + }, + &appsv1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "beijing", + }, + Spec: appsv1beta1.NodePoolSpec{ + Type: appsv1beta1.Edge, + Labels: map[string]string{ + "region": "beijing", + }, + }, + }, + } + return pools +} + +func TestValidateDelete(t *testing.T) { + nodes := prepareNodes() + pools := prepareNodePools() + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + t.Fatal("Fail to add kubernetes clint-go custom resource") + } + apis.AddToScheme(scheme) + + c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(pools...).WithObjects(nodes...).Build() + + testcases := map[string]struct { + pool runtime.Object + errcode int + }{ + "delete a empty nodepool": { + pool: &appsv1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "beijing", + }, + }, + errcode: 0, + }, + "delete a nodepool with node in it": { + pool: &appsv1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hangzhou", + }, + }, + errcode: http.StatusUnprocessableEntity, + }, + "it is not a nodepool": { + pool: &corev1.Node{}, + errcode: http.StatusBadRequest, + }, + } + + handler := &NodePoolHandler{ + Client: c, + } + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + err := handler.ValidateDelete(context.TODO(), tc.pool) + if tc.errcode == 0 && err != nil { + t.Errorf("Expected error code %d, got %v", tc.errcode, err) + } else if tc.errcode != 0 { + statusErr := err.(*errors.StatusError) + if tc.errcode != int(statusErr.Status().Code) { + t.Errorf("Expected error code %d, got %v", tc.errcode, err) + } + } + }) + } +} diff --git a/pkg/webhook/server.go b/pkg/webhook/server.go index 60278a30a2d..b72f5faad57 100644 --- a/pkg/webhook/server.go +++ b/pkg/webhook/server.go @@ -21,6 +21,7 @@ import ( "fmt" "time" + "k8s.io/client-go/rest" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -35,7 +36,6 @@ import ( "github.com/openyurtio/openyurt/pkg/controller/yurtstaticset" v1beta1gateway "github.com/openyurtio/openyurt/pkg/webhook/gateway/v1beta1" v1node "github.com/openyurtio/openyurt/pkg/webhook/node/v1" - v1alpha1nodepool "github.com/openyurtio/openyurt/pkg/webhook/nodepool/v1alpha1" v1beta1nodepool "github.com/openyurtio/openyurt/pkg/webhook/nodepool/v1beta1" v1alpha1platformadmin "github.com/openyurtio/openyurt/pkg/webhook/platformadmin/v1alpha1" v1alpha2platformadmin "github.com/openyurtio/openyurt/pkg/webhook/platformadmin/v1alpha2" @@ -74,7 +74,6 @@ func addControllerWebhook(name string, handler SetupWebhookWithManager) { func init() { addControllerWebhook(raven.ControllerName, &v1beta1gateway.GatewayHandler{}) - addControllerWebhook(nodepool.ControllerName, &v1alpha1nodepool.NodePoolHandler{}) addControllerWebhook(nodepool.ControllerName, &v1beta1nodepool.NodePoolHandler{}) addControllerWebhook(yurtstaticset.ControllerName, &v1alpha1yurtstaticset.YurtStaticSetHandler{}) addControllerWebhook(yurtappset.ControllerName, &v1alpha1yurtappset.YurtAppSetHandler{}) @@ -146,8 +145,8 @@ type GateFunc func() (enabled bool) // +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=validatingwebhookconfigurations,verbs=get;list;watch;update;patch // +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list;watch;update;patch -func Initialize(ctx context.Context, cc *config.CompletedConfig) error { - c, err := webhookcontroller.New(WebhookHandlerPath, cc) +func Initialize(ctx context.Context, cc *config.CompletedConfig, restCfg *rest.Config) error { + c, err := webhookcontroller.New(WebhookHandlerPath, cc, restCfg) if err != nil { return err } diff --git a/pkg/webhook/util/controller/webhook_controller.go b/pkg/webhook/util/controller/webhook_controller.go index fddc3264186..5ca3e807bbf 100644 --- a/pkg/webhook/util/controller/webhook_controller.go +++ b/pkg/webhook/util/controller/webhook_controller.go @@ -17,6 +17,7 @@ limitations under the License. package controller import ( + "bytes" "context" "fmt" "sync" @@ -24,6 +25,11 @@ import ( admissionregistrationv1 "k8s.io/api/admissionregistration/v1" v1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + apiextensionsinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions" + apiextensionslister "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" @@ -31,6 +37,7 @@ import ( admissionregistrationinformers "k8s.io/client-go/informers/admissionregistration/v1" coreinformers "k8s.io/client-go/informers/core/v1" clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" @@ -59,17 +66,20 @@ func Inited() chan struct{} { } type Controller struct { - kubeClient clientset.Interface - handlers map[string]struct{} + kubeClient clientset.Interface + extensionsLister apiextensionslister.CustomResourceDefinitionLister + extensionsClient apiextensionsclientset.Interface + handlers map[string]struct{} - informerFactory informers.SharedInformerFactory - synced []cache.InformerSynced + informerFactory informers.SharedInformerFactory + extensionsInformerFactory apiextensionsinformers.SharedInformerFactory + synced []cache.InformerSynced queue workqueue.RateLimitingInterface webhookPort int } -func New(handlers map[string]struct{}, cc *config.CompletedConfig) (*Controller, error) { +func New(handlers map[string]struct{}, cc *config.CompletedConfig, restCfg *rest.Config) (*Controller, error) { c := &Controller{ kubeClient: extclient.GetGenericClientWithName("webhook-controller").KubeClient, handlers: handlers, @@ -82,6 +92,32 @@ func New(handlers map[string]struct{}, cc *config.CompletedConfig) (*Controller, secretInformer := coreinformers.New(c.informerFactory, webhookutil.GetNamespace(), nil).Secrets() admissionRegistrationInformer := admissionregistrationinformers.New(c.informerFactory, v1.NamespaceAll, nil) + extensionsClient, err := apiextensionsclientset.NewForConfig(restCfg) + if err != nil { + return nil, err + } + apiExtensionsInformerFactory := apiextensionsinformers.NewSharedInformerFactory(extensionsClient, 0) + c.extensionsInformerFactory = apiExtensionsInformerFactory + crdInformer := apiExtensionsInformerFactory.Apiextensions().V1().CustomResourceDefinitions() + crdInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + crd := obj.(*apiextensionsv1.CustomResourceDefinition) + if crdHasWebhookConversion(crd) { + klog.Infof("CRD %s with conversion added", crd.Name) + c.queue.Add(crd.Name) + } + }, + UpdateFunc: func(old, new interface{}) { + crd := new.(*apiextensionsv1.CustomResourceDefinition) + if crdHasWebhookConversion(crd) { + klog.Infof("CRD %s with conversion updated", crd.Name) + c.queue.Add(crd.Name) + } + }, + }) + c.extensionsClient = extensionsClient + c.extensionsLister = crdInformer.Lister() + secretInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { secret := obj.(*v1.Secret) @@ -137,6 +173,7 @@ func New(handlers map[string]struct{}, cc *config.CompletedConfig) (*Controller, secretInformer.Informer().HasSynced, admissionRegistrationInformer.MutatingWebhookConfigurations().Informer().HasSynced, admissionRegistrationInformer.ValidatingWebhookConfigurations().Informer().HasSynced, + crdInformer.Informer().HasSynced, } return c, nil @@ -150,6 +187,7 @@ func (c *Controller) Start(ctx context.Context) { defer klog.Infof("Shutting down webhook-controller") c.informerFactory.Start(ctx.Done()) + c.extensionsInformerFactory.Start(ctx.Done()) if !cache.WaitForNamedCacheSync("webhook-controller", ctx.Done(), c.synced...) { klog.Errorf("Wait For Cache sync webhook-controller faild") return @@ -171,7 +209,7 @@ func (c *Controller) processNextWorkItem() bool { } defer c.queue.Done(key) - err := c.sync() + err := c.sync(key.(string)) if err == nil { c.queue.AddAfter(key, defaultResyncPeriod) c.queue.Forget(key) @@ -184,7 +222,7 @@ func (c *Controller) processNextWorkItem() bool { return true } -func (c *Controller) sync() error { +func (c *Controller) sync(key string) error { klog.V(5).Infof("Starting to sync webhook certs and configurations") defer func() { klog.V(5).Infof("Finished to sync webhook certs and configurations") @@ -225,8 +263,51 @@ func (c *Controller) sync() error { return fmt.Errorf("failed to ensure configuration: %v", err) } + if len(key) != 0 { + crd, err := c.extensionsLister.Get(key) + if err != nil { + klog.Errorf("failed to get crd(%s), %v", key, err) + return err + } + + if err := ensureCRDConversionCA(c.extensionsClient, crd, certs.CACert); err != nil { + klog.Errorf("failed to ensure conversion configuration for crd(%s), %v", crd.Name, err) + return err + } + } + onceInit.Do(func() { close(uninit) }) return nil } + +func crdHasWebhookConversion(crd *apiextensionsv1.CustomResourceDefinition) bool { + conversion := crd.Spec.Conversion + if conversion == nil { + return false + } + + if conversion.Strategy == apiextensionsv1.WebhookConverter { + return true + } + + return false +} + +func ensureCRDConversionCA(client apiextensionsclientset.Interface, crd *apiextensionsv1.CustomResourceDefinition, newCABundle []byte) error { + if crd.Spec.Conversion == nil || + crd.Spec.Conversion.Webhook == nil || + crd.Spec.Conversion.Webhook.ClientConfig == nil { + return nil + } + + if bytes.Equal(crd.Spec.Conversion.Webhook.ClientConfig.CABundle, newCABundle) { + return nil + } + + crd.Spec.Conversion.Webhook.ClientConfig.CABundle = newCABundle + // update crd + _, err := client.ApiextensionsV1().CustomResourceDefinitions().Update(context.TODO(), crd, metav1.UpdateOptions{}) + return err +} diff --git a/pkg/yurthub/network/dummyif_test.go b/pkg/yurthub/network/dummyif_test.go index 16c1f6f542c..a3c7e121d09 100644 --- a/pkg/yurthub/network/dummyif_test.go +++ b/pkg/yurthub/network/dummyif_test.go @@ -19,6 +19,7 @@ limitations under the License. package network +/* import ( "net" "testing" @@ -97,3 +98,4 @@ func TestEnsureDummyInterface(t *testing.T) { }) } } +*/ From 271f77462554f7b81fefffb83a9a9c5dd03afc18 Mon Sep 17 00:00:00 2001 From: rambohe Date: Tue, 15 Aug 2023 20:24:23 +0800 Subject: [PATCH 15/21] add nodepool hostnetwork propagation filter (#1654) Signed-off-by: rambohe-ch --- pkg/yurthub/filter/constant.go | 22 +- .../filter/hostnetworkpropagation/filter.go | 160 ++++++ .../hostnetworkpropagation/filter_test.go | 514 ++++++++++++++++++ pkg/yurthub/filter/manager/manager.go | 2 + pkg/yurthub/filter/manager/manager_test.go | 18 + .../filter/nodeportisolation/filter.go | 4 +- .../filter/nodeportisolation/filter_test.go | 4 +- 7 files changed, 712 insertions(+), 12 deletions(-) create mode 100644 pkg/yurthub/filter/hostnetworkpropagation/filter.go create mode 100644 pkg/yurthub/filter/hostnetworkpropagation/filter_test.go diff --git a/pkg/yurthub/filter/constant.go b/pkg/yurthub/filter/constant.go index 0ec8faf8189..b2623e6a71a 100644 --- a/pkg/yurthub/filter/constant.go +++ b/pkg/yurthub/filter/constant.go @@ -33,9 +33,14 @@ const ( // in order to make kube-proxy to use InClusterConfig to access kube-apiserver. InClusterConfigFilterName = "inclusterconfig" - // NodePortIsolationName filter is used to discard or keep NodePort service in specified NodePool + // NodePortIsolationFilterName filter is used to discard or keep NodePort service in specified NodePool // in order to make NodePort will not be listened by kube-proxy component in specified NodePool. - NodePortIsolationName = "nodeportisolation" + NodePortIsolationFilterName = "nodeportisolation" + + // HostNetworkPropagationFilterName filter is used to set pod.spec.HostNetwork to true when the + // hostNetwork field(nodePool.spec.HostNetwork) is true. this is equivalent to the nodepool + // propagating the hostNetwork configuration to the pods running in it. + HostNetworkPropagationFilterName = "hostnetworkpropagation" // SkipDiscardServiceAnnotation is annotation used by LB service. // If end users want to use specified LB service at the edge side, @@ -45,14 +50,15 @@ const ( var ( // DisabledInCloudMode contains the filters that should be disabled when yurthub is working in cloud mode. - DisabledInCloudMode = []string{DiscardCloudServiceFilterName} + DisabledInCloudMode = []string{DiscardCloudServiceFilterName, HostNetworkPropagationFilterName} // SupportedComponentsForFilter is used for specifying which components are supported by filters as default setting. SupportedComponentsForFilter = map[string]string{ - MasterServiceFilterName: "kubelet", - DiscardCloudServiceFilterName: "kube-proxy", - ServiceTopologyFilterName: "kube-proxy, coredns, nginx-ingress-controller", - InClusterConfigFilterName: "kubelet", - NodePortIsolationName: "kube-proxy", + MasterServiceFilterName: "kubelet", + DiscardCloudServiceFilterName: "kube-proxy", + ServiceTopologyFilterName: "kube-proxy, coredns, nginx-ingress-controller", + InClusterConfigFilterName: "kubelet", + NodePortIsolationFilterName: "kube-proxy", + HostNetworkPropagationFilterName: "kubelet", } ) diff --git a/pkg/yurthub/filter/hostnetworkpropagation/filter.go b/pkg/yurthub/filter/hostnetworkpropagation/filter.go new file mode 100644 index 00000000000..dc0c5baef3e --- /dev/null +++ b/pkg/yurthub/filter/hostnetworkpropagation/filter.go @@ -0,0 +1,160 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hostnetworkpropagation + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/dynamic/dynamicinformer" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/pkg/apis/apps" + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" + "github.com/openyurtio/openyurt/pkg/yurthub/filter" +) + +// Register registers a filter +func Register(filters *filter.Filters) { + filters.Register(filter.HostNetworkPropagationFilterName, func() (filter.ObjectFilter, error) { + return NewHostNetworkPropagationFilter() + }) +} + +func NewHostNetworkPropagationFilter() (filter.ObjectFilter, error) { + return &hostNetworkPropagationFilter{}, nil +} + +type hostNetworkPropagationFilter struct { + nodePoolLister cache.GenericLister + nodePoolSynced cache.InformerSynced + nodePoolName string + client kubernetes.Interface + isHostNetworkPool *bool +} + +func (hpf *hostNetworkPropagationFilter) Name() string { + return filter.HostNetworkPropagationFilterName +} + +func (hpf *hostNetworkPropagationFilter) SupportedResourceAndVerbs() map[string]sets.String { + return map[string]sets.String{ + "pods": sets.NewString("list", "watch"), + } +} + +func (hpf *hostNetworkPropagationFilter) SetNodePoolInformerFactory(dynamicInformerFactory dynamicinformer.DynamicSharedInformerFactory) error { + gvr := v1beta1.GroupVersion.WithResource("nodepools") + hpf.nodePoolLister = dynamicInformerFactory.ForResource(gvr).Lister() + hpf.nodePoolSynced = dynamicInformerFactory.ForResource(gvr).Informer().HasSynced + + return nil +} + +func (hpf *hostNetworkPropagationFilter) SetNodePoolName(poolName string) error { + hpf.nodePoolName = poolName + return nil +} + +func (hpf *hostNetworkPropagationFilter) SetKubeClient(client kubernetes.Interface) error { + hpf.client = client + return nil +} + +func (hpf *hostNetworkPropagationFilter) resolveNodePoolName(pod *corev1.Pod) string { + if len(hpf.nodePoolName) != 0 { + return hpf.nodePoolName + } + + node, err := hpf.client.CoreV1().Nodes().Get(context.Background(), pod.Spec.NodeName, metav1.GetOptions{}) + if err != nil { + klog.Warningf("could not get node(%s) in hostNetworkPropagationFilter, %v", pod.Spec.NodeName, err) + return hpf.nodePoolName + } + hpf.nodePoolName = node.Labels[apps.NodePoolLabel] + return hpf.nodePoolName +} + +func (hpf *hostNetworkPropagationFilter) Filter(obj runtime.Object, stopCh <-chan struct{}) runtime.Object { + pod, ok := obj.(*corev1.Pod) + if !ok { + return obj + } + + // when pod hostnetwork is already set to true, the function will + // short-circuit and return + if pod.Spec.HostNetwork { + return obj + } + + if hpf.isHostNetworkPool == nil { + // go to configure IsHostNetworkPool + } else if *hpf.isHostNetworkPool { + pod.Spec.HostNetwork = true + return obj + } else { + // nodepool hostNetwork field is false, only short-circuit and return + return obj + } + + if ok := cache.WaitForCacheSync(stopCh, hpf.nodePoolSynced); !ok { + return obj + } + + nodePoolName := hpf.resolveNodePoolName(pod) + if len(nodePoolName) == 0 { + klog.Infof("node(%s) is not added into node pool, so skip hostnetworkpropagation", pod.Spec.NodeName) + return obj + } + + runtimeObj, err := hpf.nodePoolLister.Get(nodePoolName) + if err != nil { + klog.Warningf("hostNetworkPropagationFilter: could not get nodepool %s, err: %v", nodePoolName, err) + return obj + } + var nodePool *v1beta1.NodePool + switch poolObj := runtimeObj.(type) { + case *v1beta1.NodePool: + nodePool = poolObj + case *unstructured.Unstructured: + nodePool = new(v1beta1.NodePool) + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(poolObj.UnstructuredContent(), nodePool); err != nil { + klog.Warningf("hostNetworkPropagationFilter: object(%#+v) is not a v1beta1.NodePool", poolObj) + return obj + } + default: + klog.Warningf("object(%#+v) is not a unknown type", poolObj) + return obj + } + + if nodePool.Spec.HostNetwork { + valTrue := true + hpf.isHostNetworkPool = &valTrue + pod.Spec.HostNetwork = true + } else { + valFalse := false + hpf.isHostNetworkPool = &valFalse + } + + return obj +} diff --git a/pkg/yurthub/filter/hostnetworkpropagation/filter_test.go b/pkg/yurthub/filter/hostnetworkpropagation/filter_test.go new file mode 100644 index 00000000000..8be6a6a54b6 --- /dev/null +++ b/pkg/yurthub/filter/hostnetworkpropagation/filter_test.go @@ -0,0 +1,514 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hostnetworkpropagation + +import ( + "reflect" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/dynamic/dynamicinformer" + "k8s.io/client-go/dynamic/fake" + k8sfake "k8s.io/client-go/kubernetes/fake" + + "github.com/openyurtio/openyurt/pkg/apis" + "github.com/openyurtio/openyurt/pkg/apis/apps" + "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" + "github.com/openyurtio/openyurt/pkg/util" + "github.com/openyurtio/openyurt/pkg/yurthub/filter" +) + +func TestName(t *testing.T) { + hpf, _ := NewHostNetworkPropagationFilter() + if hpf.Name() != filter.HostNetworkPropagationFilterName { + t.Errorf("expect %s, but got %s", filter.HostNetworkPropagationFilterName, hpf.Name()) + } +} + +func TestSupportedResourceAndVerbs(t *testing.T) { + hpf, _ := NewHostNetworkPropagationFilter() + rvs := hpf.SupportedResourceAndVerbs() + if len(rvs) != 1 { + t.Errorf("supported not one resource, %v", rvs) + } + + for resource, verbs := range rvs { + if resource != "pods" { + t.Errorf("expect resource is pods, but got %s", resource) + } + + if !verbs.Equal(sets.NewString("list", "watch")) { + t.Errorf("expect verbs are list/watch, but got %v", verbs.UnsortedList()) + } + } +} + +func TestFilter(t *testing.T) { + scheme := runtime.NewScheme() + apis.AddToScheme(scheme) + gvrToListKind := map[schema.GroupVersionResource]string{ + {Group: "apps.openyurt.io", Version: "v1beta1", Resource: "nodepools"}: "NodePoolList", + } + + testcases := map[string]struct { + poolName string + responseObject []runtime.Object + kubeClient *k8sfake.Clientset + dynamicClient *fake.FakeDynamicClient + expectObject []runtime.Object + }{ + "pod hostnetwork is true": { + responseObject: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-foo", + }, + Spec: corev1.PodSpec{ + NodeName: "node-foo", + HostNetwork: true, + }, + }, + }, + expectObject: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-foo", + }, + Spec: corev1.PodSpec{ + NodeName: "node-foo", + HostNetwork: true, + }, + }, + }, + }, + "it is not a pod": { + responseObject: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-foo", + }, + }, + }, + expectObject: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-foo", + }, + }, + }, + }, + "pool hostNetwork is false with specified nodepool": { + poolName: "pool-foo", + responseObject: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-foo", + }, + Spec: corev1.PodSpec{ + NodeName: "node-foo", + }, + }, + }, + dynamicClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pool-foo", + }, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, + }, + }, + ), + expectObject: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-foo", + }, + Spec: corev1.PodSpec{ + NodeName: "node-foo", + }, + }, + }, + }, + "pool hostNetwork true false with specified nodepool": { + poolName: "pool-foo", + responseObject: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-foo", + }, + Spec: corev1.PodSpec{ + NodeName: "node-foo", + }, + }, + }, + dynamicClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pool-foo", + }, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, + HostNetwork: true, + }, + }, + ), + expectObject: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-foo", + }, + Spec: corev1.PodSpec{ + NodeName: "node-foo", + HostNetwork: true, + }, + }, + }, + }, + "pool hostNetwork is false without specified node pool": { + responseObject: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-foo", + }, + Spec: corev1.PodSpec{ + NodeName: "node-foo", + }, + }, + }, + kubeClient: k8sfake.NewSimpleClientset( + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-foo", + Labels: map[string]string{ + apps.NodePoolLabel: "pool-foo", + }, + }, + }, + ), + dynamicClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pool-foo", + }, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, + }, + }, + ), + expectObject: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-foo", + }, + Spec: corev1.PodSpec{ + NodeName: "node-foo", + }, + }, + }, + }, + "pool hostNetwork is true without specified node pool": { + responseObject: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-foo", + }, + Spec: corev1.PodSpec{ + NodeName: "node-foo", + }, + }, + }, + kubeClient: k8sfake.NewSimpleClientset( + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-foo", + Labels: map[string]string{ + apps.NodePoolLabel: "pool-foo", + }, + }, + }, + ), + dynamicClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pool-foo", + }, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, + HostNetwork: true, + }, + }, + ), + expectObject: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-foo", + }, + Spec: corev1.PodSpec{ + NodeName: "node-foo", + HostNetwork: true, + }, + }, + }, + }, + "pool hostNetwork is false with specified unknown node": { + responseObject: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-foo", + }, + Spec: corev1.PodSpec{ + NodeName: "unknown-node", + }, + }, + }, + kubeClient: k8sfake.NewSimpleClientset( + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-foo", + Labels: map[string]string{ + apps.NodePoolLabel: "pool-foo", + }, + }, + }, + ), + dynamicClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pool-foo", + }, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, + HostNetwork: true, + }, + }, + ), + expectObject: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-foo", + }, + Spec: corev1.PodSpec{ + NodeName: "unknown-node", + }, + }, + }, + }, + "pool hostNetwork is false with unknown pool": { + responseObject: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-foo", + }, + Spec: corev1.PodSpec{ + NodeName: "node-foo", + }, + }, + }, + kubeClient: k8sfake.NewSimpleClientset( + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-foo", + Labels: map[string]string{ + apps.NodePoolLabel: "unknown-pool", + }, + }, + }, + ), + dynamicClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pool-foo", + }, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, + HostNetwork: true, + }, + }, + ), + expectObject: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-foo", + }, + Spec: corev1.PodSpec{ + NodeName: "node-foo", + }, + }, + }, + }, + "two pods with hostnetwork nodepool": { + responseObject: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-foo", + }, + Spec: corev1.PodSpec{ + NodeName: "node-foo", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-bar", + }, + Spec: corev1.PodSpec{ + NodeName: "node-foo", + }, + }, + }, + kubeClient: k8sfake.NewSimpleClientset( + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-foo", + Labels: map[string]string{ + apps.NodePoolLabel: "pool-foo", + }, + }, + }, + ), + dynamicClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pool-foo", + }, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, + HostNetwork: true, + }, + }, + ), + expectObject: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-foo", + }, + Spec: corev1.PodSpec{ + NodeName: "node-foo", + HostNetwork: true, + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-bar", + }, + Spec: corev1.PodSpec{ + NodeName: "node-foo", + HostNetwork: true, + }, + }, + }, + }, + "two pods with not hostnetwork nodepool": { + responseObject: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-foo", + }, + Spec: corev1.PodSpec{ + NodeName: "node-foo", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-bar", + }, + Spec: corev1.PodSpec{ + NodeName: "node-foo", + }, + }, + }, + kubeClient: k8sfake.NewSimpleClientset( + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-foo", + Labels: map[string]string{ + apps.NodePoolLabel: "pool-foo", + }, + }, + }, + ), + dynamicClient: fake.NewSimpleDynamicClientWithCustomListKinds(scheme, gvrToListKind, + &v1beta1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pool-foo", + }, + Spec: v1beta1.NodePoolSpec{ + Type: v1beta1.Edge, + }, + }, + ), + expectObject: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-foo", + }, + Spec: corev1.PodSpec{ + NodeName: "node-foo", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-bar", + }, + Spec: corev1.PodSpec{ + NodeName: "node-foo", + }, + }, + }, + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + hpf := &hostNetworkPropagationFilter{ + nodePoolName: tc.poolName, + client: tc.kubeClient, + } + + if tc.dynamicClient != nil { + gvr := v1beta1.GroupVersion.WithResource("nodepools") + dynamicInformerFactory := dynamicinformer.NewDynamicSharedInformerFactory(tc.dynamicClient, 24*time.Hour) + nodePoolInformer := dynamicInformerFactory.ForResource(gvr) + nodePoolLister := nodePoolInformer.Lister() + nodePoolSynced := nodePoolInformer.Informer().HasSynced + + stopper := make(chan struct{}) + defer close(stopper) + dynamicInformerFactory.Start(stopper) + dynamicInformerFactory.WaitForCacheSync(stopper) + hpf.nodePoolLister = nodePoolLister + hpf.nodePoolSynced = nodePoolSynced + } + + stopCh := make(<-chan struct{}) + for i := range tc.responseObject { + newObj := hpf.Filter(tc.responseObject[i], stopCh) + if util.IsNil(newObj) { + t.Errorf("empty object is returned") + } + if !reflect.DeepEqual(newObj, tc.expectObject[i]) { + t.Errorf("hostNetworkPropagationFilter expect: \n%#+v\nbut got: \n%#+v\n", tc.expectObject, newObj) + } + } + }) + } +} diff --git a/pkg/yurthub/filter/manager/manager.go b/pkg/yurthub/filter/manager/manager.go index b50dc3037b8..77d34a2d6f1 100644 --- a/pkg/yurthub/filter/manager/manager.go +++ b/pkg/yurthub/filter/manager/manager.go @@ -29,6 +29,7 @@ import ( "github.com/openyurtio/openyurt/cmd/yurthub/app/options" "github.com/openyurtio/openyurt/pkg/yurthub/filter" "github.com/openyurtio/openyurt/pkg/yurthub/filter/discardcloudservice" + "github.com/openyurtio/openyurt/pkg/yurthub/filter/hostnetworkpropagation" "github.com/openyurtio/openyurt/pkg/yurthub/filter/inclusterconfig" "github.com/openyurtio/openyurt/pkg/yurthub/filter/initializer" "github.com/openyurtio/openyurt/pkg/yurthub/filter/masterservice" @@ -134,4 +135,5 @@ func registerAllFilters(filters *filter.Filters) { discardcloudservice.Register(filters) inclusterconfig.Register(filters) nodeportisolation.Register(filters) + hostnetworkpropagation.Register(filters) } diff --git a/pkg/yurthub/filter/manager/manager_test.go b/pkg/yurthub/filter/manager/manager_test.go index 9c4e6d31159..3ff79d6ce6c 100644 --- a/pkg/yurthub/filter/manager/manager_test.go +++ b/pkg/yurthub/filter/manager/manager_test.go @@ -114,6 +114,24 @@ func TestFindResponseFilter(t *testing.T) { isFound: true, names: sets.NewString("nodeportisolation"), }, + "get hostnetwork propagation filter": { + enableResourceFilter: true, + accessServerThroughHub: true, + userAgent: "kubelet", + verb: "GET", + path: "/api/v1/pods", + isFound: true, + names: sets.NewString("hostnetworkpropagation"), + }, + "could not get hostnetwork propagation filter in cloud mode": { + enableResourceFilter: true, + accessServerThroughHub: true, + workingMode: "cloud", + userAgent: "kubelet", + verb: "GET", + path: "/api/v1/pods", + isFound: false, + }, } resolver := newTestRequestInfoResolver() diff --git a/pkg/yurthub/filter/nodeportisolation/filter.go b/pkg/yurthub/filter/nodeportisolation/filter.go index 8ab3f428943..69c2a028020 100644 --- a/pkg/yurthub/filter/nodeportisolation/filter.go +++ b/pkg/yurthub/filter/nodeportisolation/filter.go @@ -38,7 +38,7 @@ const ( // Register registers a filter func Register(filters *filter.Filters) { - filters.Register(filter.NodePortIsolationName, func() (filter.ObjectFilter, error) { + filters.Register(filter.NodePortIsolationFilterName, func() (filter.ObjectFilter, error) { return NewNodePortIsolationFilter() }) } @@ -54,7 +54,7 @@ func NewNodePortIsolationFilter() (filter.ObjectFilter, error) { } func (nif *nodePortIsolationFilter) Name() string { - return filter.NodePortIsolationName + return filter.NodePortIsolationFilterName } func (nif *nodePortIsolationFilter) SupportedResourceAndVerbs() map[string]sets.String { diff --git a/pkg/yurthub/filter/nodeportisolation/filter_test.go b/pkg/yurthub/filter/nodeportisolation/filter_test.go index 99ea95fb637..8ff333256ac 100644 --- a/pkg/yurthub/filter/nodeportisolation/filter_test.go +++ b/pkg/yurthub/filter/nodeportisolation/filter_test.go @@ -33,8 +33,8 @@ import ( func TestName(t *testing.T) { nif, _ := NewNodePortIsolationFilter() - if nif.Name() != filter.NodePortIsolationName { - t.Errorf("expect %s, but got %s", filter.NodePortIsolationName, nif.Name()) + if nif.Name() != filter.NodePortIsolationFilterName { + t.Errorf("expect %s, but got %s", filter.NodePortIsolationFilterName, nif.Name()) } } From 7086ecd8aef7a8319595b517a7e9978e9a3426a3 Mon Sep 17 00:00:00 2001 From: rambohe Date: Wed, 16 Aug 2023 09:42:24 +0800 Subject: [PATCH 16/21] fix verify join token and bootstrapfile only in token bootstrap mode (#1653) Signed-off-by: rambohe-ch --- cmd/yurthub/app/options/options.go | 9 ++++++--- pkg/yurthub/certificate/interfaces.go | 11 +++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/cmd/yurthub/app/options/options.go b/cmd/yurthub/app/options/options.go index a6f0deadbaf..bb8daa6497d 100644 --- a/cmd/yurthub/app/options/options.go +++ b/cmd/yurthub/app/options/options.go @@ -31,6 +31,7 @@ import ( utilnet "k8s.io/utils/net" "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate" "github.com/openyurtio/openyurt/pkg/yurthub/storage/disk" "github.com/openyurtio/openyurt/pkg/yurthub/util" ) @@ -106,7 +107,7 @@ func NewYurtHubOptions() *YurtHubOptions { HeartbeatTimeoutSeconds: 2, HeartbeatIntervalSeconds: 10, MaxRequestInFlight: 250, - BootstrapMode: "token", + BootstrapMode: certificate.TokenBoostrapMode, RootDir: filepath.Join("/var/lib/", projectinfo.GetHubName()), EnableProfiling: true, EnableDummyIf: true, @@ -148,8 +149,10 @@ func (options *YurtHubOptions) Validate() error { return fmt.Errorf("server-address is empty") } - if len(options.JoinToken) == 0 && len(options.BootstrapFile) == 0 { - return fmt.Errorf("bootstrap token and bootstrap file are empty, one of them must be set") + if options.BootstrapMode != certificate.KubeletCertificateBootstrapMode { + if len(options.JoinToken) == 0 && len(options.BootstrapFile) == 0 { + return fmt.Errorf("bootstrap token and bootstrap file are empty, one of them must be set") + } } if !util.IsSupportedLBMode(options.LBMode) { diff --git a/pkg/yurthub/certificate/interfaces.go b/pkg/yurthub/certificate/interfaces.go index ecbaedc8220..0a8c74d7b03 100644 --- a/pkg/yurthub/certificate/interfaces.go +++ b/pkg/yurthub/certificate/interfaces.go @@ -20,6 +20,17 @@ import ( "crypto/tls" ) +const ( + // KubeletCertificateBootstrapMode means that yurthub uses kubelet certificate + // that located at /var/lib/kubelet/pki/current-kubelet.pem to bootstrap instead of + // generating client certificates. + KubeletCertificateBootstrapMode = "kubeletcertificate" + + // TokenBoostrapMode means that yurthub uses join token to create client certificates + // and bootstrap itself. + TokenBoostrapMode = "token" +) + // YurtCertificateManager is responsible for managing node certificate for yurthub type YurtCertificateManager interface { YurtClientCertificateManager From 96c32a7eb591bf8e0dd1ca5712f2f590e532c036 Mon Sep 17 00:00:00 2001 From: wesleysu <59680532+River-sh@users.noreply.github.com> Date: Wed, 16 Aug 2023 13:51:23 +0800 Subject: [PATCH 17/21] add gateway pickup controller (#1648) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 珩轩 --- .../yurt-manager-auto-generated.yaml | 3 + .../app/options/gatewaycontroller.go | 18 +- cmd/yurt-manager/app/options/options.go | 8 +- pkg/controller/apis/config/types.go | 6 +- pkg/controller/controller.go | 5 +- pkg/controller/raven/common.go | 5 +- .../raven/gateway/gateway_controller.go | 295 ----------- .../raven/gateway/gateway_controller_test.go | 297 ----------- .../raven/gateway/gateway_enqueue_handlers.go | 100 ---- .../raven/{ => gatewaypickup}/config/types.go | 10 +- .../gateway_pickup_controller.go | 366 ++++++++++++++ .../gateway_pickup_controller_test.go | 467 ++++++++++++++++++ .../gateway_pickup_enqueue_handlers.go | 179 +++++++ .../raven/service/service_controller.go | 303 ------------ pkg/controller/raven/utils/utils.go | 29 ++ 15 files changed, 1070 insertions(+), 1021 deletions(-) delete mode 100644 pkg/controller/raven/gateway/gateway_controller.go delete mode 100644 pkg/controller/raven/gateway/gateway_controller_test.go delete mode 100644 pkg/controller/raven/gateway/gateway_enqueue_handlers.go rename pkg/controller/raven/{ => gatewaypickup}/config/types.go (56%) create mode 100644 pkg/controller/raven/gatewaypickup/gateway_pickup_controller.go create mode 100644 pkg/controller/raven/gatewaypickup/gateway_pickup_controller_test.go create mode 100644 pkg/controller/raven/gatewaypickup/gateway_pickup_enqueue_handlers.go delete mode 100644 pkg/controller/raven/service/service_controller.go diff --git a/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml b/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml index 6faa1d2d0f1..46800849e86 100644 --- a/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml +++ b/charts/yurt-manager/templates/yurt-manager-auto-generated.yaml @@ -489,8 +489,11 @@ rules: resources: - gateways verbs: + - create + - delete - get - list + - update - watch - apiGroups: - raven.openyurt.io diff --git a/cmd/yurt-manager/app/options/gatewaycontroller.go b/cmd/yurt-manager/app/options/gatewaycontroller.go index 0db2aeabd1c..e125dbbff1f 100644 --- a/cmd/yurt-manager/app/options/gatewaycontroller.go +++ b/cmd/yurt-manager/app/options/gatewaycontroller.go @@ -19,21 +19,21 @@ package options import ( "github.com/spf13/pflag" - "github.com/openyurtio/openyurt/pkg/controller/raven/config" + "github.com/openyurtio/openyurt/pkg/controller/raven/gatewaypickup/config" ) -type GatewayControllerOptions struct { - *config.GatewayControllerConfiguration +type GatewayPickupControllerOptions struct { + *config.GatewayPickupControllerConfiguration } -func NewGatewayControllerOptions() *GatewayControllerOptions { - return &GatewayControllerOptions{ - &config.GatewayControllerConfiguration{}, +func NewGatewayPickupControllerOptions() *GatewayPickupControllerOptions { + return &GatewayPickupControllerOptions{ + &config.GatewayPickupControllerConfiguration{}, } } // AddFlags adds flags related to nodepool for yurt-manager to the specified FlagSet. -func (g *GatewayControllerOptions) AddFlags(fs *pflag.FlagSet) { +func (g *GatewayPickupControllerOptions) AddFlags(fs *pflag.FlagSet) { if g == nil { return } @@ -41,7 +41,7 @@ func (g *GatewayControllerOptions) AddFlags(fs *pflag.FlagSet) { } // ApplyTo fills up nodepool config with options. -func (g *GatewayControllerOptions) ApplyTo(cfg *config.GatewayControllerConfiguration) error { +func (g *GatewayPickupControllerOptions) ApplyTo(cfg *config.GatewayPickupControllerConfiguration) error { if g == nil { return nil } @@ -50,7 +50,7 @@ func (g *GatewayControllerOptions) ApplyTo(cfg *config.GatewayControllerConfigur } // Validate checks validation of GatewayControllerOptions. -func (g *GatewayControllerOptions) Validate() []error { +func (g *GatewayPickupControllerOptions) Validate() []error { if g == nil { return nil } diff --git a/cmd/yurt-manager/app/options/options.go b/cmd/yurt-manager/app/options/options.go index 2ed09217543..369c5ff944f 100644 --- a/cmd/yurt-manager/app/options/options.go +++ b/cmd/yurt-manager/app/options/options.go @@ -27,7 +27,7 @@ import ( type YurtManagerOptions struct { Generic *GenericOptions NodePoolController *NodePoolControllerOptions - GatewayController *GatewayControllerOptions + GatewayPickupController *GatewayPickupControllerOptions YurtStaticSetController *YurtStaticSetControllerOptions YurtAppSetController *YurtAppSetControllerOptions YurtAppDaemonController *YurtAppDaemonControllerOptions @@ -40,7 +40,7 @@ func NewYurtManagerOptions() (*YurtManagerOptions, error) { s := YurtManagerOptions{ Generic: NewGenericOptions(), NodePoolController: NewNodePoolControllerOptions(), - GatewayController: NewGatewayControllerOptions(), + GatewayPickupController: NewGatewayPickupControllerOptions(), YurtStaticSetController: NewYurtStaticSetControllerOptions(), YurtAppSetController: NewYurtAppSetControllerOptions(), YurtAppDaemonController: NewYurtAppDaemonControllerOptions(), @@ -54,7 +54,7 @@ func (y *YurtManagerOptions) Flags() cliflag.NamedFlagSets { fss := cliflag.NamedFlagSets{} y.Generic.AddFlags(fss.FlagSet("generic")) y.NodePoolController.AddFlags(fss.FlagSet("nodepool controller")) - y.GatewayController.AddFlags(fss.FlagSet("gateway controller")) + y.GatewayPickupController.AddFlags(fss.FlagSet("gateway controller")) y.YurtStaticSetController.AddFlags(fss.FlagSet("yurtstaticset controller")) y.YurtAppDaemonController.AddFlags(fss.FlagSet("yurtappdaemon controller")) y.PlatformAdminController.AddFlags(fss.FlagSet("iot controller")) @@ -68,7 +68,7 @@ func (y *YurtManagerOptions) Validate() error { var errs []error errs = append(errs, y.Generic.Validate()...) errs = append(errs, y.NodePoolController.Validate()...) - errs = append(errs, y.GatewayController.Validate()...) + errs = append(errs, y.GatewayPickupController.Validate()...) errs = append(errs, y.YurtStaticSetController.Validate()...) errs = append(errs, y.YurtAppDaemonController.Validate()...) errs = append(errs, y.PlatformAdminController.Validate()...) diff --git a/pkg/controller/apis/config/types.go b/pkg/controller/apis/config/types.go index 58a86a245f2..ef74904330c 100644 --- a/pkg/controller/apis/config/types.go +++ b/pkg/controller/apis/config/types.go @@ -21,7 +21,7 @@ import ( nodepoolconfig "github.com/openyurtio/openyurt/pkg/controller/nodepool/config" platformadminconfig "github.com/openyurtio/openyurt/pkg/controller/platformadmin/config" - gatewayconfig "github.com/openyurtio/openyurt/pkg/controller/raven/config" + gatewaypickupconfig "github.com/openyurtio/openyurt/pkg/controller/raven/gatewaypickup/config" yurtappdaemonconfig "github.com/openyurtio/openyurt/pkg/controller/yurtappdaemon/config" yurtappsetconfig "github.com/openyurtio/openyurt/pkg/controller/yurtappset/config" yurtstaticsetconfig "github.com/openyurtio/openyurt/pkg/controller/yurtstaticset/config" @@ -34,8 +34,8 @@ type YurtManagerConfiguration struct { // NodePoolControllerConfiguration holds configuration for NodePoolController related features. NodePoolController nodepoolconfig.NodePoolControllerConfiguration - // GatewayControllerConfiguration holds configuration for GatewayController related features. - GatewayController gatewayconfig.GatewayControllerConfiguration + // GatewayPickupControllerConfiguration holds configuration for GatewayController related features. + GatewayPickupController gatewaypickupconfig.GatewayPickupControllerConfiguration // YurtAppSetControllerConfiguration holds configuration for YurtAppSetController related features. YurtAppSetController yurtappsetconfig.YurtAppSetControllerConfiguration diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index f4861eb9be9..b9a81666c17 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -27,8 +27,7 @@ import ( "github.com/openyurtio/openyurt/pkg/controller/nodepool" "github.com/openyurtio/openyurt/pkg/controller/platformadmin" "github.com/openyurtio/openyurt/pkg/controller/raven" - "github.com/openyurtio/openyurt/pkg/controller/raven/gateway" - "github.com/openyurtio/openyurt/pkg/controller/raven/service" + "github.com/openyurtio/openyurt/pkg/controller/raven/gatewaypickup" "github.com/openyurtio/openyurt/pkg/controller/servicetopology" servicetopologyendpoints "github.com/openyurtio/openyurt/pkg/controller/servicetopology/endpoints" servicetopologyendpointslice "github.com/openyurtio/openyurt/pkg/controller/servicetopology/endpointslice" @@ -56,7 +55,7 @@ func init() { controllerAddFuncs[daemonpodupdater.ControllerName] = []AddControllerFn{daemonpodupdater.Add} controllerAddFuncs[delegatelease.ControllerName] = []AddControllerFn{delegatelease.Add} controllerAddFuncs[podbinding.ControllerName] = []AddControllerFn{podbinding.Add} - controllerAddFuncs[raven.ControllerName] = []AddControllerFn{gateway.Add, service.Add} + controllerAddFuncs[raven.GatewayPickupControllerName] = []AddControllerFn{gatewaypickup.Add} controllerAddFuncs[nodepool.ControllerName] = []AddControllerFn{nodepool.Add} controllerAddFuncs[yurtcoordinatorcert.ControllerName] = []AddControllerFn{yurtcoordinatorcert.Add} controllerAddFuncs[servicetopology.ControllerName] = []AddControllerFn{servicetopologyendpoints.Add, servicetopologyendpointslice.Add} diff --git a/pkg/controller/raven/common.go b/pkg/controller/raven/common.go index f9325510320..3ea1fd29fcd 100644 --- a/pkg/controller/raven/common.go +++ b/pkg/controller/raven/common.go @@ -17,9 +17,10 @@ limitations under the License. package raven var ( - ConcurrentReconciles = 3 + ConcurrentReconciles = 1 ) const ( - ControllerName = "gateway" + ControllerName = "gateway" + GatewayPickupControllerName = "raven-gateway-pickup" ) diff --git a/pkg/controller/raven/gateway/gateway_controller.go b/pkg/controller/raven/gateway/gateway_controller.go deleted file mode 100644 index aa9c31c81b9..00000000000 --- a/pkg/controller/raven/gateway/gateway_controller.go +++ /dev/null @@ -1,295 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the License); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an AS IS BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gateway - -import ( - "context" - "fmt" - "reflect" - "strings" - "time" - - calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/tools/record" - "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - - appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" - "github.com/openyurtio/openyurt/pkg/apis/raven" - ravenv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/raven/v1alpha1" - common "github.com/openyurtio/openyurt/pkg/controller/raven" - "github.com/openyurtio/openyurt/pkg/controller/raven/config" - "github.com/openyurtio/openyurt/pkg/controller/raven/utils" - nodeutil "github.com/openyurtio/openyurt/pkg/controller/util/node" - utilclient "github.com/openyurtio/openyurt/pkg/util/client" - utildiscovery "github.com/openyurtio/openyurt/pkg/util/discovery" -) - -var ( - controllerKind = ravenv1alpha1.SchemeGroupVersion.WithKind("Gateway") -) - -func Format(format string, args ...interface{}) string { - s := fmt.Sprintf(format, args...) - return fmt.Sprintf("%s-gateway: %s", common.ControllerName, s) -} - -// Add creates a new Gateway Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller -// and Start it when the Manager is Started. -func Add(c *appconfig.CompletedConfig, mgr manager.Manager) error { - if !utildiscovery.DiscoverGVK(controllerKind) { - return nil - } - // init global variables - cfg := c.ComponentConfig.Generic - ravenv1alpha1.ServiceNamespacedName.Namespace = cfg.WorkingNamespace - - klog.Infof("ravenl3-gateway-controller add controller %s", controllerKind.String()) - return add(mgr, newReconciler(c, mgr)) -} - -var _ reconcile.Reconciler = &ReconcileGateway{} - -// ReconcileGateway reconciles a Gateway object -type ReconcileGateway struct { - client.Client - scheme *runtime.Scheme - recorder record.EventRecorder - Configration config.GatewayControllerConfiguration -} - -// newReconciler returns a new reconcile.Reconciler -func newReconciler(c *appconfig.CompletedConfig, mgr manager.Manager) reconcile.Reconciler { - return &ReconcileGateway{ - Client: utilclient.NewClientFromManager(mgr, common.ControllerName), - scheme: mgr.GetScheme(), - recorder: mgr.GetEventRecorderFor(common.ControllerName), - Configration: c.ComponentConfig.GatewayController, - } -} - -// add adds a new Controller to mgr with r as the reconcile.Reconciler -func add(mgr manager.Manager, r reconcile.Reconciler) error { - // Create a new controller - c, err := controller.New(fmt.Sprintf("%s-gateway", common.ControllerName), mgr, controller.Options{ - Reconciler: r, MaxConcurrentReconciles: common.ConcurrentReconciles, - }) - if err != nil { - return err - } - - // Watch for changes to Gateway - err = c.Watch(&source.Kind{Type: &ravenv1alpha1.Gateway{}}, &handler.EnqueueRequestForObject{}) - if err != nil { - return err - } - - // Watch for changes to Nodes - err = c.Watch(&source.Kind{Type: &corev1.Node{}}, &EnqueueGatewayForNode{}) - if err != nil { - return err - } - - return nil -} - -//+kubebuilder:rbac:groups=raven.openyurt.io,resources=gateways,verbs=get;list;watch; -//+kubebuilder:rbac:groups=raven.openyurt.io,resources=gateways/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=raven.openyurt.io,resources=gateways/finalizers,verbs=update -//+kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch;update;patch -//+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=core,resources=endpoints,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=crd.projectcalico.org,resources=blockaffinities,verbs=get;list;watch - -// Reconcile reads that state of the cluster for a Gateway object and makes changes based on the state read -// and what is in the Gateway.Spec -func (r *ReconcileGateway) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - - // Note !!!!!!!!!! - // We strongly recommend use Format() to encapsulation because Format() can print logs by module - // @kadisi - klog.V(4).Info(Format("started reconciling Gateway %s/%s", req.Namespace, req.Name)) - defer func() { - klog.V(4).Info(Format("finished reconciling Gateway %s/%s", req.Namespace, req.Name)) - }() - - var gw ravenv1alpha1.Gateway - if err := r.Get(ctx, req.NamespacedName, &gw); err != nil { - return reconcile.Result{}, client.IgnoreNotFound(err) - } - - // get all managed nodes - var nodeList corev1.NodeList - nodeSelector, err := labels.Parse(fmt.Sprintf(raven.LabelCurrentGateway+"=%s", gw.Name)) - if err != nil { - return reconcile.Result{}, err - } - err = r.List(ctx, &nodeList, &client.ListOptions{ - LabelSelector: nodeSelector, - }) - if err != nil { - err = fmt.Errorf("unable to list nodes: %s", err) - return reconcile.Result{}, err - } - - // 1. try to elect an active endpoint if possible - activeEp := r.electActiveEndpoint(nodeList, &gw) - r.recordEndpointEvent(ctx, &gw, gw.Status.ActiveEndpoint, activeEp) - if utils.IsGatewayExposeByLB(&gw) { - var svc corev1.Service - if err := r.Get(ctx, ravenv1alpha1.ServiceNamespacedName, &svc); err != nil { - klog.V(2).Info(Format("waiting for service sync, error: %s", err)) - return reconcile.Result{Requeue: true, RequeueAfter: 5 * time.Second}, nil - } - if len(svc.Status.LoadBalancer.Ingress) == 0 { - klog.V(2).Info("waiting for LB ingress sync") - return reconcile.Result{Requeue: true, RequeueAfter: 5 * time.Second}, nil - } - activeEp.PublicIP = svc.Status.LoadBalancer.Ingress[0].IP - } - gw.Status.ActiveEndpoint = activeEp - - // 2. get nodeInfo list of nodes managed by the Gateway - var nodes []ravenv1alpha1.NodeInfo - for _, v := range nodeList.Items { - podCIDRs, err := r.getPodCIDRs(ctx, v) - if err != nil { - klog.ErrorS(err, "unable to get podCIDR") - return reconcile.Result{}, err - } - nodes = append(nodes, ravenv1alpha1.NodeInfo{ - NodeName: v.Name, - PrivateIP: utils.GetNodeInternalIP(v), - Subnets: podCIDRs, - }) - } - klog.V(4).Info(Format("managed node info list, nodes: %v", nodes)) - gw.Status.Nodes = nodes - - err = r.Status().Update(ctx, &gw) - if err != nil { - klog.V(4).ErrorS(err, Format("unable to Update Gateway.status")) - return reconcile.Result{}, err - } - - return reconcile.Result{}, nil -} - -func (r *ReconcileGateway) recordEndpointEvent(ctx context.Context, sourceObj *ravenv1alpha1.Gateway, previous, current *ravenv1alpha1.Endpoint) { - if current != nil && !reflect.DeepEqual(previous, current) { - r.recorder.Event(sourceObj.DeepCopy(), corev1.EventTypeNormal, - ravenv1alpha1.EventActiveEndpointElected, - fmt.Sprintf("The endpoint hosted by node %s has been elected active endpoint, publicIP: %s", current.NodeName, current.PublicIP)) - klog.V(2).InfoS(Format("elected new active endpoint"), "nodeName", current.NodeName, "publicIP", current.PublicIP) - return - } - if current == nil && previous != nil { - r.recorder.Event(sourceObj.DeepCopy(), corev1.EventTypeWarning, - ravenv1alpha1.EventActiveEndpointLost, - fmt.Sprintf("The active endpoint hosted by node %s was lost, publicIP: %s", previous.NodeName, previous.PublicIP)) - klog.V(2).InfoS(Format("active endpoint lost"), "nodeName", previous.NodeName, "publicIP", previous.PublicIP) - return - } -} - -// electActiveEndpoint trys to elect an active Endpoint. -// If the current active endpoint remains valid, then we don't change it. -// Otherwise, try to elect a new one. -func (r *ReconcileGateway) electActiveEndpoint(nodeList corev1.NodeList, gw *ravenv1alpha1.Gateway) (ep *ravenv1alpha1.Endpoint) { - // get all ready nodes referenced by endpoints - readyNodes := make(map[string]corev1.Node) - for _, v := range nodeList.Items { - if isNodeReady(v) { - readyNodes[v.Name] = v - } - } - // checkActive check if the given endpoint is able to become the active endpoint. - checkActive := func(ep *ravenv1alpha1.Endpoint) bool { - if ep == nil { - return false - } - // check if the node status is ready - if _, ok := readyNodes[ep.NodeName]; ok { - var inList bool - // check if ep is in the Endpoint list - for _, v := range gw.Spec.Endpoints { - if reflect.DeepEqual(v, *ep) { - inList = true - break - } - } - return inList - } - return false - } - - // the current active endpoint is still competent. - if checkActive(gw.Status.ActiveEndpoint) { - for _, v := range gw.Spec.Endpoints { - if v.NodeName == gw.Status.ActiveEndpoint.NodeName { - return v.DeepCopy() - } - } - } - - // try to elect an active endpoint. - for _, v := range gw.Spec.Endpoints { - if checkActive(&v) { - return v.DeepCopy() - } - } - return -} - -// isNodeReady checks if the `node` is `corev1.NodeReady` -func isNodeReady(node corev1.Node) bool { - _, nc := nodeutil.GetNodeCondition(&node.Status, corev1.NodeReady) - // GetNodeCondition will return nil and -1 if the condition is not present - return nc != nil && nc.Status == corev1.ConditionTrue -} - -// getPodCIDRs returns the pod IP ranges assigned to the node. -func (r *ReconcileGateway) getPodCIDRs(ctx context.Context, node corev1.Node) ([]string, error) { - podCIDRs := make([]string, 0) - for key := range node.Annotations { - if strings.Contains(key, "projectcalico.org") { - var blockAffinityList calicov3.BlockAffinityList - err := r.List(ctx, &blockAffinityList) - if err != nil { - err = fmt.Errorf(Format("unable to list calico blockaffinity: %s", err)) - return nil, err - } - for _, v := range blockAffinityList.Items { - if v.Spec.Node != node.Name || v.Spec.State != "confirmed" { - continue - } - podCIDRs = append(podCIDRs, v.Spec.CIDR) - } - return podCIDRs, nil - } - } - return append(podCIDRs, node.Spec.PodCIDR), nil -} diff --git a/pkg/controller/raven/gateway/gateway_controller_test.go b/pkg/controller/raven/gateway/gateway_controller_test.go deleted file mode 100644 index 31cfe9211da..00000000000 --- a/pkg/controller/raven/gateway/gateway_controller_test.go +++ /dev/null @@ -1,297 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. -Licensed under the Apache License, Version 2.0 (the License); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an AS IS BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gateway - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - ravenv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/raven/v1alpha1" -) - -var ( - nodeReadyStatus = corev1.NodeStatus{ - Conditions: []corev1.NodeCondition{ - { - Type: corev1.NodeReady, - Status: corev1.ConditionTrue, - }, - }, - } - nodeNotReadyStatus = corev1.NodeStatus{ - Conditions: []corev1.NodeCondition{ - { - Type: corev1.NodeReady, - Status: corev1.ConditionFalse, - }, - }, - } -) - -func TestReconcileGateway_electActiveEndpoint(t *testing.T) { - mockReconciler := &ReconcileGateway{} - var tt = []struct { - name string - nodeList corev1.NodeList - gw *ravenv1alpha1.Gateway - expectedEp *ravenv1alpha1.Endpoint - }{ - - { - // The node hosting active endpoint becomes NotReady, and it is the only node in the Gateway, - // then the active endpoint should be removed. - name: "lost active endpoint", - nodeList: corev1.NodeList{ - Items: []corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - }, - Status: nodeNotReadyStatus, - }, - }, - }, - gw: &ravenv1alpha1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "gateway-1", - }, - Spec: ravenv1alpha1.GatewaySpec{ - Endpoints: []ravenv1alpha1.Endpoint{ - { - NodeName: "node-1", - }, - }, - }, - Status: ravenv1alpha1.GatewayStatus{ - ActiveEndpoint: &ravenv1alpha1.Endpoint{ - NodeName: "node-1", - }, - }, - }, - expectedEp: nil, - }, - { - // The node hosting active endpoint becomes NotReady, but there are at least one Ready node, - // then a new endpoint should be elected active endpoint to replace the old one. - name: "switch active endpoint", - nodeList: corev1.NodeList{ - Items: []corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - }, - Status: nodeNotReadyStatus, - }, { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-2", - }, - Status: nodeReadyStatus, - }, - }, - }, - gw: &ravenv1alpha1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "gateway-1", - }, - Spec: ravenv1alpha1.GatewaySpec{ - Endpoints: []ravenv1alpha1.Endpoint{ - { - NodeName: "node-1", - }, - { - NodeName: "node-2", - }, - }, - }, - Status: ravenv1alpha1.GatewayStatus{ - ActiveEndpoint: &ravenv1alpha1.Endpoint{ - NodeName: "node-1", - }, - }, - }, - expectedEp: &ravenv1alpha1.Endpoint{ - NodeName: "node-2", - }, - }, - { - - name: "elect new active endpoint", - nodeList: corev1.NodeList{ - Items: []corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - }, - Status: nodeNotReadyStatus, - }, { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-2", - }, - Status: nodeReadyStatus, - }, - }, - }, - gw: &ravenv1alpha1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "gateway-1", - }, - Spec: ravenv1alpha1.GatewaySpec{ - Endpoints: []ravenv1alpha1.Endpoint{ - { - NodeName: "node-1", - }, - { - NodeName: "node-2", - }, - }, - }, - Status: ravenv1alpha1.GatewayStatus{ - ActiveEndpoint: &ravenv1alpha1.Endpoint{ - NodeName: "node-1", - }, - }, - }, - expectedEp: &ravenv1alpha1.Endpoint{ - NodeName: "node-2", - }, - }, - { - name: "no available active endpoint", - nodeList: corev1.NodeList{ - Items: []corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - }, - Status: nodeNotReadyStatus, - }, { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-2", - }, - Status: nodeNotReadyStatus, - }, - }, - }, - gw: &ravenv1alpha1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "gateway-1", - }, - Spec: ravenv1alpha1.GatewaySpec{ - Endpoints: []ravenv1alpha1.Endpoint{ - { - NodeName: "node-1", - }, - { - NodeName: "node-2", - }, - }, - }, - Status: ravenv1alpha1.GatewayStatus{ - ActiveEndpoint: nil, - }, - }, - expectedEp: nil, - }, - { - // The node hosting the active endpoint is still ready, do not change it. - name: "don't switch active endpoint", - nodeList: corev1.NodeList{ - Items: []corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - }, - Status: nodeNotReadyStatus, - }, { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-2", - }, - Status: nodeReadyStatus, - }, - }, - }, - gw: &ravenv1alpha1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "gateway-1", - }, - Spec: ravenv1alpha1.GatewaySpec{ - Endpoints: []ravenv1alpha1.Endpoint{ - { - NodeName: "node-1", - }, - { - NodeName: "node-2", - }, - }, - }, - Status: ravenv1alpha1.GatewayStatus{ - ActiveEndpoint: &ravenv1alpha1.Endpoint{ - NodeName: "node-2", - }, - }, - }, - expectedEp: &ravenv1alpha1.Endpoint{ - NodeName: "node-2", - }, - }, - } - for _, v := range tt { - t.Run(v.name, func(t *testing.T) { - a := assert.New(t) - ep := mockReconciler.electActiveEndpoint(v.nodeList, v.gw) - a.Equal(v.expectedEp, ep) - }) - } - -} - -func TestReconcileGateway_getPodCIDRs(t *testing.T) { - mockReconciler := &ReconcileGateway{} - var tt = []struct { - name string - node corev1.Node - expectPodCIDR []string - }{ - { - name: "node has pod CIDR", - node: corev1.Node{ - Spec: corev1.NodeSpec{ - PodCIDR: "10.0.0.1/24", - }, - }, - expectPodCIDR: []string{"10.0.0.1/24"}, - }, - { - name: "node hasn't pod CIDR", - node: corev1.Node{ - Spec: corev1.NodeSpec{}, - }, - expectPodCIDR: []string{""}, - }, - } - for _, v := range tt { - t.Run(v.name, func(t *testing.T) { - a := assert.New(t) - podCIDRs, err := mockReconciler.getPodCIDRs(context.Background(), v.node) - if a.NoError(err) { - a.Equal(v.expectPodCIDR, podCIDRs) - } - - }) - } -} diff --git a/pkg/controller/raven/gateway/gateway_enqueue_handlers.go b/pkg/controller/raven/gateway/gateway_enqueue_handlers.go deleted file mode 100644 index 0ab3ed17b45..00000000000 --- a/pkg/controller/raven/gateway/gateway_enqueue_handlers.go +++ /dev/null @@ -1,100 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. -Licensed under the Apache License, Version 2.0 (the License); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an AS IS BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gateway - -import ( - corev1 "k8s.io/api/core/v1" - "k8s.io/client-go/util/workqueue" - "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/event" - - "github.com/openyurtio/openyurt/pkg/apis/raven" - "github.com/openyurtio/openyurt/pkg/controller/raven/utils" -) - -type EnqueueGatewayForNode struct{} - -// Create implements EventHandler -func (e *EnqueueGatewayForNode) Create(evt event.CreateEvent, - q workqueue.RateLimitingInterface) { - node, ok := evt.Object.(*corev1.Node) - if !ok { - klog.Error(Format("fail to assert runtime Object to v1.Node")) - return - } - klog.V(5).Infof(Format("will enqueue gateway as node(%s) has been created", - node.GetName())) - if gwName, exist := node.Labels[raven.LabelCurrentGateway]; exist { - utils.AddGatewayToWorkQueue(gwName, q) - return - } - klog.V(4).Infof(Format("node(%s) does not belong to any gateway", node.GetName())) -} - -// Update implements EventHandler -func (e *EnqueueGatewayForNode) Update(evt event.UpdateEvent, - q workqueue.RateLimitingInterface) { - newNode, ok := evt.ObjectNew.(*corev1.Node) - if !ok { - klog.Errorf(Format("Fail to assert runtime Object(%s) to v1.Node", - evt.ObjectNew.GetName())) - return - } - oldNode, ok := evt.ObjectOld.(*corev1.Node) - if !ok { - klog.Errorf(Format("fail to assert runtime Object(%s) to v1.Node", - evt.ObjectOld.GetName())) - return - } - klog.V(5).Infof(Format("Will enqueue gateway as node(%s) has been updated", - newNode.GetName())) - - oldGwName := oldNode.Labels[raven.LabelCurrentGateway] - newGwName := newNode.Labels[raven.LabelCurrentGateway] - - // check if NodeReady condition changed - statusChanged := func(oldObj, newObj *corev1.Node) bool { - return isNodeReady(*oldObj) != isNodeReady(*newObj) - } - - if oldGwName != newGwName || statusChanged(oldNode, newNode) { - utils.AddGatewayToWorkQueue(oldGwName, q) - utils.AddGatewayToWorkQueue(newGwName, q) - } -} - -// Delete implements EventHandler -func (e *EnqueueGatewayForNode) Delete(evt event.DeleteEvent, - q workqueue.RateLimitingInterface) { - node, ok := evt.Object.(*corev1.Node) - if !ok { - klog.Error(Format("Fail to assert runtime Object to v1.Node")) - return - } - - gwName, exist := node.Labels[raven.LabelCurrentGateway] - if !exist { - klog.V(5).Infof(Format("Node(%s) doesn't belong to any gateway", node.GetName())) - return - } - // enqueue the gateway that the node belongs to - klog.V(5).Infof(Format("Will enqueue pool(%s) as node(%s) has been deleted", - gwName, node.GetName())) - utils.AddGatewayToWorkQueue(gwName, q) -} - -// Generic implements EventHandler -func (e *EnqueueGatewayForNode) Generic(evt event.GenericEvent, - q workqueue.RateLimitingInterface) { -} diff --git a/pkg/controller/raven/config/types.go b/pkg/controller/raven/gatewaypickup/config/types.go similarity index 56% rename from pkg/controller/raven/config/types.go rename to pkg/controller/raven/gatewaypickup/config/types.go index 0f57d52092b..ee7572d6aef 100644 --- a/pkg/controller/raven/config/types.go +++ b/pkg/controller/raven/gatewaypickup/config/types.go @@ -1,14 +1,14 @@ /* Copyright 2023 The OpenYurt Authors. -Licensed under the Apache License, Version 2.0 (the License); +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an AS IS BASIS, +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. @@ -16,6 +16,6 @@ limitations under the License. package config -// GatewayControllerConfiguration contains elements describing GatewayController. -type GatewayControllerConfiguration struct { +// GatewayPickupControllerConfiguration contains elements describing GatewayPickController. +type GatewayPickupControllerConfiguration struct { } diff --git a/pkg/controller/raven/gatewaypickup/gateway_pickup_controller.go b/pkg/controller/raven/gatewaypickup/gateway_pickup_controller.go new file mode 100644 index 00000000000..75d26b9477d --- /dev/null +++ b/pkg/controller/raven/gatewaypickup/gateway_pickup_controller.go @@ -0,0 +1,366 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gatewaypickup + +import ( + "context" + "fmt" + "reflect" + "sort" + "strings" + "time" + + calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + corev1 "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + "github.com/openyurtio/openyurt/pkg/apis/raven" + ravenv1beta1 "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" + common "github.com/openyurtio/openyurt/pkg/controller/raven" + "github.com/openyurtio/openyurt/pkg/controller/raven/gatewaypickup/config" + "github.com/openyurtio/openyurt/pkg/controller/raven/utils" + nodeutil "github.com/openyurtio/openyurt/pkg/controller/util/node" + utilclient "github.com/openyurtio/openyurt/pkg/util/client" + utildiscovery "github.com/openyurtio/openyurt/pkg/util/discovery" +) + +var ( + controllerKind = ravenv1beta1.SchemeGroupVersion.WithKind("Gateway") +) + +func Format(format string, args ...interface{}) string { + s := fmt.Sprintf(format, args...) + return fmt.Sprintf("%s: %s", common.GatewayPickupControllerName, s) +} + +const ( + ActiveEndpointsName = "ActiveEndpointName" + ActiveEndpointsPublicIP = "ActiveEndpointsPublicIP" + ActiveEndpointsType = "ActiveEndpointsType" +) + +// Add creates a new Gateway Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(c *appconfig.CompletedConfig, mgr manager.Manager) error { + if !utildiscovery.DiscoverGVK(controllerKind) { + return nil + } + klog.Infof("raven-gateway-controller add controller %s", controllerKind.String()) + return add(mgr, newReconciler(c, mgr)) +} + +var _ reconcile.Reconciler = &ReconcileGateway{} + +// ReconcileGateway reconciles a Gateway object +type ReconcileGateway struct { + client.Client + scheme *runtime.Scheme + recorder record.EventRecorder + Configration config.GatewayPickupControllerConfiguration +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(c *appconfig.CompletedConfig, mgr manager.Manager) reconcile.Reconciler { + return &ReconcileGateway{ + Client: utilclient.NewClientFromManager(mgr, common.GatewayPickupControllerName), + scheme: mgr.GetScheme(), + recorder: mgr.GetEventRecorderFor(common.GatewayPickupControllerName), + Configration: c.ComponentConfig.GatewayPickupController, + } +} + +// add is used to add a new Controller to mgr +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New(common.GatewayPickupControllerName, mgr, controller.Options{ + Reconciler: r, MaxConcurrentReconciles: common.ConcurrentReconciles, + }) + if err != nil { + return err + } + + // Watch for changes to Gateway + err = c.Watch(&source.Kind{Type: &ravenv1beta1.Gateway{}}, &handler.EnqueueRequestForObject{}) + if err != nil { + return err + } + + // Watch for changes to Nodes + err = c.Watch(&source.Kind{Type: &corev1.Node{}}, &EnqueueGatewayForNode{}) + if err != nil { + return err + } + + err = c.Watch(&source.Kind{Type: &corev1.ConfigMap{}}, &EnqueueGatewayForRavenConfig{client: utilclient.NewClientFromManager(mgr, "raven-config")}, predicate.NewPredicateFuncs( + func(object client.Object) bool { + cm, ok := object.(*corev1.ConfigMap) + if !ok { + return false + } + if cm.GetNamespace() != utils.WorkingNamespace { + return false + } + if cm.GetName() != utils.RavenGlobalConfig { + return false + } + return true + })) + if err != nil { + return err + } + return nil +} + +//+kubebuilder:rbac:groups=raven.openyurt.io,resources=gateways,verbs=get;list;watch;create;delete;update +//+kubebuilder:rbac:groups=raven.openyurt.io,resources=gateways/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=raven.openyurt.io,resources=gateways/finalizers,verbs=update +//+kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch;update;patch +//+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=core,resources=endpoints,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=crd.projectcalico.org,resources=blockaffinities,verbs=get;list;watch + +// Reconcile reads that state of the cluster for a Gateway object and makes changes based on the state read +// and what is in the Gateway.Spec +func (r *ReconcileGateway) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + + // Note !!!!!!!!!! + // We strongly recommend use Format() to encapsulation because Format() can print logs by module + // @kadisi + klog.V(2).Info(Format("started reconciling Gateway %s/%s", req.Namespace, req.Name)) + defer func() { + klog.V(2).Info(Format("finished reconciling Gateway %s/%s", req.Namespace, req.Name)) + }() + + var gw ravenv1beta1.Gateway + if err := r.Get(ctx, req.NamespacedName, &gw); err != nil { + return reconcile.Result{}, client.IgnoreNotFound(err) + } + + // get all managed nodes + var nodeList corev1.NodeList + nodeSelector, err := labels.Parse(fmt.Sprintf(raven.LabelCurrentGateway+"=%s", gw.Name)) + if err != nil { + return reconcile.Result{}, err + } + err = r.List(ctx, &nodeList, &client.ListOptions{ + LabelSelector: nodeSelector, + }) + if err != nil { + err = fmt.Errorf("unable to list nodes: %s", err) + return reconcile.Result{}, err + } + klog.V(1).Info(Format("list gateway %d node %v", len(nodeList.Items), nodeList.Items)) + // 1. try to elect an active endpoint if possible + activeEp := r.electActiveEndpoint(nodeList, &gw) + r.recordEndpointEvent(&gw, gw.Status.ActiveEndpoints, activeEp) + gw.Status.ActiveEndpoints = activeEp + // 2. get nodeInfo list of nodes managed by the Gateway + var nodes []ravenv1beta1.NodeInfo + for _, v := range nodeList.Items { + podCIDRs, err := r.getPodCIDRs(ctx, v) + if err != nil { + klog.ErrorS(err, "unable to get podCIDR") + return reconcile.Result{}, err + } + nodes = append(nodes, ravenv1beta1.NodeInfo{ + NodeName: v.Name, + PrivateIP: utils.GetNodeInternalIP(v), + Subnets: podCIDRs, + }) + } + sort.Slice(nodes, func(i, j int) bool { return nodes[i].NodeName < nodes[j].NodeName }) + klog.V(4).Info(Format("managed node info list, nodes: %v", nodes)) + gw.Status.Nodes = nodes + err = r.Status().Update(ctx, &gw) + if err != nil { + if apierrs.IsConflict(err) { + klog.Warning(err, Format("unable to update gateway.status, error %s", err.Error())) + return reconcile.Result{Requeue: true, RequeueAfter: 5 * time.Second}, nil + } + return reconcile.Result{Requeue: true, RequeueAfter: 5 * time.Second}, + fmt.Errorf("unable to update %s gateway.status, error %s", gw.GetName(), err.Error()) + } + return reconcile.Result{}, nil +} + +func (r *ReconcileGateway) recordEndpointEvent(sourceObj *ravenv1beta1.Gateway, previous, current []*ravenv1beta1.Endpoint) { + sort.Slice(previous, func(i, j int) bool { return previous[i].NodeName < previous[j].NodeName }) + sort.Slice(current, func(i, j int) bool { return current[i].NodeName < current[j].NodeName }) + if len(current) != 0 && !reflect.DeepEqual(previous, current) { + eps, num := getActiveEndpointsInfo(current) + for i := 0; i < num; i++ { + r.recorder.Event(sourceObj.DeepCopy(), corev1.EventTypeNormal, + ravenv1beta1.EventActiveEndpointElected, + fmt.Sprintf("The endpoint hosted by node %s has been elected active endpoint, publicIP: %s, type: %s", eps[ActiveEndpointsName][i], eps[ActiveEndpointsPublicIP][i], eps[ActiveEndpointsType][i])) + } + + klog.V(2).InfoS(Format("elected new active endpoint"), "nodeName", eps[ActiveEndpointsName], "publicIP", eps[ActiveEndpointsPublicIP], "type", eps[ActiveEndpointsType]) + return + } + if len(previous) != 0 && !reflect.DeepEqual(previous, current) { + eps, num := getActiveEndpointsInfo(previous) + for i := 0; i < num; i++ { + r.recorder.Event(sourceObj.DeepCopy(), corev1.EventTypeWarning, + ravenv1beta1.EventActiveEndpointLost, + fmt.Sprintf("The active endpoint hosted by node %s was change, publicIP: %s, type :%s", eps[ActiveEndpointsName][i], eps[ActiveEndpointsPublicIP][i], eps[ActiveEndpointsType][i])) + } + klog.V(2).InfoS(Format("active endpoint lost"), "nodeName", eps[ActiveEndpointsName], "publicIP", eps[ActiveEndpointsPublicIP], "type", eps[ActiveEndpointsType]) + return + } +} + +// electActiveEndpoint trys to elect an active Endpoint. +// If the current active endpoint remains valid, then we don't change it. +// Otherwise, try to elect a new one. +func (r *ReconcileGateway) electActiveEndpoint(nodeList corev1.NodeList, gw *ravenv1beta1.Gateway) []*ravenv1beta1.Endpoint { + // get all ready nodes referenced by endpoints + readyNodes := make(map[string]*corev1.Node) + for _, v := range nodeList.Items { + if isNodeReady(v) { + readyNodes[v.Name] = &v + } + } + klog.V(1).Infof(Format("Ready node has %d, node %v", len(readyNodes), readyNodes)) + // init a endpoints slice + enableProxy, enableTunnel := utils.CheckServer(context.TODO(), r.Client) + eps := make([]*ravenv1beta1.Endpoint, 0) + if enableProxy { + eps = append(eps, electEndpoints(gw, ravenv1beta1.Proxy, readyNodes)...) + } + if enableTunnel { + eps = append(eps, electEndpoints(gw, ravenv1beta1.Tunnel, readyNodes)...) + } + sort.Slice(eps, func(i, j int) bool { return eps[i].NodeName < eps[j].NodeName }) + return eps +} + +func electEndpoints(gw *ravenv1beta1.Gateway, endpointType string, readyNodes map[string]*corev1.Node) []*ravenv1beta1.Endpoint { + eps := make([]*ravenv1beta1.Endpoint, 0) + var replicas int + switch endpointType { + case ravenv1beta1.Proxy: + replicas = gw.Spec.ProxyConfig.Replicas + case ravenv1beta1.Tunnel: + replicas = gw.Spec.TunnelConfig.Replicas + default: + replicas = 1 + } + + checkCandidates := func(ep *ravenv1beta1.Endpoint) bool { + if _, ok := readyNodes[ep.NodeName]; ok && ep.Type == endpointType { + return true + } + return false + } + + // the current active endpoint is still competent. + candidates := make(map[string]*ravenv1beta1.Endpoint, 0) + for _, activeEndpoint := range gw.Status.ActiveEndpoints { + if checkCandidates(activeEndpoint) { + for _, ep := range gw.Spec.Endpoints { + if ep.NodeName == activeEndpoint.NodeName && ep.Type == activeEndpoint.Type { + candidates[activeEndpoint.NodeName] = ep.DeepCopy() + } + } + } + } + for _, aep := range candidates { + if len(eps) == replicas { + aepInfo, _ := getActiveEndpointsInfo(eps) + klog.V(4).InfoS(Format("elect %d active endpoints %s for gateway %s/%s", + len(eps), fmt.Sprintf("[%s]", strings.Join(aepInfo[ActiveEndpointsName], ",")), gw.GetNamespace(), gw.GetName())) + return eps + } + klog.V(1).Infof(Format("node %s is active endpoints, type is %s", aep.NodeName, aep.Type)) + klog.V(1).Infof(Format("add node %v", aep.DeepCopy())) + eps = append(eps, aep.DeepCopy()) + } + + for _, ep := range gw.Spec.Endpoints { + if _, ok := candidates[ep.NodeName]; !ok && checkCandidates(&ep) { + if len(eps) == replicas { + aepInfo, _ := getActiveEndpointsInfo(eps) + klog.V(4).InfoS(Format("elect %d active endpoints %s for gateway %s/%s", + len(eps), fmt.Sprintf("[%s]", strings.Join(aepInfo[ActiveEndpointsName], ",")), gw.GetNamespace(), gw.GetName())) + return eps + } + klog.V(1).Infof(Format("node %s is active endpoints, type is %s", ep.NodeName, ep.Type)) + klog.V(1).Infof(Format("add node %v", ep.DeepCopy())) + eps = append(eps, ep.DeepCopy()) + } + } + return eps +} + +// isNodeReady checks if the `node` is `corev1.NodeReady` +func isNodeReady(node corev1.Node) bool { + _, nc := nodeutil.GetNodeCondition(&node.Status, corev1.NodeReady) + // GetNodeCondition will return nil and -1 if the condition is not present + return nc != nil && nc.Status == corev1.ConditionTrue +} + +// getPodCIDRs returns the pod IP ranges assigned to the node. +func (r *ReconcileGateway) getPodCIDRs(ctx context.Context, node corev1.Node) ([]string, error) { + podCIDRs := make([]string, 0) + for key := range node.Annotations { + if strings.Contains(key, "projectcalico.org") { + var blockAffinityList calicov3.BlockAffinityList + err := r.List(ctx, &blockAffinityList) + if err != nil { + err = fmt.Errorf(Format("unable to list calico blockaffinity: %s", err)) + return nil, err + } + for _, v := range blockAffinityList.Items { + if v.Spec.Node != node.Name || v.Spec.State != "confirmed" { + continue + } + podCIDRs = append(podCIDRs, v.Spec.CIDR) + } + return podCIDRs, nil + } + } + return append(podCIDRs, node.Spec.PodCIDR), nil +} + +func getActiveEndpointsInfo(eps []*ravenv1beta1.Endpoint) (map[string][]string, int) { + infos := make(map[string][]string) + infos[ActiveEndpointsName] = make([]string, 0) + infos[ActiveEndpointsPublicIP] = make([]string, 0) + infos[ActiveEndpointsType] = make([]string, 0) + if len(eps) == 0 { + return infos, 0 + } + for _, ep := range eps { + infos[ActiveEndpointsName] = append(infos[ActiveEndpointsName], ep.NodeName) + infos[ActiveEndpointsPublicIP] = append(infos[ActiveEndpointsPublicIP], ep.PublicIP) + infos[ActiveEndpointsType] = append(infos[ActiveEndpointsType], ep.Type) + } + return infos, len(infos[ActiveEndpointsName]) +} diff --git a/pkg/controller/raven/gatewaypickup/gateway_pickup_controller_test.go b/pkg/controller/raven/gatewaypickup/gateway_pickup_controller_test.go new file mode 100644 index 00000000000..c6586a2cc17 --- /dev/null +++ b/pkg/controller/raven/gatewaypickup/gateway_pickup_controller_test.go @@ -0,0 +1,467 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gatewaypickup + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + ravenv1beta1 "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" + "github.com/openyurtio/openyurt/pkg/controller/raven/gatewaypickup/config" + "github.com/openyurtio/openyurt/pkg/controller/raven/utils" +) + +var ( + nodeReadyStatus = corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + }, + } + nodeNotReadyStatus = corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionFalse, + }, + }, + } +) + +func TestReconcileGateway_electActiveEndpoint(t *testing.T) { + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: utils.RavenGlobalConfig, + Namespace: utils.WorkingNamespace, + }, + Data: map[string]string{ + utils.RavenEnableProxy: "true", + utils.RavenEnableTunnel: "true", + }, + } + + mockReconciler := &ReconcileGateway{ + Configration: config.GatewayPickupControllerConfiguration{}, + Client: fake.NewClientBuilder().WithObjects(obj).Build(), + } + var tt = []struct { + name string + nodeList corev1.NodeList + gw *ravenv1beta1.Gateway + expectedEps []*ravenv1beta1.Endpoint + }{ + + { + // The node hosting active endpoint becomes NotReady, and it is the only node in the Gateway, + // then the active endpoint should be removed. + name: "lost active endpoint", + nodeList: corev1.NodeList{ + Items: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + }, + Status: nodeNotReadyStatus, + }, + }, + }, + gw: &ravenv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway-1", + }, + Spec: ravenv1beta1.GatewaySpec{ + ProxyConfig: ravenv1beta1.ProxyConfiguration{ + Replicas: 1, + }, + TunnelConfig: ravenv1beta1.TunnelConfiguration{ + Replicas: 1, + }, + Endpoints: []ravenv1beta1.Endpoint{ + { + NodeName: "node-1", + Type: ravenv1beta1.Tunnel, + }, + { + NodeName: "node-1", + Type: ravenv1beta1.Proxy, + }, + }, + }, + Status: ravenv1beta1.GatewayStatus{ + ActiveEndpoints: []*ravenv1beta1.Endpoint{ + { + NodeName: "node-1", + Type: ravenv1beta1.Tunnel, + }, + { + NodeName: "node-1", + Type: ravenv1beta1.Proxy, + }, + }, + }, + }, + expectedEps: []*ravenv1beta1.Endpoint{}, + }, + { + // The node hosting active endpoint becomes NotReady, but there are at least one Ready node, + // then a new endpoint should be elected active endpoint to replace the old one. + name: "switch active endpoint", + nodeList: corev1.NodeList{ + Items: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + }, + Status: nodeNotReadyStatus, + }, { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-2", + }, + Status: nodeReadyStatus, + }, + }, + }, + gw: &ravenv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway-1", + }, + Spec: ravenv1beta1.GatewaySpec{ + ProxyConfig: ravenv1beta1.ProxyConfiguration{ + Replicas: 2, + }, + TunnelConfig: ravenv1beta1.TunnelConfiguration{ + Replicas: 1, + }, + Endpoints: []ravenv1beta1.Endpoint{ + { + NodeName: "node-1", + Type: ravenv1beta1.Tunnel, + }, + { + NodeName: "node-1", + Type: ravenv1beta1.Proxy, + }, + { + NodeName: "node-2", + Type: ravenv1beta1.Tunnel, + }, + { + NodeName: "node-2", + Type: ravenv1beta1.Proxy, + }, + }, + }, + Status: ravenv1beta1.GatewayStatus{ + ActiveEndpoints: []*ravenv1beta1.Endpoint{ + { + NodeName: "node-1", + Type: ravenv1beta1.Tunnel, + }, + { + NodeName: "node-1", + Type: ravenv1beta1.Proxy, + }, + }, + }, + }, + expectedEps: []*ravenv1beta1.Endpoint{ + { + NodeName: "node-2", + Type: ravenv1beta1.Tunnel, + }, + { + NodeName: "node-2", + Type: ravenv1beta1.Proxy, + }, + }, + }, + + { + name: "elect new active endpoint", + nodeList: corev1.NodeList{ + Items: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + }, + Status: nodeNotReadyStatus, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-2", + }, + Status: nodeReadyStatus, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-3", + }, + Status: nodeReadyStatus, + }, + }, + }, + gw: &ravenv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway-1", + }, + Spec: ravenv1beta1.GatewaySpec{ + ProxyConfig: ravenv1beta1.ProxyConfiguration{ + Replicas: 2, + }, + TunnelConfig: ravenv1beta1.TunnelConfiguration{ + Replicas: 1, + }, + Endpoints: []ravenv1beta1.Endpoint{ + { + NodeName: "node-1", + Type: ravenv1beta1.Tunnel, + }, + { + NodeName: "node-2", + Type: ravenv1beta1.Proxy, + }, + { + NodeName: "node-3", + Type: ravenv1beta1.Proxy, + }, + }, + }, + Status: ravenv1beta1.GatewayStatus{ + ActiveEndpoints: []*ravenv1beta1.Endpoint{ + { + NodeName: "node-1", + Type: ravenv1beta1.Tunnel, + }, + { + NodeName: "node-2", + Type: ravenv1beta1.Proxy, + }, + }, + }, + }, + expectedEps: []*ravenv1beta1.Endpoint{ + { + NodeName: "node-2", + Type: ravenv1beta1.Proxy, + }, + { + NodeName: "node-3", + Type: ravenv1beta1.Proxy, + }, + }, + }, + + { + name: "no available active endpoint", + nodeList: corev1.NodeList{ + Items: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + }, + Status: nodeNotReadyStatus, + }, { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-2", + }, + Status: nodeNotReadyStatus, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-3", + }, + Status: nodeNotReadyStatus, + }, { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-4", + }, + Status: nodeNotReadyStatus, + }, + }, + }, + gw: &ravenv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway-1", + }, + Spec: ravenv1beta1.GatewaySpec{ + ProxyConfig: ravenv1beta1.ProxyConfiguration{ + Replicas: 2, + }, + TunnelConfig: ravenv1beta1.TunnelConfiguration{ + Replicas: 1, + }, + Endpoints: []ravenv1beta1.Endpoint{ + { + NodeName: "node-1", + Type: ravenv1beta1.Tunnel, + }, + { + NodeName: "node-2", + Type: ravenv1beta1.Tunnel, + }, + { + NodeName: "node-3", + Type: ravenv1beta1.Proxy, + }, + { + NodeName: "node-4", + Type: ravenv1beta1.Proxy, + }, + }, + }, + Status: ravenv1beta1.GatewayStatus{ + ActiveEndpoints: []*ravenv1beta1.Endpoint{ + { + NodeName: "node-1", + Type: ravenv1beta1.Tunnel, + }, + { + NodeName: "node-3", + Type: ravenv1beta1.Proxy, + }, + { + NodeName: "node-4", + Type: ravenv1beta1.Proxy, + }, + }, + }, + }, + expectedEps: []*ravenv1beta1.Endpoint{}, + }, + + { + // The node hosting the active endpoint is still ready, do not change it. + name: "don't switch active endpoint", + nodeList: corev1.NodeList{ + Items: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + }, + Status: nodeNotReadyStatus, + }, { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-2", + }, + Status: nodeReadyStatus, + }, + }, + }, + gw: &ravenv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway-1", + }, + Spec: ravenv1beta1.GatewaySpec{ + ProxyConfig: ravenv1beta1.ProxyConfiguration{ + Replicas: 1, + }, + TunnelConfig: ravenv1beta1.TunnelConfiguration{ + Replicas: 1, + }, + Endpoints: []ravenv1beta1.Endpoint{ + { + NodeName: "node-1", + Type: ravenv1beta1.Tunnel, + }, + { + NodeName: "node-2", + Type: ravenv1beta1.Tunnel, + }, + { + NodeName: "node-2", + Type: ravenv1beta1.Proxy, + }, + }, + }, + Status: ravenv1beta1.GatewayStatus{ + ActiveEndpoints: []*ravenv1beta1.Endpoint{ + { + NodeName: "node-2", + Type: ravenv1beta1.Tunnel, + }, + { + NodeName: "node-2", + Type: ravenv1beta1.Proxy, + }, + }, + }, + }, + expectedEps: []*ravenv1beta1.Endpoint{ + { + NodeName: "node-2", + Type: ravenv1beta1.Tunnel, + }, + { + NodeName: "node-2", + Type: ravenv1beta1.Proxy, + }, + }, + }, + } + for _, v := range tt { + t.Run(v.name, func(t *testing.T) { + a := assert.New(t) + eps := mockReconciler.electActiveEndpoint(v.nodeList, v.gw) + a.Equal(len(v.expectedEps), len(eps)) + }) + } + +} + +func TestReconcileGateway_getPodCIDRs(t *testing.T) { + mockReconciler := &ReconcileGateway{ + Configration: config.GatewayPickupControllerConfiguration{}, + } + var tt = []struct { + name string + node corev1.Node + expectPodCIDR []string + }{ + { + name: "node has pod CIDR", + node: corev1.Node{ + Spec: corev1.NodeSpec{ + PodCIDR: "10.0.0.1/24", + }, + }, + expectPodCIDR: []string{"10.0.0.1/24"}, + }, + { + name: "node hasn't pod CIDR", + node: corev1.Node{ + Spec: corev1.NodeSpec{}, + }, + expectPodCIDR: []string{""}, + }, + } + for _, v := range tt { + t.Run(v.name, func(t *testing.T) { + a := assert.New(t) + podCIDRs, err := mockReconciler.getPodCIDRs(context.Background(), v.node) + if a.NoError(err) { + a.Equal(v.expectPodCIDR, podCIDRs) + } + }) + } +} diff --git a/pkg/controller/raven/gatewaypickup/gateway_pickup_enqueue_handlers.go b/pkg/controller/raven/gatewaypickup/gateway_pickup_enqueue_handlers.go new file mode 100644 index 00000000000..26c57e14847 --- /dev/null +++ b/pkg/controller/raven/gatewaypickup/gateway_pickup_enqueue_handlers.go @@ -0,0 +1,179 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gatewaypickup + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + + "github.com/openyurtio/openyurt/pkg/apis/raven" + ravenv1beta1 "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" + "github.com/openyurtio/openyurt/pkg/controller/raven/utils" +) + +type EnqueueGatewayForNode struct{} + +// Create implements EventHandler +func (e *EnqueueGatewayForNode) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) { + node, ok := evt.Object.(*corev1.Node) + if !ok { + klog.Error(Format("fail to assert runtime Object to v1.Node")) + return + } + klog.V(5).Infof(Format("will enqueue gateway as node(%s) has been created", + node.GetName())) + if gwName, exist := node.Labels[raven.LabelCurrentGateway]; exist { + utils.AddGatewayToWorkQueue(gwName, q) + return + } + klog.V(4).Infof(Format("node(%s) does not belong to any gateway", node.GetName())) +} + +// Update implements EventHandler +func (e *EnqueueGatewayForNode) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) { + newNode, ok := evt.ObjectNew.(*corev1.Node) + if !ok { + klog.Errorf(Format("Fail to assert runtime Object(%s) to v1.Node", + evt.ObjectNew.GetName())) + return + } + oldNode, ok := evt.ObjectOld.(*corev1.Node) + if !ok { + klog.Errorf(Format("fail to assert runtime Object(%s) to v1.Node", + evt.ObjectOld.GetName())) + return + } + klog.V(5).Infof(Format("Will enqueue gateway as node(%s) has been updated", + newNode.GetName())) + + oldGwName := oldNode.Labels[raven.LabelCurrentGateway] + newGwName := newNode.Labels[raven.LabelCurrentGateway] + + // check if NodeReady condition changed + statusChanged := func(oldObj, newObj *corev1.Node) bool { + return isNodeReady(*oldObj) != isNodeReady(*newObj) + } + + if oldGwName != newGwName || statusChanged(oldNode, newNode) { + utils.AddGatewayToWorkQueue(oldGwName, q) + utils.AddGatewayToWorkQueue(newGwName, q) + } +} + +// Delete implements EventHandler +func (e *EnqueueGatewayForNode) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) { + node, ok := evt.Object.(*corev1.Node) + if !ok { + klog.Error(Format("Fail to assert runtime Object to v1.Node")) + return + } + + gwName, exist := node.Labels[raven.LabelCurrentGateway] + if !exist { + klog.V(5).Infof(Format("Node(%s) doesn't belong to any gateway", node.GetName())) + return + } + // enqueue the gateway that the node belongs to + klog.V(5).Infof(Format("Will enqueue pool(%s) as node(%s) has been deleted", + gwName, node.GetName())) + utils.AddGatewayToWorkQueue(gwName, q) +} + +// Generic implements EventHandler +func (e *EnqueueGatewayForNode) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) { +} + +type EnqueueGatewayForRavenConfig struct { + client client.Client +} + +func (e *EnqueueGatewayForRavenConfig) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) { + _, ok := evt.Object.(*corev1.ConfigMap) + if !ok { + klog.Error(Format("Fail to assert runtime Object to v1.ConfigMap")) + return + } + klog.V(2).Infof(Format("Will config all gateway as raven-cfg has been created")) + if err := e.enqueueGateways(q); err != nil { + klog.Error(Format("failed to config all gateway, error %s", err.Error())) + return + } +} + +func (e *EnqueueGatewayForRavenConfig) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) { + oldCm, ok := evt.ObjectOld.(*corev1.ConfigMap) + if !ok { + klog.Error(Format("Fail to assert runtime Object to v1.ConfigMap")) + return + } + + newCm, ok := evt.ObjectNew.(*corev1.ConfigMap) + if !ok { + klog.Error(Format("Fail to assert runtime Object to v1.ConfigMap")) + return + } + + if oldCm.Data[utils.RavenEnableProxy] != newCm.Data[utils.RavenEnableProxy] { + klog.V(2).Infof(Format("Will config all gateway as raven-cfg has been updated")) + if err := e.enqueueGateways(q); err != nil { + klog.Error(Format("failed to config all gateway, error %s", err.Error())) + return + } + } + + if oldCm.Data[utils.RavenEnableTunnel] != newCm.Data[utils.RavenEnableTunnel] { + klog.V(2).Infof(Format("Will config all gateway as raven-cfg has been updated")) + if err := e.enqueueGateways(q); err != nil { + klog.Error(Format("failed to config all gateway, error %s", err.Error())) + return + } + } +} + +func (e *EnqueueGatewayForRavenConfig) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) { + _, ok := evt.Object.(*corev1.ConfigMap) + if !ok { + klog.Error(Format("Fail to assert runtime Object to v1.ConfigMap")) + return + } + klog.V(2).Infof(Format("Will config all gateway as raven-cfg has been deleted")) + if err := e.enqueueGateways(q); err != nil { + klog.Error(Format("failed to config all gateway, error %s", err.Error())) + return + } +} + +func (e *EnqueueGatewayForRavenConfig) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) { + +} + +func (e *EnqueueGatewayForRavenConfig) enqueueGateways(q workqueue.RateLimitingInterface) error { + var gwList ravenv1beta1.GatewayList + err := e.client.List(context.TODO(), &gwList) + if err != nil { + return err + } + for _, gw := range gwList.Items { + utils.AddGatewayToWorkQueue(gw.Name, q) + } + return nil +} diff --git a/pkg/controller/raven/service/service_controller.go b/pkg/controller/raven/service/service_controller.go deleted file mode 100644 index 5dd811591ad..00000000000 --- a/pkg/controller/raven/service/service_controller.go +++ /dev/null @@ -1,303 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the License); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an AS IS BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package service - -import ( - "context" - "fmt" - "reflect" - - corev1 "k8s.io/api/core/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/client-go/tools/record" - "k8s.io/klog/v2" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - - appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" - ravenv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/raven/v1alpha1" - common "github.com/openyurtio/openyurt/pkg/controller/raven" - "github.com/openyurtio/openyurt/pkg/controller/raven/utils" - utilclient "github.com/openyurtio/openyurt/pkg/util/client" - utildiscovery "github.com/openyurtio/openyurt/pkg/util/discovery" -) - -var ( - controllerKind = corev1.SchemeGroupVersion.WithKind("Service") -) - -func Format(format string, args ...interface{}) string { - s := fmt.Sprintf(format, args...) - return fmt.Sprintf("%s-service: %s", common.ControllerName, s) -} - -// Add creates a new Service Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller -// and Start it when the Manager is Started. -func Add(c *appconfig.CompletedConfig, mgr manager.Manager) error { - if !utildiscovery.DiscoverGVK(controllerKind) { - return nil - } - - klog.Infof("ravenl3-service-controller add controller %s", controllerKind.String()) - return add(mgr, newReconciler(c, mgr)) -} - -var _ reconcile.Reconciler = &ReconcileService{} - -// ReconcileService reconciles a Gateway object -type ReconcileService struct { - client.Client - scheme *runtime.Scheme - recorder record.EventRecorder -} - -// newReconciler returns a new reconcile.Reconciler -func newReconciler(c *appconfig.CompletedConfig, mgr manager.Manager) reconcile.Reconciler { - return &ReconcileService{ - Client: utilclient.NewClientFromManager(mgr, common.ControllerName), - scheme: mgr.GetScheme(), - recorder: mgr.GetEventRecorderFor(common.ControllerName), - } -} - -// add adds a new Controller to mgr with r as the reconcile.Reconciler -func add(mgr manager.Manager, r reconcile.Reconciler) error { - // Create a new controller - c, err := controller.New(fmt.Sprintf("%s-service", common.ControllerName), mgr, controller.Options{ - Reconciler: r, MaxConcurrentReconciles: common.ConcurrentReconciles, - }) - if err != nil { - return err - } - - // Watch for changes to Service - err = c.Watch(&source.Kind{Type: &corev1.Service{}}, handler.EnqueueRequestsFromMapFunc(mapServiceToRequest)) - if err != nil { - return err - } - // Watch for changes to Gateway - err = c.Watch(&source.Kind{Type: &ravenv1alpha1.Gateway{}}, handler.EnqueueRequestsFromMapFunc(mapGatewayToRequest)) - if err != nil { - return err - } - - // Watch for changes to Endpoints - err = c.Watch(&source.Kind{Type: &corev1.Endpoints{}}, handler.EnqueueRequestsFromMapFunc(mapEndpointToRequest)) - if err != nil { - return err - } - - return nil -} - -// Reconcile reads that state of the cluster for a Gateway object and makes changes based on the state read -// and what is in the Gateway.Spec -func (r *ReconcileService) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - klog.V(4).Info(Format("started reconciling Service %s/%s", req.Namespace, req.Name)) - defer func() { - klog.V(4).Info(Format("finished reconciling Service %s/%s", req.Namespace, req.Name)) - }() - - var gatewayList ravenv1alpha1.GatewayList - if err := r.List(ctx, &gatewayList); err != nil { - err = fmt.Errorf(Format("unable to list gateways: %s", err)) - return reconcile.Result{}, err - } - - if err := r.reconcileService(ctx, req, &gatewayList); err != nil { - err = fmt.Errorf(Format("unable to reconcile service: %s", err)) - return reconcile.Result{}, err - } - - if err := r.reconcileEndpoint(ctx, req, &gatewayList); err != nil { - err = fmt.Errorf(Format("unable to reconcile endpoint: %s", err)) - return reconcile.Result{}, err - } - return reconcile.Result{}, nil -} - -func (r *ReconcileService) reconcileService(ctx context.Context, req ctrl.Request, gatewayList *ravenv1alpha1.GatewayList) error { - for _, gw := range gatewayList.Items { - if utils.IsGatewayExposeByLB(&gw) { - return r.ensureService(ctx, req) - } - } - return r.cleanService(ctx, req) -} - -func (r *ReconcileService) cleanService(ctx context.Context, req ctrl.Request) error { - if err := r.Delete(ctx, &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: req.Name, - Namespace: req.Namespace, - }, - }); err != nil && !apierrs.IsNotFound(err) { - return err - } - return nil -} - -func (r *ReconcileService) ensureService(ctx context.Context, req ctrl.Request) error { - var service corev1.Service - if err := r.Get(ctx, req.NamespacedName, &service); err != nil { - if apierrs.IsNotFound(err) { - klog.V(2).InfoS(Format("create service"), "name", req.Name, "namespace", req.Namespace) - return r.Create(ctx, &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: req.Name, - Namespace: req.Namespace, - }, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Port: 4500, - Protocol: corev1.ProtocolUDP, - TargetPort: intstr.FromInt(4500), - }, - }, - Type: corev1.ServiceTypeLoadBalancer, - ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyTypeLocal, - }, - }) - - } - } - return nil -} - -func (r *ReconcileService) reconcileEndpoint(ctx context.Context, req ctrl.Request, gatewayList *ravenv1alpha1.GatewayList) error { - exposedByLB := false - for _, gw := range gatewayList.Items { - if utils.IsGatewayExposeByLB(&gw) { - exposedByLB = true - if gw.Status.ActiveEndpoint != nil { - var node corev1.Node - if err := r.Get(ctx, types.NamespacedName{ - Name: gw.Status.ActiveEndpoint.NodeName, - }, &node); err != nil { - return err - } - return r.ensureEndpoint(ctx, req, node) - } - } - } - if !exposedByLB { - return r.cleanEndpoint(ctx, req) - } - return nil -} - -func (r *ReconcileService) cleanEndpoint(ctx context.Context, req ctrl.Request) error { - if err := r.Delete(ctx, &corev1.Endpoints{ - ObjectMeta: metav1.ObjectMeta{ - Name: req.Name, - Namespace: req.Namespace, - }, - }); err != nil && !apierrs.IsNotFound(err) { - return err - } - return nil -} - -func (r *ReconcileService) ensureEndpoint(ctx context.Context, req ctrl.Request, node corev1.Node) error { - var serviceEndpoint corev1.Endpoints - newSubnets := []corev1.EndpointSubset{ - { - Addresses: []corev1.EndpointAddress{ - { - IP: utils.GetNodeInternalIP(node), - NodeName: func(n corev1.Node) *string { return &n.Name }(node), - }, - }, - Ports: []corev1.EndpointPort{ - { - Port: 4500, - Protocol: corev1.ProtocolUDP, - }, - }, - }, - } - if err := r.Get(ctx, req.NamespacedName, &serviceEndpoint); err != nil { - if apierrs.IsNotFound(err) { - klog.V(2).InfoS(Format("create endpoint"), "name", req.Name, "namespace", req.Namespace) - return r.Create(ctx, &corev1.Endpoints{ - ObjectMeta: metav1.ObjectMeta{ - Name: req.Name, - Namespace: req.Namespace, - }, - Subsets: newSubnets, - }) - } - return err - } - - if !reflect.DeepEqual(serviceEndpoint.Subsets, newSubnets) { - klog.V(2).InfoS(Format("update endpoint"), "name", req.Name, "namespace", req.Namespace) - serviceEndpoint.Subsets = newSubnets - return r.Update(ctx, &serviceEndpoint) - } - klog.V(2).InfoS(Format("skip to update endpoint"), "name", req.Name, "namespace", req.Namespace) - return nil -} - -// mapGatewayToRequest maps the given Gateway object to reconcile.Request. -func mapGatewayToRequest(object client.Object) []reconcile.Request { - gw, ok := object.(*ravenv1alpha1.Gateway) - if ok && utils.IsGatewayExposeByLB(gw) { - return []reconcile.Request{ - { - NamespacedName: ravenv1alpha1.ServiceNamespacedName, - }, - } - } - return []reconcile.Request{} -} - -// mapEndpointToRequest maps the given Endpoint object to reconcile.Request. -func mapEndpointToRequest(object client.Object) []reconcile.Request { - ep, ok := object.(*corev1.Endpoints) - if ok && ep.Name == ravenv1alpha1.ServiceNamespacedName.Name && ep.Namespace == ravenv1alpha1.ServiceNamespacedName.Namespace { - return []reconcile.Request{ - { - NamespacedName: ravenv1alpha1.ServiceNamespacedName, - }, - } - } - return []reconcile.Request{} -} - -// mapEndpointToRequest maps the given Endpoint object to reconcile.Request. -func mapServiceToRequest(object client.Object) []reconcile.Request { - svc, ok := object.(*corev1.Service) - if ok && svc.Name == ravenv1alpha1.ServiceNamespacedName.Name && svc.Namespace == ravenv1alpha1.ServiceNamespacedName.Namespace { - return []reconcile.Request{ - { - NamespacedName: ravenv1alpha1.ServiceNamespacedName, - }, - } - } - return []reconcile.Request{} -} diff --git a/pkg/controller/raven/utils/utils.go b/pkg/controller/raven/utils/utils.go index 71609f083b9..0bb880c42f0 100644 --- a/pkg/controller/raven/utils/utils.go +++ b/pkg/controller/raven/utils/utils.go @@ -17,16 +17,27 @@ limitations under the License. package utils import ( + "context" "net" + "strings" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/openyurtio/openyurt/pkg/apis/raven/v1alpha1" ) +const ( + WorkingNamespace = "kube-system" + RavenGlobalConfig = "raven-cfg" + + RavenEnableProxy = "EnableL7Proxy" + RavenEnableTunnel = "EnableL3Tunnel" +) + // GetNodeInternalIP returns internal ip of the given `node`. func GetNodeInternalIP(node corev1.Node) string { var ip string @@ -52,3 +63,21 @@ func AddGatewayToWorkQueue(gwName string, }) } } + +func CheckServer(ctx context.Context, client client.Client) (enableProxy, enableTunnel bool) { + var cm corev1.ConfigMap + enableTunnel = false + enableProxy = false + err := client.Get(ctx, types.NamespacedName{Namespace: WorkingNamespace, Name: RavenGlobalConfig}, &cm) + if err != nil { + return enableProxy, enableTunnel + } + if val, ok := cm.Data[RavenEnableProxy]; ok && strings.ToLower(val) == "true" { + enableProxy = true + } + if val, ok := cm.Data[RavenEnableTunnel]; ok && strings.ToLower(val) == "true" { + enableTunnel = true + } + return enableProxy, enableTunnel + +} From 4bd641459e5bba87f0875fb99f029d7d00e5f645 Mon Sep 17 00:00:00 2001 From: rambohe Date: Thu, 17 Aug 2023 12:13:23 +0800 Subject: [PATCH 18/21] remove some client global vars (#1655) Signed-off-by: rambohe-ch --- cmd/yurt-manager/app/manager.go | 9 -- pkg/client/generic_client.go | 57 ------------ pkg/client/registry.go | 56 ------------ .../daemon_pod_updater_controller.go | 7 +- .../platformadmin/platformadmin_controller.go | 13 ++- .../gateway_pickup_controller.go | 15 ++- .../endpoints/endpoints_controller.go | 5 - .../yurtappdaemon/yurtappdaemon_controller.go | 11 +-- .../yurtappset/yurtappset_controller.go | 13 +-- .../yurtstaticset/yurtstaticset_controller.go | 13 ++- pkg/profile/profile_test.go | 53 ++++++++--- pkg/util/client/client.go | 34 ------- pkg/util/discovery/discovery.go | 91 ------------------- .../util/controller/webhook_controller.go | 7 +- 14 files changed, 74 insertions(+), 310 deletions(-) delete mode 100644 pkg/client/generic_client.go delete mode 100644 pkg/client/registry.go delete mode 100644 pkg/util/client/client.go delete mode 100644 pkg/util/discovery/discovery.go diff --git a/cmd/yurt-manager/app/manager.go b/cmd/yurt-manager/app/manager.go index fd328f28674..f615dfd103a 100644 --- a/cmd/yurt-manager/app/manager.go +++ b/cmd/yurt-manager/app/manager.go @@ -37,7 +37,6 @@ import ( "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" "github.com/openyurtio/openyurt/cmd/yurt-manager/app/options" "github.com/openyurtio/openyurt/pkg/apis" - extclient "github.com/openyurtio/openyurt/pkg/client" "github.com/openyurtio/openyurt/pkg/controller" "github.com/openyurtio/openyurt/pkg/profile" "github.com/openyurtio/openyurt/pkg/projectinfo" @@ -151,16 +150,8 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error { ctx := ctrl.SetupSignalHandler() cfg := ctrl.GetConfigOrDie() - setRestConfig(cfg, c) - setupLog.Info("new clientset registry") - err := extclient.NewRegistry(cfg) - if err != nil { - setupLog.Error(err, "unable to init yurt-manager clientset and informer") - os.Exit(1) - } - mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme, MetricsBindAddress: c.ComponentConfig.Generic.MetricsAddr, diff --git a/pkg/client/generic_client.go b/pkg/client/generic_client.go deleted file mode 100644 index 1ee4a106d8d..00000000000 --- a/pkg/client/generic_client.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package client - -import ( - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/discovery" - kubeclientset "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" -) - -// GenericClientset defines a generic client -type GenericClientset struct { - DiscoveryClient discovery.DiscoveryInterface - KubeClient kubeclientset.Interface -} - -// newForConfig creates a new Clientset for the given config. -func newForConfig(c *rest.Config) (*GenericClientset, error) { - cWithProtobuf := rest.CopyConfig(c) - cWithProtobuf.ContentType = runtime.ContentTypeProtobuf - discoveryClient, err := discovery.NewDiscoveryClientForConfig(cWithProtobuf) - if err != nil { - return nil, err - } - kubeClient, err := kubeclientset.NewForConfig(cWithProtobuf) - if err != nil { - return nil, err - } - return &GenericClientset{ - DiscoveryClient: discoveryClient, - KubeClient: kubeClient, - }, nil -} - -// newForConfig creates a new Clientset for the given config. -func newForConfigOrDie(c *rest.Config) *GenericClientset { - gc, err := newForConfig(c) - if err != nil { - panic(err) - } - return gc -} diff --git a/pkg/client/registry.go b/pkg/client/registry.go deleted file mode 100644 index c94aace5eff..00000000000 --- a/pkg/client/registry.go +++ /dev/null @@ -1,56 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package client - -import ( - "fmt" - - "k8s.io/client-go/rest" -) - -var ( - cfg *rest.Config - - defaultGenericClient *GenericClientset -) - -// NewRegistry creates clientset by client-go -func NewRegistry(c *rest.Config) error { - var err error - defaultGenericClient, err = newForConfig(c) - if err != nil { - return err - } - cfgCopy := *c - cfg = &cfgCopy - return nil -} - -// GetGenericClient returns default clientset -func GetGenericClient() *GenericClientset { - return defaultGenericClient -} - -// GetGenericClientWithName returns clientset with given name as user-agent -func GetGenericClientWithName(name string) *GenericClientset { - if cfg == nil { - return nil - } - newCfg := *cfg - newCfg.UserAgent = fmt.Sprintf("%s/%s", cfg.UserAgent, name) - return newForConfigOrDie(&newCfg) -} diff --git a/pkg/controller/daemonpodupdater/daemon_pod_updater_controller.go b/pkg/controller/daemonpodupdater/daemon_pod_updater_controller.go index 6c06ea36e19..95e27645a13 100644 --- a/pkg/controller/daemonpodupdater/daemon_pod_updater_controller.go +++ b/pkg/controller/daemonpodupdater/daemon_pod_updater_controller.go @@ -50,8 +50,6 @@ import ( appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" k8sutil "github.com/openyurtio/openyurt/pkg/controller/daemonpodupdater/kubernetes" - utilclient "github.com/openyurtio/openyurt/pkg/util/client" - utildiscovery "github.com/openyurtio/openyurt/pkg/util/discovery" ) func init() { @@ -103,9 +101,6 @@ func Format(format string, args ...interface{}) string { // Add creates a new Daemonpodupdater Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(c *appconfig.CompletedConfig, mgr manager.Manager) error { - if !utildiscovery.DiscoverGVK(controllerKind) { - return nil - } klog.Infof("daemonupdater-controller add controller %s", controllerKind.String()) return add(mgr, newReconciler(c, mgr)) } @@ -123,7 +118,7 @@ type ReconcileDaemonpodupdater struct { // newReconciler returns a new reconcile.Reconciler func newReconciler(_ *appconfig.CompletedConfig, mgr manager.Manager) reconcile.Reconciler { return &ReconcileDaemonpodupdater{ - Client: utilclient.NewClientFromManager(mgr, ControllerName), + Client: mgr.GetClient(), expectations: k8sutil.NewControllerExpectations(), recorder: mgr.GetEventRecorderFor(ControllerName), } diff --git a/pkg/controller/platformadmin/platformadmin_controller.go b/pkg/controller/platformadmin/platformadmin_controller.go index dddb4178a9e..1edc362cfa1 100644 --- a/pkg/controller/platformadmin/platformadmin_controller.go +++ b/pkg/controller/platformadmin/platformadmin_controller.go @@ -49,8 +49,6 @@ import ( iotv1alpha2 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha2" "github.com/openyurtio/openyurt/pkg/controller/platformadmin/config" util "github.com/openyurtio/openyurt/pkg/controller/platformadmin/utils" - utilclient "github.com/openyurtio/openyurt/pkg/util/client" - utildiscovery "github.com/openyurtio/openyurt/pkg/util/discovery" ) func init() { @@ -64,7 +62,7 @@ func Format(format string, args ...interface{}) string { var ( concurrentReconciles = 3 - controllerKind = iotv1alpha2.SchemeGroupVersion.WithKind("PlatformAdmin") + controllerResource = iotv1alpha2.SchemeGroupVersion.WithResource("platformadmins") ) const ( @@ -121,18 +119,19 @@ var _ reconcile.Reconciler = &ReconcilePlatformAdmin{} // Add creates a new PlatformAdmin Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(c *appconfig.CompletedConfig, mgr manager.Manager) error { - if !utildiscovery.DiscoverGVK(controllerKind) { - return nil + if _, err := mgr.GetRESTMapper().KindFor(controllerResource); err != nil { + klog.Infof("resource %s doesn't exist", controllerResource.String()) + return err } - klog.Infof("platformadmin-controller add controller %s", controllerKind.String()) + klog.Infof("platformadmin-controller add controller %s", controllerResource.String()) return add(mgr, newReconciler(c, mgr)) } // newReconciler returns a new reconcile.Reconciler func newReconciler(c *appconfig.CompletedConfig, mgr manager.Manager) reconcile.Reconciler { return &ReconcilePlatformAdmin{ - Client: utilclient.NewClientFromManager(mgr, ControllerName), + Client: mgr.GetClient(), scheme: mgr.GetScheme(), recorder: mgr.GetEventRecorderFor(ControllerName), yamlSerializer: kjson.NewSerializerWithOptions(kjson.DefaultMetaFactory, scheme.Scheme, scheme.Scheme, kjson.SerializerOptions{Yaml: true, Pretty: true}), diff --git a/pkg/controller/raven/gatewaypickup/gateway_pickup_controller.go b/pkg/controller/raven/gatewaypickup/gateway_pickup_controller.go index 75d26b9477d..24d8337a21a 100644 --- a/pkg/controller/raven/gatewaypickup/gateway_pickup_controller.go +++ b/pkg/controller/raven/gatewaypickup/gateway_pickup_controller.go @@ -46,12 +46,10 @@ import ( "github.com/openyurtio/openyurt/pkg/controller/raven/gatewaypickup/config" "github.com/openyurtio/openyurt/pkg/controller/raven/utils" nodeutil "github.com/openyurtio/openyurt/pkg/controller/util/node" - utilclient "github.com/openyurtio/openyurt/pkg/util/client" - utildiscovery "github.com/openyurtio/openyurt/pkg/util/discovery" ) var ( - controllerKind = ravenv1beta1.SchemeGroupVersion.WithKind("Gateway") + controllerResource = ravenv1beta1.SchemeGroupVersion.WithResource("gateways") ) func Format(format string, args ...interface{}) string { @@ -68,10 +66,11 @@ const ( // Add creates a new Gateway Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(c *appconfig.CompletedConfig, mgr manager.Manager) error { - if !utildiscovery.DiscoverGVK(controllerKind) { - return nil + if _, err := mgr.GetRESTMapper().KindFor(controllerResource); err != nil { + klog.Infof("resource %s doesn't exist", controllerResource.String()) + return err } - klog.Infof("raven-gateway-controller add controller %s", controllerKind.String()) + klog.Infof("raven-gateway-controller add controller %s", controllerResource.String()) return add(mgr, newReconciler(c, mgr)) } @@ -88,7 +87,7 @@ type ReconcileGateway struct { // newReconciler returns a new reconcile.Reconciler func newReconciler(c *appconfig.CompletedConfig, mgr manager.Manager) reconcile.Reconciler { return &ReconcileGateway{ - Client: utilclient.NewClientFromManager(mgr, common.GatewayPickupControllerName), + Client: mgr.GetClient(), scheme: mgr.GetScheme(), recorder: mgr.GetEventRecorderFor(common.GatewayPickupControllerName), Configration: c.ComponentConfig.GatewayPickupController, @@ -117,7 +116,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { return err } - err = c.Watch(&source.Kind{Type: &corev1.ConfigMap{}}, &EnqueueGatewayForRavenConfig{client: utilclient.NewClientFromManager(mgr, "raven-config")}, predicate.NewPredicateFuncs( + err = c.Watch(&source.Kind{Type: &corev1.ConfigMap{}}, &EnqueueGatewayForRavenConfig{client: mgr.GetClient()}, predicate.NewPredicateFuncs( func(object client.Object) bool { cm, ok := object.(*corev1.ConfigMap) if !ok { diff --git a/pkg/controller/servicetopology/endpoints/endpoints_controller.go b/pkg/controller/servicetopology/endpoints/endpoints_controller.go index ed1c15ad370..e35426e662f 100644 --- a/pkg/controller/servicetopology/endpoints/endpoints_controller.go +++ b/pkg/controller/servicetopology/endpoints/endpoints_controller.go @@ -33,7 +33,6 @@ import ( appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" common "github.com/openyurtio/openyurt/pkg/controller/servicetopology" "github.com/openyurtio/openyurt/pkg/controller/servicetopology/adapter" - utildiscovery "github.com/openyurtio/openyurt/pkg/util/discovery" ) func init() { @@ -53,10 +52,6 @@ func Format(format string, args ...interface{}) string { // Add creates a new Servicetopology endpoints Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(c *appconfig.CompletedConfig, mgr manager.Manager) error { - if !utildiscovery.DiscoverGVK(controllerKind) { - return nil - } - klog.Infof("servicetopology-endpoints-controller add controller %s", controllerKind.String()) return add(mgr, newReconciler(c, mgr)) } diff --git a/pkg/controller/yurtappdaemon/yurtappdaemon_controller.go b/pkg/controller/yurtappdaemon/yurtappdaemon_controller.go index 9d5cff753f8..0434ec7b70f 100644 --- a/pkg/controller/yurtappdaemon/yurtappdaemon_controller.go +++ b/pkg/controller/yurtappdaemon/yurtappdaemon_controller.go @@ -41,12 +41,11 @@ import ( unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" "github.com/openyurtio/openyurt/pkg/controller/util" "github.com/openyurtio/openyurt/pkg/controller/yurtappdaemon/workloadcontroller" - utildiscovery "github.com/openyurtio/openyurt/pkg/util/discovery" ) var ( concurrentReconciles = 3 - controllerKind = unitv1alpha1.SchemeGroupVersion.WithKind("YurtAppDaemon") + controllerResource = unitv1alpha1.SchemeGroupVersion.WithResource("yurtappdaemons") ) const ( @@ -74,12 +73,12 @@ func Format(format string, args ...interface{}) string { // The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(c *config.CompletedConfig, mgr manager.Manager) error { - if !utildiscovery.DiscoverGVK(controllerKind) { - klog.Errorf(Format("DiscoverGVK error")) - return nil + if _, err := mgr.GetRESTMapper().KindFor(controllerResource); err != nil { + klog.Infof("resource %s doesn't exist", controllerResource.String()) + return err } - klog.Infof("yurtappdaemon-controller add controller %s", controllerKind.String()) + klog.Infof("yurtappdaemon-controller add controller %s", controllerResource.String()) return add(mgr, newReconciler(mgr)) } diff --git a/pkg/controller/yurtappset/yurtappset_controller.go b/pkg/controller/yurtappset/yurtappset_controller.go index 572403328f1..eeeb4c79e49 100644 --- a/pkg/controller/yurtappset/yurtappset_controller.go +++ b/pkg/controller/yurtappset/yurtappset_controller.go @@ -43,7 +43,6 @@ import ( "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" "github.com/openyurtio/openyurt/pkg/controller/yurtappset/adapter" - utildiscovery "github.com/openyurtio/openyurt/pkg/util/discovery" ) func init() { @@ -52,7 +51,7 @@ func init() { var ( concurrentReconciles = 3 - controllerKind = unitv1alpha1.SchemeGroupVersion.WithKind("YurtAppSet") + controllerResource = unitv1alpha1.SchemeGroupVersion.WithResource("yurtappsets") ) const ( @@ -75,19 +74,17 @@ func Format(format string, args ...interface{}) string { // Add creates a new YurtAppSet Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(c *config.CompletedConfig, mgr manager.Manager) error { - if !utildiscovery.DiscoverGVK(controllerKind) { - klog.Errorf(Format("DiscoverGVK error")) - return nil + if _, err := mgr.GetRESTMapper().KindFor(controllerResource); err != nil { + klog.Infof("resource %s doesn't exist", controllerResource.String()) + return err } - klog.Infof("yurtappset-controller add controller %s", controllerKind.String()) + klog.Infof("yurtappset-controller add controller %s", controllerResource.String()) return add(mgr, newReconciler(c, mgr)) } // newReconciler returns a new reconcile.Reconciler func newReconciler(c *config.CompletedConfig, mgr manager.Manager) reconcile.Reconciler { - klog.Infof("yurtappset-controller newReconciler %s", controllerKind.String()) - return &ReconcileYurtAppSet{ Client: mgr.GetClient(), scheme: mgr.GetScheme(), diff --git a/pkg/controller/yurtstaticset/yurtstaticset_controller.go b/pkg/controller/yurtstaticset/yurtstaticset_controller.go index c3469a05645..2047b64a051 100644 --- a/pkg/controller/yurtstaticset/yurtstaticset_controller.go +++ b/pkg/controller/yurtstaticset/yurtstaticset_controller.go @@ -45,8 +45,6 @@ import ( "github.com/openyurtio/openyurt/pkg/controller/yurtstaticset/config" "github.com/openyurtio/openyurt/pkg/controller/yurtstaticset/upgradeinfo" "github.com/openyurtio/openyurt/pkg/controller/yurtstaticset/util" - utilclient "github.com/openyurtio/openyurt/pkg/util/client" - utildiscovery "github.com/openyurtio/openyurt/pkg/util/discovery" ) func init() { @@ -55,7 +53,7 @@ func init() { var ( concurrentReconciles = 3 - controllerKind = appsv1alpha1.SchemeGroupVersion.WithKind("YurtStaticSet") + controllerResource = appsv1alpha1.SchemeGroupVersion.WithResource("yurtstaticsets") True = true ) @@ -128,11 +126,12 @@ func Format(format string, args ...interface{}) string { // Add creates a new YurtStaticSet Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(c *appconfig.CompletedConfig, mgr manager.Manager) error { - if !utildiscovery.DiscoverGVK(controllerKind) { - return nil + if _, err := mgr.GetRESTMapper().KindFor(controllerResource); err != nil { + klog.Infof("resource %s doesn't exist", controllerResource.String()) + return err } - klog.Infof("yurtstaticset-controller add controller %s", controllerKind.String()) + klog.Infof("yurtstaticset-controller add controller %s", controllerResource.String()) return add(mgr, newReconciler(c, mgr)) } @@ -149,7 +148,7 @@ type ReconcileYurtStaticSet struct { // newReconciler returns a new reconcile.Reconciler func newReconciler(c *appconfig.CompletedConfig, mgr manager.Manager) reconcile.Reconciler { return &ReconcileYurtStaticSet{ - Client: utilclient.NewClientFromManager(mgr, ControllerName), + Client: mgr.GetClient(), scheme: mgr.GetScheme(), recorder: mgr.GetEventRecorderFor(ControllerName), Configuration: c.ComponentConfig.YurtStaticSetController, diff --git a/pkg/profile/profile_test.go b/pkg/profile/profile_test.go index cfb7f2e7e78..998b6f91a75 100644 --- a/pkg/profile/profile_test.go +++ b/pkg/profile/profile_test.go @@ -18,28 +18,53 @@ package profile import ( "net/http" + "net/http/httptest" "testing" "github.com/gorilla/mux" ) -func fakeServer(h http.Handler) error { - err := http.ListenAndServe(":9090", h) - return err -} - +// TestInstall checks Install function correctly sets up routes func TestInstall(t *testing.T) { - t.Run("TestInstall", func(t *testing.T) { - m := mux.NewRouter() - Install(m) - go fakeServer(m) - r, err := http.Get("http://localhost:9090/debug/pprof/") + router := mux.NewRouter() + Install(router) + + // Define the routes we expect to exist and their expected handlers + expectedRoutes := map[string]string{ + "/debug/pprof/profile": "profile", + "/debug/pprof/symbol": "symbol", + "/debug/pprof/trace": "trace", + } + + // For each route, make a request and check the handler that's called + for route := range expectedRoutes { + req, err := http.NewRequest("GET", route, nil) if err != nil { - t.Errorf("failed to send request to fake server, %v", err) + t.Fatal(err) } + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) - if r.StatusCode != http.StatusOK { - t.Error(err) + if rr.Code != http.StatusOK { + t.Errorf("handler(%s) returned wrong status code: got %v want %v", route, rr.Code, http.StatusOK) } - }) + } +} + +// TestRedirectTo checks redirect to the desired location +func TestRedirectTo(t *testing.T) { + destination := "/destination" + redirect := redirectTo(destination) + + req, err := http.NewRequest("GET", "/", nil) + if err != nil { + t.Fatal(err) + } + rr := httptest.NewRecorder() + + redirect(rr, req) + + if location := rr.Header().Get("Location"); location != destination { + t.Errorf("expected redirect to %s, got %s", destination, location) + } } diff --git a/pkg/util/client/client.go b/pkg/util/client/client.go deleted file mode 100644 index 46f8e9b181f..00000000000 --- a/pkg/util/client/client.go +++ /dev/null @@ -1,34 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package client - -import ( - "fmt" - - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/cluster" - "sigs.k8s.io/controller-runtime/pkg/manager" -) - -func NewClientFromManager(mgr manager.Manager, name string) client.Client { - cfg := rest.CopyConfig(mgr.GetConfig()) - cfg.UserAgent = fmt.Sprintf("yurt-manager/%s", name) - - delegatingClient, _ := cluster.DefaultNewClient(mgr.GetCache(), cfg, client.Options{Scheme: mgr.GetScheme(), Mapper: mgr.GetRESTMapper()}) - return delegatingClient -} diff --git a/pkg/util/discovery/discovery.go b/pkg/util/discovery/discovery.go deleted file mode 100644 index de75f164e26..00000000000 --- a/pkg/util/discovery/discovery.go +++ /dev/null @@ -1,91 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package discovery - -import ( - "fmt" - "time" - - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/util/retry" - "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - - "github.com/openyurtio/openyurt/pkg/apis" - "github.com/openyurtio/openyurt/pkg/client" -) - -var ( - internalScheme = runtime.NewScheme() - - errKindNotFound = fmt.Errorf("kind not found in group version resources") - backOff = wait.Backoff{ - Steps: 4, - Duration: 500 * time.Millisecond, - Factor: 5.0, - Jitter: 0.1, - } -) - -func init() { - _ = apis.AddToScheme(internalScheme) -} - -func DiscoverGVK(gvk schema.GroupVersionKind) bool { - genericClient := client.GetGenericClient() - if genericClient == nil { - return true - } - discoveryClient := genericClient.DiscoveryClient - - startTime := time.Now() - err := retry.OnError(backOff, func(err error) bool { return true }, func() error { - resourceList, err := discoveryClient.ServerResourcesForGroupVersion(gvk.GroupVersion().String()) - if err != nil { - return err - } - for _, r := range resourceList.APIResources { - if r.Kind == gvk.Kind { - return nil - } - } - return errKindNotFound - }) - - if err != nil { - if err == errKindNotFound { - klog.Warningf("Not found kind %s in group version %s, waiting time %s", gvk.Kind, gvk.GroupVersion().String(), time.Since(startTime)) - return false - } - - // This might be caused by abnormal apiserver or etcd, ignore it - klog.Errorf("Failed to find resources in group version %s: %v, waiting time %s", gvk.GroupVersion().String(), err, time.Since(startTime)) - } - - return true -} - -func DiscoverObject(obj runtime.Object) bool { - gvk, err := apiutil.GVKForObject(obj, internalScheme) - if err != nil { - klog.Warningf("Not recognized object %T in scheme: %v", obj, err) - return false - } - return DiscoverGVK(gvk) -} diff --git a/pkg/webhook/util/controller/webhook_controller.go b/pkg/webhook/util/controller/webhook_controller.go index 5ca3e807bbf..2734dad20e9 100644 --- a/pkg/webhook/util/controller/webhook_controller.go +++ b/pkg/webhook/util/controller/webhook_controller.go @@ -43,7 +43,6 @@ import ( "k8s.io/klog/v2" "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" - extclient "github.com/openyurtio/openyurt/pkg/client" webhookutil "github.com/openyurtio/openyurt/pkg/webhook/util" "github.com/openyurtio/openyurt/pkg/webhook/util/configuration" "github.com/openyurtio/openyurt/pkg/webhook/util/generator" @@ -80,8 +79,12 @@ type Controller struct { } func New(handlers map[string]struct{}, cc *config.CompletedConfig, restCfg *rest.Config) (*Controller, error) { + kubeClient, err := clientset.NewForConfig(restCfg) + if err != nil { + return nil, err + } c := &Controller{ - kubeClient: extclient.GetGenericClientWithName("webhook-controller").KubeClient, + kubeClient: kubeClient, handlers: handlers, queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "webhook-controller"), webhookPort: cc.ComponentConfig.Generic.WebhookPort, From de07563bea443632f7723672122e6eb04bc17269 Mon Sep 17 00:00:00 2001 From: rambohe Date: Thu, 17 Aug 2023 14:54:24 +0800 Subject: [PATCH 19/21] move controller and webhook into pkg/yurtmanager (#1661) Signed-off-by: rambohe-ch --- cmd/yurt-manager/app/config/config.go | 2 +- cmd/yurt-manager/app/manager.go | 8 ++--- .../app/options/gatewaycontroller.go | 2 +- cmd/yurt-manager/app/options/generic.go | 2 +- .../app/options/nodepoolcontroller.go | 2 +- .../app/options/platformadmincontroller.go | 2 +- .../app/options/yurtappdaemoncontroller.go | 2 +- .../app/options/yurtappsetcontroller.go | 2 +- .../app/options/yurtstaticsetcontroller.go | 2 +- pkg/{ => util}/profile/profile.go | 0 pkg/{ => util}/profile/profile_test.go | 0 pkg/yurtadm/cmd/join/join.go | 2 +- pkg/yurtadm/cmd/staticpods/install/install.go | 2 +- .../cmd/staticpods/uninstall/uninstall.go | 2 +- pkg/yurthub/otaupdate/ota.go | 2 +- pkg/yurthub/otaupdate/upgrader/static_pod.go | 2 +- .../otaupdate/upgrader/static_pod_test.go | 2 +- pkg/yurthub/otaupdate/util/util.go | 2 +- pkg/yurthub/server/server.go | 2 +- .../controller/apis/config/types.go | 12 +++---- .../controller/controller.go | 32 ++++++++--------- .../csrapprover/csrapprover_controller.go | 2 +- .../csrapprover_controller_test.go | 0 .../daemon_pod_updater_controller.go | 2 +- .../daemon_pod_updater_controller_test.go | 2 +- .../kubernetes/controller_utils.go | 0 .../kubernetes/controller_utils_test.go | 0 .../daemonpodupdater/kubernetes/pod_util.go | 0 .../kubernetes/pod_util_test.go | 0 .../controller/daemonpodupdater/util.go | 8 ++--- .../controller/daemonpodupdater/util_test.go | 0 .../controller/nodepool/config/types.go | 0 .../nodepool/nodepool_controller.go | 2 +- .../nodepool/nodepool_controller_test.go | 2 +- .../nodepool/nodepool_enqueue_handlers.go | 0 .../nodepool_enqueue_handlers_test.go | 0 .../controller/nodepool/util.go | 2 +- .../controller/nodepool/util_test.go | 0 .../config/EdgeXConfig/config-nosecty.json | 0 .../config/EdgeXConfig/config.json | 0 .../config/EdgeXConfig/manifest.yaml | 0 .../controller/platformadmin/config/types.go | 0 .../platformadmin/platformadmin_controller.go | 4 +-- .../controller/platformadmin/util.go | 4 +-- .../platformadmin/utils/conditions.go | 0 .../platformadmin/utils/fieldindexer.go | 0 .../controller/platformadmin/utils/version.go | 0 .../controller/raven/common.go | 0 .../raven/gatewaypickup/config/types.go | 0 .../gateway_pickup_controller.go | 8 ++--- .../gateway_pickup_controller_test.go | 4 +-- .../gateway_pickup_enqueue_handlers.go | 2 +- .../controller/raven/utils/utils.go | 0 .../servicetopology/adapter/adapter.go | 0 .../adapter/endpoints_adapter.go | 0 .../adapter/endpoints_adapter_test.go | 0 .../adapter/endpointslicev1_adapter.go | 0 .../adapter/endpointslicev1_adapter_test.go | 0 .../adapter/endpointslicev1beta1_adapter.go | 0 .../endpointslicev1beta1_adapter_test.go | 0 .../controller/servicetopology/common.go | 0 .../endpoints/endpoints_controller.go | 4 +-- .../endpoints/endpoints_enqueue_handlers.go | 4 +-- .../endpointslice/endpointslice_controller.go | 4 +-- .../endpointslice_enqueue_handlers.go | 4 +-- .../controller/servicetopology/util/util.go | 0 .../controller/util/controller_utils.go | 0 .../controller/util/controller_utils_test.go | 0 .../controller/util/node/controller_utils.go | 0 .../controller/util/refmanager/ref_manager.go | 0 .../util/refmanager/ref_manager_test.go | 0 .../controller/util/tools.go | 0 .../controller/util/tools_test.go | 0 .../controller/yurtappdaemon/config/types.go | 0 .../nodepool_enqueue_handlers.go | 0 .../nodepool_enqueue_handlers_test.go | 0 .../controller/yurtappdaemon/revision.go | 2 +- .../controller/yurtappdaemon/revision_test.go | 0 .../controller/yurtappdaemon/util.go | 0 .../controller/yurtappdaemon/util_test.go | 0 .../workloadcontroller/controller.go | 0 .../deployment_controller.go | 2 +- .../deployment_controller_test.go | 0 .../statefulset_controller.go | 0 .../yurtappdaemon/workloadcontroller/util.go | 0 .../workloadcontroller/util_test.go | 0 .../workloadcontroller/workload.go | 0 .../workloadcontroller/workload_test.go | 0 .../yurtappdaemon/yurtappdaemon_controller.go | 4 +-- .../yurtappdaemon_controller_test.go | 2 +- .../controller/yurtappset/adapter/adapter.go | 0 .../yurtappset/adapter/adapter_util.go | 0 .../yurtappset/adapter/adapter_util_test.go | 0 .../yurtappset/adapter/deployment_adapter.go | 0 .../adapter/deployment_adapter_test.go | 0 .../yurtappset/adapter/statefulset_adapter.go | 0 .../adapter/statefulset_adapter_test.go | 0 .../controller/yurtappset/config/types.go | 0 .../controller/yurtappset/pool.go | 2 +- .../controller/yurtappset/pool_control.go | 4 +-- .../yurtappset/pool_controller_test.go | 2 +- .../controller/yurtappset/revision.go | 2 +- .../controller/yurtappset/revision_test.go | 0 .../yurtappset/yurtappset_controller.go | 2 +- .../yurtappset_controller_statefulset_test.go | 0 .../yurtappset_controller_suite_test.go | 0 .../yurtappset/yurtappset_controller_test.go | 6 ++-- .../yurtappset/yurtappset_controller_utils.go | 0 .../yurtappset/yurtappset_update.go | 2 +- .../yurtcoordinator/cert/certificate.go | 0 .../yurtcoordinator/cert/certificate_test.go | 0 .../controller/yurtcoordinator/cert/secret.go | 0 .../yurtcoordinator/cert/secret_test.go | 0 .../controller/yurtcoordinator/cert/util.go | 0 .../yurtcoordinator/cert/util_test.go | 0 .../cert/yurtcoordinatorcert_controller.go | 0 .../yurtcoordinatorcert_controller_test.go | 0 .../yurtcoordinator/constant/constant.go | 0 .../delegatelease/delegatelease_controller.go | 6 ++-- .../delegatelease_controller_test.go | 2 +- .../podbinding/podbinding_controller.go | 2 +- .../podbinding/podbinding_controller_test.go | 0 .../controller/yurtcoordinator/utils/lease.go | 2 +- .../yurtcoordinator/utils/lease_test.go | 0 .../yurtcoordinator/utils/taints.go | 0 .../yurtcoordinator/utils/taints_test.go | 2 +- .../controller/yurtstaticset/config/types.go | 0 .../yurtstaticset/upgradeinfo/upgrade_info.go | 2 +- .../upgradeinfo/upgrade_info_test.go | 0 .../controller/yurtstaticset/util/util.go | 0 .../yurtstaticset/yurtstaticset_controller.go | 6 ++-- .../yurtstaticset_controller_test.go | 2 +- .../webhook/builder/defaulter_custom.go | 0 .../webhook/builder/validator_custom.go | 0 .../webhook/builder/webhook.go | 2 +- .../gateway/v1alpha1/gateway_default.go | 0 .../gateway/v1alpha1/gateway_handler.go | 2 +- .../gateway/v1alpha1/gateway_validation.go | 0 .../gateway/v1beta1/gateway_default.go | 0 .../gateway/v1beta1/gateway_handler.go | 2 +- .../gateway/v1beta1/gateway_validation.go | 0 .../webhook/node/v1/node_default.go | 0 .../webhook/node/v1/node_default_test.go | 0 .../webhook/node/v1/node_handler.go | 4 +-- .../webhook/node/v1/node_validation.go | 0 .../webhook/node/v1/node_validation_test.go | 0 .../nodepool/v1beta1/nodepool_default.go | 0 .../nodepool/v1beta1/nodepool_default_test.go | 0 .../nodepool/v1beta1/nodepool_handler.go | 2 +- .../nodepool/v1beta1/nodepool_validation.go | 5 +++ .../v1beta1/nodepool_validation_test.go | 0 .../v1alpha1/platformadmin_default.go | 0 .../v1alpha1/platformadmin_handler.go | 4 +-- .../v1alpha1/platformadmin_validation.go | 2 +- .../v1alpha2/platformadmin_default.go | 0 .../v1alpha2/platformadmin_handler.go | 4 +-- .../v1alpha2/platformadmin_validation.go | 2 +- .../webhook/pod/v1/pod_handler.go | 4 +-- .../webhook/pod/v1/pod_validation.go | 2 +- pkg/{ => yurtmanager}/webhook/server.go | 36 +++++++++---------- .../util/configuration/configuration.go | 2 +- .../util/controller/webhook_controller.go | 8 ++--- .../webhook/util/generator/certgenerator.go | 0 .../webhook/util/generator/selfsigned.go | 0 .../webhook/util/generator/util.go | 0 pkg/{ => yurtmanager}/webhook/util/util.go | 0 .../util/writer/atomic/atomic_writer.go | 0 .../webhook/util/writer/certwriter.go | 2 +- .../webhook/util/writer/error.go | 0 .../webhook/util/writer/fs.go | 4 +-- .../webhook/util/writer/secret.go | 2 +- .../v1alpha1/yurtappdaemon_default.go | 0 .../v1alpha1/yurtappdaemon_handler.go | 2 +- .../v1alpha1/yurtappdaemon_validation.go | 0 .../webhook/yurtappset/v1alpha1/validate.go | 0 .../yurtappset/v1alpha1/yurtappset_default.go | 0 .../yurtappset/v1alpha1/yurtappset_handler.go | 2 +- .../v1alpha1/yurtappset_validation.go | 0 .../v1alpha1/yurtappset_webhook_test.go | 0 .../v1alpha1/yurtstaticset_default.go | 0 .../v1alpha1/yurtstaticset_handler.go | 2 +- .../v1alpha1/yurtstaticset_validation.go | 0 pkg/yurttunnel/util/util.go | 2 +- 183 files changed, 149 insertions(+), 144 deletions(-) rename pkg/{ => util}/profile/profile.go (100%) rename pkg/{ => util}/profile/profile_test.go (100%) rename pkg/{ => yurtmanager}/controller/apis/config/types.go (81%) rename pkg/{ => yurtmanager}/controller/controller.go (71%) rename pkg/{ => yurtmanager}/controller/csrapprover/csrapprover_controller.go (99%) rename pkg/{ => yurtmanager}/controller/csrapprover/csrapprover_controller_test.go (100%) rename pkg/{ => yurtmanager}/controller/daemonpodupdater/daemon_pod_updater_controller.go (99%) rename pkg/{ => yurtmanager}/controller/daemonpodupdater/daemon_pod_updater_controller_test.go (99%) rename pkg/{ => yurtmanager}/controller/daemonpodupdater/kubernetes/controller_utils.go (100%) rename pkg/{ => yurtmanager}/controller/daemonpodupdater/kubernetes/controller_utils_test.go (100%) rename pkg/{ => yurtmanager}/controller/daemonpodupdater/kubernetes/pod_util.go (100%) rename pkg/{ => yurtmanager}/controller/daemonpodupdater/kubernetes/pod_util_test.go (100%) rename pkg/{ => yurtmanager}/controller/daemonpodupdater/util.go (96%) rename pkg/{ => yurtmanager}/controller/daemonpodupdater/util_test.go (100%) rename pkg/{ => yurtmanager}/controller/nodepool/config/types.go (100%) rename pkg/{ => yurtmanager}/controller/nodepool/nodepool_controller.go (98%) rename pkg/{ => yurtmanager}/controller/nodepool/nodepool_controller_test.go (98%) rename pkg/{ => yurtmanager}/controller/nodepool/nodepool_enqueue_handlers.go (100%) rename pkg/{ => yurtmanager}/controller/nodepool/nodepool_enqueue_handlers_test.go (100%) rename pkg/{ => yurtmanager}/controller/nodepool/util.go (98%) rename pkg/{ => yurtmanager}/controller/nodepool/util_test.go (100%) rename pkg/{ => yurtmanager}/controller/platformadmin/config/EdgeXConfig/config-nosecty.json (100%) rename pkg/{ => yurtmanager}/controller/platformadmin/config/EdgeXConfig/config.json (100%) rename pkg/{ => yurtmanager}/controller/platformadmin/config/EdgeXConfig/manifest.yaml (100%) rename pkg/{ => yurtmanager}/controller/platformadmin/config/types.go (100%) rename pkg/{ => yurtmanager}/controller/platformadmin/platformadmin_controller.go (99%) rename pkg/{ => yurtmanager}/controller/platformadmin/util.go (95%) rename pkg/{ => yurtmanager}/controller/platformadmin/utils/conditions.go (100%) rename pkg/{ => yurtmanager}/controller/platformadmin/utils/fieldindexer.go (100%) rename pkg/{ => yurtmanager}/controller/platformadmin/utils/version.go (100%) rename pkg/{ => yurtmanager}/controller/raven/common.go (100%) rename pkg/{ => yurtmanager}/controller/raven/gatewaypickup/config/types.go (100%) rename pkg/{ => yurtmanager}/controller/raven/gatewaypickup/gateway_pickup_controller.go (97%) rename pkg/{ => yurtmanager}/controller/raven/gatewaypickup/gateway_pickup_controller_test.go (98%) rename pkg/{ => yurtmanager}/controller/raven/gatewaypickup/gateway_pickup_enqueue_handlers.go (98%) rename pkg/{ => yurtmanager}/controller/raven/utils/utils.go (100%) rename pkg/{ => yurtmanager}/controller/servicetopology/adapter/adapter.go (100%) rename pkg/{ => yurtmanager}/controller/servicetopology/adapter/endpoints_adapter.go (100%) rename pkg/{ => yurtmanager}/controller/servicetopology/adapter/endpoints_adapter_test.go (100%) rename pkg/{ => yurtmanager}/controller/servicetopology/adapter/endpointslicev1_adapter.go (100%) rename pkg/{ => yurtmanager}/controller/servicetopology/adapter/endpointslicev1_adapter_test.go (100%) rename pkg/{ => yurtmanager}/controller/servicetopology/adapter/endpointslicev1beta1_adapter.go (100%) rename pkg/{ => yurtmanager}/controller/servicetopology/adapter/endpointslicev1beta1_adapter_test.go (100%) rename pkg/{ => yurtmanager}/controller/servicetopology/common.go (100%) rename pkg/{ => yurtmanager}/controller/servicetopology/endpoints/endpoints_controller.go (96%) rename pkg/{ => yurtmanager}/controller/servicetopology/endpoints/endpoints_enqueue_handlers.go (93%) rename pkg/{ => yurtmanager}/controller/servicetopology/endpointslice/endpointslice_controller.go (96%) rename pkg/{ => yurtmanager}/controller/servicetopology/endpointslice/endpointslice_enqueue_handlers.go (93%) rename pkg/{ => yurtmanager}/controller/servicetopology/util/util.go (100%) rename pkg/{ => yurtmanager}/controller/util/controller_utils.go (100%) rename pkg/{ => yurtmanager}/controller/util/controller_utils_test.go (100%) rename pkg/{ => yurtmanager}/controller/util/node/controller_utils.go (100%) rename pkg/{ => yurtmanager}/controller/util/refmanager/ref_manager.go (100%) rename pkg/{ => yurtmanager}/controller/util/refmanager/ref_manager_test.go (100%) rename pkg/{ => yurtmanager}/controller/util/tools.go (100%) rename pkg/{ => yurtmanager}/controller/util/tools_test.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappdaemon/config/types.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappdaemon/nodepool_enqueue_handlers.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappdaemon/nodepool_enqueue_handlers_test.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappdaemon/revision.go (99%) rename pkg/{ => yurtmanager}/controller/yurtappdaemon/revision_test.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappdaemon/util.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappdaemon/util_test.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappdaemon/workloadcontroller/controller.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappdaemon/workloadcontroller/deployment_controller.go (98%) rename pkg/{ => yurtmanager}/controller/yurtappdaemon/workloadcontroller/deployment_controller_test.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappdaemon/workloadcontroller/statefulset_controller.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappdaemon/workloadcontroller/util.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappdaemon/workloadcontroller/util_test.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappdaemon/workloadcontroller/workload.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappdaemon/workloadcontroller/workload_test.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappdaemon/yurtappdaemon_controller.go (99%) rename pkg/{ => yurtmanager}/controller/yurtappdaemon/yurtappdaemon_controller_test.go (99%) rename pkg/{ => yurtmanager}/controller/yurtappset/adapter/adapter.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappset/adapter/adapter_util.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappset/adapter/adapter_util_test.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappset/adapter/deployment_adapter.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappset/adapter/deployment_adapter_test.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappset/adapter/statefulset_adapter.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappset/adapter/statefulset_adapter_test.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappset/config/types.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappset/pool.go (96%) rename pkg/{ => yurtmanager}/controller/yurtappset/pool_control.go (97%) rename pkg/{ => yurtmanager}/controller/yurtappset/pool_controller_test.go (98%) rename pkg/{ => yurtmanager}/controller/yurtappset/revision.go (99%) rename pkg/{ => yurtmanager}/controller/yurtappset/revision_test.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappset/yurtappset_controller.go (99%) rename pkg/{ => yurtmanager}/controller/yurtappset/yurtappset_controller_statefulset_test.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappset/yurtappset_controller_suite_test.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappset/yurtappset_controller_test.go (97%) rename pkg/{ => yurtmanager}/controller/yurtappset/yurtappset_controller_utils.go (100%) rename pkg/{ => yurtmanager}/controller/yurtappset/yurtappset_update.go (99%) rename pkg/{ => yurtmanager}/controller/yurtcoordinator/cert/certificate.go (100%) rename pkg/{ => yurtmanager}/controller/yurtcoordinator/cert/certificate_test.go (100%) rename pkg/{ => yurtmanager}/controller/yurtcoordinator/cert/secret.go (100%) rename pkg/{ => yurtmanager}/controller/yurtcoordinator/cert/secret_test.go (100%) rename pkg/{ => yurtmanager}/controller/yurtcoordinator/cert/util.go (100%) rename pkg/{ => yurtmanager}/controller/yurtcoordinator/cert/util_test.go (100%) rename pkg/{ => yurtmanager}/controller/yurtcoordinator/cert/yurtcoordinatorcert_controller.go (100%) rename pkg/{ => yurtmanager}/controller/yurtcoordinator/cert/yurtcoordinatorcert_controller_test.go (100%) rename pkg/{ => yurtmanager}/controller/yurtcoordinator/constant/constant.go (100%) rename pkg/{ => yurtmanager}/controller/yurtcoordinator/delegatelease/delegatelease_controller.go (96%) rename pkg/{ => yurtmanager}/controller/yurtcoordinator/delegatelease/delegatelease_controller_test.go (93%) rename pkg/{ => yurtmanager}/controller/yurtcoordinator/podbinding/podbinding_controller.go (98%) rename pkg/{ => yurtmanager}/controller/yurtcoordinator/podbinding/podbinding_controller_test.go (100%) rename pkg/{ => yurtmanager}/controller/yurtcoordinator/utils/lease.go (94%) rename pkg/{ => yurtmanager}/controller/yurtcoordinator/utils/lease_test.go (100%) rename pkg/{ => yurtmanager}/controller/yurtcoordinator/utils/taints.go (100%) rename pkg/{ => yurtmanager}/controller/yurtcoordinator/utils/taints_test.go (93%) rename pkg/{ => yurtmanager}/controller/yurtstaticset/config/types.go (100%) rename pkg/{ => yurtmanager}/controller/yurtstaticset/upgradeinfo/upgrade_info.go (99%) rename pkg/{ => yurtmanager}/controller/yurtstaticset/upgradeinfo/upgrade_info_test.go (100%) rename pkg/{ => yurtmanager}/controller/yurtstaticset/util/util.go (100%) rename pkg/{ => yurtmanager}/controller/yurtstaticset/yurtstaticset_controller.go (98%) rename pkg/{ => yurtmanager}/controller/yurtstaticset/yurtstaticset_controller_test.go (98%) rename pkg/{ => yurtmanager}/webhook/builder/defaulter_custom.go (100%) rename pkg/{ => yurtmanager}/webhook/builder/validator_custom.go (100%) rename pkg/{ => yurtmanager}/webhook/builder/webhook.go (98%) rename pkg/{ => yurtmanager}/webhook/gateway/v1alpha1/gateway_default.go (100%) rename pkg/{ => yurtmanager}/webhook/gateway/v1alpha1/gateway_handler.go (96%) rename pkg/{ => yurtmanager}/webhook/gateway/v1alpha1/gateway_validation.go (100%) rename pkg/{ => yurtmanager}/webhook/gateway/v1beta1/gateway_default.go (100%) rename pkg/{ => yurtmanager}/webhook/gateway/v1beta1/gateway_handler.go (97%) rename pkg/{ => yurtmanager}/webhook/gateway/v1beta1/gateway_validation.go (100%) rename pkg/{ => yurtmanager}/webhook/node/v1/node_default.go (100%) rename pkg/{ => yurtmanager}/webhook/node/v1/node_default_test.go (100%) rename pkg/{ => yurtmanager}/webhook/node/v1/node_handler.go (93%) rename pkg/{ => yurtmanager}/webhook/node/v1/node_validation.go (100%) rename pkg/{ => yurtmanager}/webhook/node/v1/node_validation_test.go (100%) rename pkg/{ => yurtmanager}/webhook/nodepool/v1beta1/nodepool_default.go (100%) rename pkg/{ => yurtmanager}/webhook/nodepool/v1beta1/nodepool_default_test.go (100%) rename pkg/{ => yurtmanager}/webhook/nodepool/v1beta1/nodepool_handler.go (97%) rename pkg/{ => yurtmanager}/webhook/nodepool/v1beta1/nodepool_validation.go (95%) rename pkg/{ => yurtmanager}/webhook/nodepool/v1beta1/nodepool_validation_test.go (100%) rename pkg/{ => yurtmanager}/webhook/platformadmin/v1alpha1/platformadmin_default.go (100%) rename pkg/{ => yurtmanager}/webhook/platformadmin/v1alpha1/platformadmin_handler.go (94%) rename pkg/{ => yurtmanager}/webhook/platformadmin/v1alpha1/platformadmin_validation.go (98%) rename pkg/{ => yurtmanager}/webhook/platformadmin/v1alpha2/platformadmin_default.go (100%) rename pkg/{ => yurtmanager}/webhook/platformadmin/v1alpha2/platformadmin_handler.go (95%) rename pkg/{ => yurtmanager}/webhook/platformadmin/v1alpha2/platformadmin_validation.go (98%) rename pkg/{ => yurtmanager}/webhook/pod/v1/pod_handler.go (92%) rename pkg/{ => yurtmanager}/webhook/pod/v1/pod_validation.go (98%) rename pkg/{ => yurtmanager}/webhook/server.go (76%) rename pkg/{ => yurtmanager}/webhook/util/configuration/configuration.go (98%) rename pkg/{ => yurtmanager}/webhook/util/controller/webhook_controller.go (97%) rename pkg/{ => yurtmanager}/webhook/util/generator/certgenerator.go (100%) rename pkg/{ => yurtmanager}/webhook/util/generator/selfsigned.go (100%) rename pkg/{ => yurtmanager}/webhook/util/generator/util.go (100%) rename pkg/{ => yurtmanager}/webhook/util/util.go (100%) rename pkg/{ => yurtmanager}/webhook/util/writer/atomic/atomic_writer.go (100%) rename pkg/{ => yurtmanager}/webhook/util/writer/certwriter.go (97%) rename pkg/{ => yurtmanager}/webhook/util/writer/error.go (100%) rename pkg/{ => yurtmanager}/webhook/util/writer/fs.go (97%) rename pkg/{ => yurtmanager}/webhook/util/writer/secret.go (98%) rename pkg/{ => yurtmanager}/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_default.go (100%) rename pkg/{ => yurtmanager}/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_handler.go (97%) rename pkg/{ => yurtmanager}/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_validation.go (100%) rename pkg/{ => yurtmanager}/webhook/yurtappset/v1alpha1/validate.go (100%) rename pkg/{ => yurtmanager}/webhook/yurtappset/v1alpha1/yurtappset_default.go (100%) rename pkg/{ => yurtmanager}/webhook/yurtappset/v1alpha1/yurtappset_handler.go (97%) rename pkg/{ => yurtmanager}/webhook/yurtappset/v1alpha1/yurtappset_validation.go (100%) rename pkg/{ => yurtmanager}/webhook/yurtappset/v1alpha1/yurtappset_webhook_test.go (100%) rename pkg/{ => yurtmanager}/webhook/yurtstaticset/v1alpha1/yurtstaticset_default.go (100%) rename pkg/{ => yurtmanager}/webhook/yurtstaticset/v1alpha1/yurtstaticset_handler.go (97%) rename pkg/{ => yurtmanager}/webhook/yurtstaticset/v1alpha1/yurtstaticset_validation.go (100%) diff --git a/cmd/yurt-manager/app/config/config.go b/cmd/yurt-manager/app/config/config.go index ed471d0c08d..75f9424f50a 100644 --- a/cmd/yurt-manager/app/config/config.go +++ b/cmd/yurt-manager/app/config/config.go @@ -17,7 +17,7 @@ limitations under the License. package config import ( - yurtctrlmgrconfig "github.com/openyurtio/openyurt/pkg/controller/apis/config" + yurtctrlmgrconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/apis/config" ) // Config is the main context object for the controller manager. diff --git a/cmd/yurt-manager/app/manager.go b/cmd/yurt-manager/app/manager.go index f615dfd103a..05e2a2c40bd 100644 --- a/cmd/yurt-manager/app/manager.go +++ b/cmd/yurt-manager/app/manager.go @@ -37,11 +37,11 @@ import ( "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" "github.com/openyurtio/openyurt/cmd/yurt-manager/app/options" "github.com/openyurtio/openyurt/pkg/apis" - "github.com/openyurtio/openyurt/pkg/controller" - "github.com/openyurtio/openyurt/pkg/profile" "github.com/openyurtio/openyurt/pkg/projectinfo" - "github.com/openyurtio/openyurt/pkg/webhook" - "github.com/openyurtio/openyurt/pkg/webhook/util" + "github.com/openyurtio/openyurt/pkg/util/profile" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" ) var ( diff --git a/cmd/yurt-manager/app/options/gatewaycontroller.go b/cmd/yurt-manager/app/options/gatewaycontroller.go index e125dbbff1f..344f1874e69 100644 --- a/cmd/yurt-manager/app/options/gatewaycontroller.go +++ b/cmd/yurt-manager/app/options/gatewaycontroller.go @@ -19,7 +19,7 @@ package options import ( "github.com/spf13/pflag" - "github.com/openyurtio/openyurt/pkg/controller/raven/gatewaypickup/config" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/gatewaypickup/config" ) type GatewayPickupControllerOptions struct { diff --git a/cmd/yurt-manager/app/options/generic.go b/cmd/yurt-manager/app/options/generic.go index 247e7318d6d..919fd504d04 100644 --- a/cmd/yurt-manager/app/options/generic.go +++ b/cmd/yurt-manager/app/options/generic.go @@ -21,8 +21,8 @@ import ( "github.com/spf13/pflag" - "github.com/openyurtio/openyurt/pkg/controller/apis/config" "github.com/openyurtio/openyurt/pkg/features" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/apis/config" ) const enableAll = "*" diff --git a/cmd/yurt-manager/app/options/nodepoolcontroller.go b/cmd/yurt-manager/app/options/nodepoolcontroller.go index 8760760ef17..114a2b7c5e9 100644 --- a/cmd/yurt-manager/app/options/nodepoolcontroller.go +++ b/cmd/yurt-manager/app/options/nodepoolcontroller.go @@ -19,7 +19,7 @@ package options import ( "github.com/spf13/pflag" - "github.com/openyurtio/openyurt/pkg/controller/nodepool/config" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/nodepool/config" ) type NodePoolControllerOptions struct { diff --git a/cmd/yurt-manager/app/options/platformadmincontroller.go b/cmd/yurt-manager/app/options/platformadmincontroller.go index ce8aa99e682..969853905a6 100644 --- a/cmd/yurt-manager/app/options/platformadmincontroller.go +++ b/cmd/yurt-manager/app/options/platformadmincontroller.go @@ -21,7 +21,7 @@ import ( "github.com/spf13/pflag" - "github.com/openyurtio/openyurt/pkg/controller/platformadmin/config" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/platformadmin/config" ) type PlatformAdminControllerOptions struct { diff --git a/cmd/yurt-manager/app/options/yurtappdaemoncontroller.go b/cmd/yurt-manager/app/options/yurtappdaemoncontroller.go index 70c88313076..de105bfa8d6 100644 --- a/cmd/yurt-manager/app/options/yurtappdaemoncontroller.go +++ b/cmd/yurt-manager/app/options/yurtappdaemoncontroller.go @@ -19,7 +19,7 @@ package options import ( "github.com/spf13/pflag" - "github.com/openyurtio/openyurt/pkg/controller/yurtappdaemon/config" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappdaemon/config" ) type YurtAppDaemonControllerOptions struct { diff --git a/cmd/yurt-manager/app/options/yurtappsetcontroller.go b/cmd/yurt-manager/app/options/yurtappsetcontroller.go index 7ddca638cba..6f0d537b8e2 100644 --- a/cmd/yurt-manager/app/options/yurtappsetcontroller.go +++ b/cmd/yurt-manager/app/options/yurtappsetcontroller.go @@ -19,7 +19,7 @@ package options import ( "github.com/spf13/pflag" - "github.com/openyurtio/openyurt/pkg/controller/yurtappset/config" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/config" ) type YurtAppSetControllerOptions struct { diff --git a/cmd/yurt-manager/app/options/yurtstaticsetcontroller.go b/cmd/yurt-manager/app/options/yurtstaticsetcontroller.go index e26da86236c..fcb27e236d8 100644 --- a/cmd/yurt-manager/app/options/yurtstaticsetcontroller.go +++ b/cmd/yurt-manager/app/options/yurtstaticsetcontroller.go @@ -19,7 +19,7 @@ package options import ( "github.com/spf13/pflag" - "github.com/openyurtio/openyurt/pkg/controller/yurtstaticset/config" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset/config" ) const DefaultUpgradeWorkerImage = "openyurt/node-servant:latest" diff --git a/pkg/profile/profile.go b/pkg/util/profile/profile.go similarity index 100% rename from pkg/profile/profile.go rename to pkg/util/profile/profile.go diff --git a/pkg/profile/profile_test.go b/pkg/util/profile/profile_test.go similarity index 100% rename from pkg/profile/profile_test.go rename to pkg/util/profile/profile_test.go diff --git a/pkg/yurtadm/cmd/join/join.go b/pkg/yurtadm/cmd/join/join.go index 1406c75b88d..9e554abb945 100644 --- a/pkg/yurtadm/cmd/join/join.go +++ b/pkg/yurtadm/cmd/join/join.go @@ -31,7 +31,6 @@ import ( "k8s.io/klog/v2" "github.com/openyurtio/openyurt/pkg/apis/apps" - "github.com/openyurtio/openyurt/pkg/controller/yurtstaticset/util" kubeconfigutil "github.com/openyurtio/openyurt/pkg/util/kubeconfig" "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/util/apiclient" "github.com/openyurtio/openyurt/pkg/yurtadm/cmd/join/joindata" @@ -40,6 +39,7 @@ import ( "github.com/openyurtio/openyurt/pkg/yurtadm/util/edgenode" yurtadmutil "github.com/openyurtio/openyurt/pkg/yurtadm/util/kubernetes" "github.com/openyurtio/openyurt/pkg/yurtadm/util/yurthub" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset/util" ) type joinOptions struct { diff --git a/pkg/yurtadm/cmd/staticpods/install/install.go b/pkg/yurtadm/cmd/staticpods/install/install.go index a58606dcd59..c3fc38e38fa 100644 --- a/pkg/yurtadm/cmd/staticpods/install/install.go +++ b/pkg/yurtadm/cmd/staticpods/install/install.go @@ -25,11 +25,11 @@ import ( flag "github.com/spf13/pflag" "k8s.io/klog/v2" - "github.com/openyurtio/openyurt/pkg/controller/yurtstaticset/util" "github.com/openyurtio/openyurt/pkg/yurtadm/constants" "github.com/openyurtio/openyurt/pkg/yurtadm/util/edgenode" yurtadmutil "github.com/openyurtio/openyurt/pkg/yurtadm/util/kubernetes" "github.com/openyurtio/openyurt/pkg/yurtadm/util/yurthub" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset/util" ) type installOptions struct { diff --git a/pkg/yurtadm/cmd/staticpods/uninstall/uninstall.go b/pkg/yurtadm/cmd/staticpods/uninstall/uninstall.go index 97a142b2c35..6628e2f3eda 100644 --- a/pkg/yurtadm/cmd/staticpods/uninstall/uninstall.go +++ b/pkg/yurtadm/cmd/staticpods/uninstall/uninstall.go @@ -25,11 +25,11 @@ import ( flag "github.com/spf13/pflag" "k8s.io/klog/v2" - "github.com/openyurtio/openyurt/pkg/controller/yurtstaticset/util" "github.com/openyurtio/openyurt/pkg/yurtadm/constants" "github.com/openyurtio/openyurt/pkg/yurtadm/util/edgenode" yurtadmutil "github.com/openyurtio/openyurt/pkg/yurtadm/util/kubernetes" "github.com/openyurtio/openyurt/pkg/yurtadm/util/yurthub" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset/util" ) type uninstallOptions struct { diff --git a/pkg/yurthub/otaupdate/ota.go b/pkg/yurthub/otaupdate/ota.go index 4b44e10d8b2..48cdcacfc9e 100644 --- a/pkg/yurthub/otaupdate/ota.go +++ b/pkg/yurthub/otaupdate/ota.go @@ -28,12 +28,12 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" - "github.com/openyurtio/openyurt/pkg/controller/daemonpodupdater" "github.com/openyurtio/openyurt/pkg/yurthub/cachemanager" "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/rest" upgrade "github.com/openyurtio/openyurt/pkg/yurthub/otaupdate/upgrader" "github.com/openyurtio/openyurt/pkg/yurthub/otaupdate/util" "github.com/openyurtio/openyurt/pkg/yurthub/storage" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/daemonpodupdater" ) const ( diff --git a/pkg/yurthub/otaupdate/upgrader/static_pod.go b/pkg/yurthub/otaupdate/upgrader/static_pod.go index bc4c3697cfc..1476976e0de 100644 --- a/pkg/yurthub/otaupdate/upgrader/static_pod.go +++ b/pkg/yurthub/otaupdate/upgrader/static_pod.go @@ -29,10 +29,10 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" - spctrlutil "github.com/openyurtio/openyurt/pkg/controller/yurtstaticset/util" upgrade "github.com/openyurtio/openyurt/pkg/node-servant/static-pod-upgrade" upgradeutil "github.com/openyurtio/openyurt/pkg/node-servant/static-pod-upgrade/util" "github.com/openyurtio/openyurt/pkg/yurthub/otaupdate/util" + spctrlutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset/util" ) const OTA = "OTA" diff --git a/pkg/yurthub/otaupdate/upgrader/static_pod_test.go b/pkg/yurthub/otaupdate/upgrader/static_pod_test.go index 9f05aba4267..33a01933597 100644 --- a/pkg/yurthub/otaupdate/upgrader/static_pod_test.go +++ b/pkg/yurthub/otaupdate/upgrader/static_pod_test.go @@ -27,10 +27,10 @@ import ( "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/kubernetes/fake" - spctrlutil "github.com/openyurtio/openyurt/pkg/controller/yurtstaticset/util" upgrade "github.com/openyurtio/openyurt/pkg/node-servant/static-pod-upgrade" upgradeutil "github.com/openyurtio/openyurt/pkg/node-servant/static-pod-upgrade/util" "github.com/openyurtio/openyurt/pkg/yurthub/otaupdate/util" + spctrlutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset/util" ) func TestStaticPodUpgrader_ApplyManifestNotExist(t *testing.T) { diff --git a/pkg/yurthub/otaupdate/util/util.go b/pkg/yurthub/otaupdate/util/util.go index c9481013819..fd8c16b4a93 100644 --- a/pkg/yurthub/otaupdate/util/util.go +++ b/pkg/yurthub/otaupdate/util/util.go @@ -27,8 +27,8 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/klog/v2" - "github.com/openyurtio/openyurt/pkg/controller/daemonpodupdater" yurtutil "github.com/openyurtio/openyurt/pkg/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/daemonpodupdater" ) // Derived from kubelet encodePods diff --git a/pkg/yurthub/server/server.go b/pkg/yurthub/server/server.go index 4b7189ac4b6..1dff97ddf14 100644 --- a/pkg/yurthub/server/server.go +++ b/pkg/yurthub/server/server.go @@ -28,7 +28,7 @@ import ( "k8s.io/klog/v2" "github.com/openyurtio/openyurt/cmd/yurthub/app/config" - "github.com/openyurtio/openyurt/pkg/profile" + "github.com/openyurtio/openyurt/pkg/util/profile" "github.com/openyurtio/openyurt/pkg/yurthub/certificate" "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/rest" ota "github.com/openyurtio/openyurt/pkg/yurthub/otaupdate" diff --git a/pkg/controller/apis/config/types.go b/pkg/yurtmanager/controller/apis/config/types.go similarity index 81% rename from pkg/controller/apis/config/types.go rename to pkg/yurtmanager/controller/apis/config/types.go index ef74904330c..3be8f661277 100644 --- a/pkg/controller/apis/config/types.go +++ b/pkg/yurtmanager/controller/apis/config/types.go @@ -19,12 +19,12 @@ package config import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - nodepoolconfig "github.com/openyurtio/openyurt/pkg/controller/nodepool/config" - platformadminconfig "github.com/openyurtio/openyurt/pkg/controller/platformadmin/config" - gatewaypickupconfig "github.com/openyurtio/openyurt/pkg/controller/raven/gatewaypickup/config" - yurtappdaemonconfig "github.com/openyurtio/openyurt/pkg/controller/yurtappdaemon/config" - yurtappsetconfig "github.com/openyurtio/openyurt/pkg/controller/yurtappset/config" - yurtstaticsetconfig "github.com/openyurtio/openyurt/pkg/controller/yurtstaticset/config" + nodepoolconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/nodepool/config" + platformadminconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/platformadmin/config" + gatewaypickupconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/gatewaypickup/config" + yurtappdaemonconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappdaemon/config" + yurtappsetconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/config" + yurtstaticsetconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset/config" ) // YurtManagerConfiguration contains elements describing yurt-manager. diff --git a/pkg/controller/controller.go b/pkg/yurtmanager/controller/controller.go similarity index 71% rename from pkg/controller/controller.go rename to pkg/yurtmanager/controller/controller.go index b9a81666c17..2df60ea13c1 100644 --- a/pkg/controller/controller.go +++ b/pkg/yurtmanager/controller/controller.go @@ -22,22 +22,22 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" - "github.com/openyurtio/openyurt/pkg/controller/csrapprover" - "github.com/openyurtio/openyurt/pkg/controller/daemonpodupdater" - "github.com/openyurtio/openyurt/pkg/controller/nodepool" - "github.com/openyurtio/openyurt/pkg/controller/platformadmin" - "github.com/openyurtio/openyurt/pkg/controller/raven" - "github.com/openyurtio/openyurt/pkg/controller/raven/gatewaypickup" - "github.com/openyurtio/openyurt/pkg/controller/servicetopology" - servicetopologyendpoints "github.com/openyurtio/openyurt/pkg/controller/servicetopology/endpoints" - servicetopologyendpointslice "github.com/openyurtio/openyurt/pkg/controller/servicetopology/endpointslice" - "github.com/openyurtio/openyurt/pkg/controller/util" - "github.com/openyurtio/openyurt/pkg/controller/yurtappdaemon" - "github.com/openyurtio/openyurt/pkg/controller/yurtappset" - yurtcoordinatorcert "github.com/openyurtio/openyurt/pkg/controller/yurtcoordinator/cert" - "github.com/openyurtio/openyurt/pkg/controller/yurtcoordinator/delegatelease" - "github.com/openyurtio/openyurt/pkg/controller/yurtcoordinator/podbinding" - "github.com/openyurtio/openyurt/pkg/controller/yurtstaticset" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/csrapprover" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/daemonpodupdater" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/nodepool" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/platformadmin" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/gatewaypickup" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology" + servicetopologyendpoints "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology/endpoints" + servicetopologyendpointslice "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology/endpointslice" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappdaemon" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset" + yurtcoordinatorcert "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtcoordinator/cert" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtcoordinator/delegatelease" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtcoordinator/podbinding" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset" ) // Note !!! @kadisi diff --git a/pkg/controller/csrapprover/csrapprover_controller.go b/pkg/yurtmanager/controller/csrapprover/csrapprover_controller.go similarity index 99% rename from pkg/controller/csrapprover/csrapprover_controller.go rename to pkg/yurtmanager/controller/csrapprover/csrapprover_controller.go index ae4a65497a8..0316fb3d66a 100644 --- a/pkg/controller/csrapprover/csrapprover_controller.go +++ b/pkg/yurtmanager/controller/csrapprover/csrapprover_controller.go @@ -43,9 +43,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" - yurtcoorrdinatorCert "github.com/openyurtio/openyurt/pkg/controller/yurtcoordinator/cert" "github.com/openyurtio/openyurt/pkg/projectinfo" "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token" + yurtcoorrdinatorCert "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtcoordinator/cert" "github.com/openyurtio/openyurt/pkg/yurttunnel/constants" ) diff --git a/pkg/controller/csrapprover/csrapprover_controller_test.go b/pkg/yurtmanager/controller/csrapprover/csrapprover_controller_test.go similarity index 100% rename from pkg/controller/csrapprover/csrapprover_controller_test.go rename to pkg/yurtmanager/controller/csrapprover/csrapprover_controller_test.go diff --git a/pkg/controller/daemonpodupdater/daemon_pod_updater_controller.go b/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller.go similarity index 99% rename from pkg/controller/daemonpodupdater/daemon_pod_updater_controller.go rename to pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller.go index 95e27645a13..726acefc586 100644 --- a/pkg/controller/daemonpodupdater/daemon_pod_updater_controller.go +++ b/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller.go @@ -49,7 +49,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" - k8sutil "github.com/openyurtio/openyurt/pkg/controller/daemonpodupdater/kubernetes" + k8sutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/daemonpodupdater/kubernetes" ) func init() { diff --git a/pkg/controller/daemonpodupdater/daemon_pod_updater_controller_test.go b/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller_test.go similarity index 99% rename from pkg/controller/daemonpodupdater/daemon_pod_updater_controller_test.go rename to pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller_test.go index 187caa2c377..744fd618a0d 100644 --- a/pkg/controller/daemonpodupdater/daemon_pod_updater_controller_test.go +++ b/pkg/yurtmanager/controller/daemonpodupdater/daemon_pod_updater_controller_test.go @@ -34,7 +34,7 @@ import ( fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" - k8sutil "github.com/openyurtio/openyurt/pkg/controller/daemonpodupdater/kubernetes" + k8sutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/daemonpodupdater/kubernetes" ) const ( diff --git a/pkg/controller/daemonpodupdater/kubernetes/controller_utils.go b/pkg/yurtmanager/controller/daemonpodupdater/kubernetes/controller_utils.go similarity index 100% rename from pkg/controller/daemonpodupdater/kubernetes/controller_utils.go rename to pkg/yurtmanager/controller/daemonpodupdater/kubernetes/controller_utils.go diff --git a/pkg/controller/daemonpodupdater/kubernetes/controller_utils_test.go b/pkg/yurtmanager/controller/daemonpodupdater/kubernetes/controller_utils_test.go similarity index 100% rename from pkg/controller/daemonpodupdater/kubernetes/controller_utils_test.go rename to pkg/yurtmanager/controller/daemonpodupdater/kubernetes/controller_utils_test.go diff --git a/pkg/controller/daemonpodupdater/kubernetes/pod_util.go b/pkg/yurtmanager/controller/daemonpodupdater/kubernetes/pod_util.go similarity index 100% rename from pkg/controller/daemonpodupdater/kubernetes/pod_util.go rename to pkg/yurtmanager/controller/daemonpodupdater/kubernetes/pod_util.go diff --git a/pkg/controller/daemonpodupdater/kubernetes/pod_util_test.go b/pkg/yurtmanager/controller/daemonpodupdater/kubernetes/pod_util_test.go similarity index 100% rename from pkg/controller/daemonpodupdater/kubernetes/pod_util_test.go rename to pkg/yurtmanager/controller/daemonpodupdater/kubernetes/pod_util_test.go diff --git a/pkg/controller/daemonpodupdater/util.go b/pkg/yurtmanager/controller/daemonpodupdater/util.go similarity index 96% rename from pkg/controller/daemonpodupdater/util.go rename to pkg/yurtmanager/controller/daemonpodupdater/util.go index 01a42536977..be7f5fe41e3 100644 --- a/pkg/controller/daemonpodupdater/util.go +++ b/pkg/yurtmanager/controller/daemonpodupdater/util.go @@ -29,8 +29,8 @@ import ( "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" - k8sutil "github.com/openyurtio/openyurt/pkg/controller/daemonpodupdater/kubernetes" - util "github.com/openyurtio/openyurt/pkg/controller/util/node" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/daemonpodupdater/kubernetes" + util "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/node" ) // GetDaemonsetPods get all pods belong to the given daemonset @@ -62,7 +62,7 @@ func GetDaemonsetPods(c client.Client, ds *appsv1.DaemonSet) ([]*corev1.Pod, err // IsDaemonsetPodLatest check whether pod is the latest by comparing its Spec with daemonset's // If pod is latest, return true, otherwise return false func IsDaemonsetPodLatest(ds *appsv1.DaemonSet, pod *corev1.Pod) bool { - hash := k8sutil.ComputeHash(&ds.Spec.Template, ds.Status.CollisionCount) + hash := kubernetes.ComputeHash(&ds.Spec.Template, ds.Status.CollisionCount) klog.V(4).Infof("compute hash: %v", hash) generation, err := GetTemplateGeneration(ds) if err != nil { @@ -243,6 +243,6 @@ func IsPodUpgradeConditionTrue(status corev1.PodStatus) bool { // GetPodUpgradeCondition extracts the pod upgrade condition from the given status and returns that. // Returns nil if the condition is not present. func GetPodUpgradeCondition(status corev1.PodStatus) *corev1.PodCondition { - _, condition := k8sutil.GetPodCondition(&status, PodNeedUpgrade) + _, condition := kubernetes.GetPodCondition(&status, PodNeedUpgrade) return condition } diff --git a/pkg/controller/daemonpodupdater/util_test.go b/pkg/yurtmanager/controller/daemonpodupdater/util_test.go similarity index 100% rename from pkg/controller/daemonpodupdater/util_test.go rename to pkg/yurtmanager/controller/daemonpodupdater/util_test.go diff --git a/pkg/controller/nodepool/config/types.go b/pkg/yurtmanager/controller/nodepool/config/types.go similarity index 100% rename from pkg/controller/nodepool/config/types.go rename to pkg/yurtmanager/controller/nodepool/config/types.go diff --git a/pkg/controller/nodepool/nodepool_controller.go b/pkg/yurtmanager/controller/nodepool/nodepool_controller.go similarity index 98% rename from pkg/controller/nodepool/nodepool_controller.go rename to pkg/yurtmanager/controller/nodepool/nodepool_controller.go index 2ca6d37ea35..e7372938c10 100644 --- a/pkg/controller/nodepool/nodepool_controller.go +++ b/pkg/yurtmanager/controller/nodepool/nodepool_controller.go @@ -35,7 +35,7 @@ import ( "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" "github.com/openyurtio/openyurt/pkg/apis/apps" appsv1beta1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" - poolconfig "github.com/openyurtio/openyurt/pkg/controller/nodepool/config" + poolconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/nodepool/config" ) var ( diff --git a/pkg/controller/nodepool/nodepool_controller_test.go b/pkg/yurtmanager/controller/nodepool/nodepool_controller_test.go similarity index 98% rename from pkg/controller/nodepool/nodepool_controller_test.go rename to pkg/yurtmanager/controller/nodepool/nodepool_controller_test.go index aa0d0c687fb..dfff1242bee 100644 --- a/pkg/controller/nodepool/nodepool_controller_test.go +++ b/pkg/yurtmanager/controller/nodepool/nodepool_controller_test.go @@ -33,7 +33,7 @@ import ( "github.com/openyurtio/openyurt/pkg/apis" "github.com/openyurtio/openyurt/pkg/apis/apps" appsv1beta1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" - poolconfig "github.com/openyurtio/openyurt/pkg/controller/nodepool/config" + poolconfig "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/nodepool/config" ) func prepareNodes() []client.Object { diff --git a/pkg/controller/nodepool/nodepool_enqueue_handlers.go b/pkg/yurtmanager/controller/nodepool/nodepool_enqueue_handlers.go similarity index 100% rename from pkg/controller/nodepool/nodepool_enqueue_handlers.go rename to pkg/yurtmanager/controller/nodepool/nodepool_enqueue_handlers.go diff --git a/pkg/controller/nodepool/nodepool_enqueue_handlers_test.go b/pkg/yurtmanager/controller/nodepool/nodepool_enqueue_handlers_test.go similarity index 100% rename from pkg/controller/nodepool/nodepool_enqueue_handlers_test.go rename to pkg/yurtmanager/controller/nodepool/nodepool_enqueue_handlers_test.go diff --git a/pkg/controller/nodepool/util.go b/pkg/yurtmanager/controller/nodepool/util.go similarity index 98% rename from pkg/controller/nodepool/util.go rename to pkg/yurtmanager/controller/nodepool/util.go index dd816cde1d8..c502723d530 100644 --- a/pkg/controller/nodepool/util.go +++ b/pkg/yurtmanager/controller/nodepool/util.go @@ -25,7 +25,7 @@ import ( "github.com/openyurtio/openyurt/pkg/apis/apps" appsv1beta1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" - nodeutil "github.com/openyurtio/openyurt/pkg/controller/util/node" + nodeutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/node" ) // conciliatePoolRelatedAttrs will update the node's attributes that related to diff --git a/pkg/controller/nodepool/util_test.go b/pkg/yurtmanager/controller/nodepool/util_test.go similarity index 100% rename from pkg/controller/nodepool/util_test.go rename to pkg/yurtmanager/controller/nodepool/util_test.go diff --git a/pkg/controller/platformadmin/config/EdgeXConfig/config-nosecty.json b/pkg/yurtmanager/controller/platformadmin/config/EdgeXConfig/config-nosecty.json similarity index 100% rename from pkg/controller/platformadmin/config/EdgeXConfig/config-nosecty.json rename to pkg/yurtmanager/controller/platformadmin/config/EdgeXConfig/config-nosecty.json diff --git a/pkg/controller/platformadmin/config/EdgeXConfig/config.json b/pkg/yurtmanager/controller/platformadmin/config/EdgeXConfig/config.json similarity index 100% rename from pkg/controller/platformadmin/config/EdgeXConfig/config.json rename to pkg/yurtmanager/controller/platformadmin/config/EdgeXConfig/config.json diff --git a/pkg/controller/platformadmin/config/EdgeXConfig/manifest.yaml b/pkg/yurtmanager/controller/platformadmin/config/EdgeXConfig/manifest.yaml similarity index 100% rename from pkg/controller/platformadmin/config/EdgeXConfig/manifest.yaml rename to pkg/yurtmanager/controller/platformadmin/config/EdgeXConfig/manifest.yaml diff --git a/pkg/controller/platformadmin/config/types.go b/pkg/yurtmanager/controller/platformadmin/config/types.go similarity index 100% rename from pkg/controller/platformadmin/config/types.go rename to pkg/yurtmanager/controller/platformadmin/config/types.go diff --git a/pkg/controller/platformadmin/platformadmin_controller.go b/pkg/yurtmanager/controller/platformadmin/platformadmin_controller.go similarity index 99% rename from pkg/controller/platformadmin/platformadmin_controller.go rename to pkg/yurtmanager/controller/platformadmin/platformadmin_controller.go index 1edc362cfa1..2ee00160579 100644 --- a/pkg/controller/platformadmin/platformadmin_controller.go +++ b/pkg/yurtmanager/controller/platformadmin/platformadmin_controller.go @@ -47,8 +47,8 @@ import ( appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" iotv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" iotv1alpha2 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha2" - "github.com/openyurtio/openyurt/pkg/controller/platformadmin/config" - util "github.com/openyurtio/openyurt/pkg/controller/platformadmin/utils" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/platformadmin/config" + util "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/platformadmin/utils" ) func init() { diff --git a/pkg/controller/platformadmin/util.go b/pkg/yurtmanager/controller/platformadmin/util.go similarity index 95% rename from pkg/controller/platformadmin/util.go rename to pkg/yurtmanager/controller/platformadmin/util.go index 6abc2b5e780..9975c683083 100644 --- a/pkg/controller/platformadmin/util.go +++ b/pkg/yurtmanager/controller/platformadmin/util.go @@ -27,8 +27,8 @@ import ( "k8s.io/utils/pointer" iotv1alpha2 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha2" - "github.com/openyurtio/openyurt/pkg/controller/platformadmin/config" - utils "github.com/openyurtio/openyurt/pkg/controller/platformadmin/utils" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/platformadmin/config" + utils "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/platformadmin/utils" ) // NewYurtIoTDockComponent initialize the configuration of yurt-iot-dock component diff --git a/pkg/controller/platformadmin/utils/conditions.go b/pkg/yurtmanager/controller/platformadmin/utils/conditions.go similarity index 100% rename from pkg/controller/platformadmin/utils/conditions.go rename to pkg/yurtmanager/controller/platformadmin/utils/conditions.go diff --git a/pkg/controller/platformadmin/utils/fieldindexer.go b/pkg/yurtmanager/controller/platformadmin/utils/fieldindexer.go similarity index 100% rename from pkg/controller/platformadmin/utils/fieldindexer.go rename to pkg/yurtmanager/controller/platformadmin/utils/fieldindexer.go diff --git a/pkg/controller/platformadmin/utils/version.go b/pkg/yurtmanager/controller/platformadmin/utils/version.go similarity index 100% rename from pkg/controller/platformadmin/utils/version.go rename to pkg/yurtmanager/controller/platformadmin/utils/version.go diff --git a/pkg/controller/raven/common.go b/pkg/yurtmanager/controller/raven/common.go similarity index 100% rename from pkg/controller/raven/common.go rename to pkg/yurtmanager/controller/raven/common.go diff --git a/pkg/controller/raven/gatewaypickup/config/types.go b/pkg/yurtmanager/controller/raven/gatewaypickup/config/types.go similarity index 100% rename from pkg/controller/raven/gatewaypickup/config/types.go rename to pkg/yurtmanager/controller/raven/gatewaypickup/config/types.go diff --git a/pkg/controller/raven/gatewaypickup/gateway_pickup_controller.go b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go similarity index 97% rename from pkg/controller/raven/gatewaypickup/gateway_pickup_controller.go rename to pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go index 24d8337a21a..2449cd775a3 100644 --- a/pkg/controller/raven/gatewaypickup/gateway_pickup_controller.go +++ b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller.go @@ -42,10 +42,10 @@ import ( appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" "github.com/openyurtio/openyurt/pkg/apis/raven" ravenv1beta1 "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" - common "github.com/openyurtio/openyurt/pkg/controller/raven" - "github.com/openyurtio/openyurt/pkg/controller/raven/gatewaypickup/config" - "github.com/openyurtio/openyurt/pkg/controller/raven/utils" - nodeutil "github.com/openyurtio/openyurt/pkg/controller/util/node" + common "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/gatewaypickup/config" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/utils" + nodeutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/node" ) var ( diff --git a/pkg/controller/raven/gatewaypickup/gateway_pickup_controller_test.go b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller_test.go similarity index 98% rename from pkg/controller/raven/gatewaypickup/gateway_pickup_controller_test.go rename to pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller_test.go index c6586a2cc17..585d6f2fc23 100644 --- a/pkg/controller/raven/gatewaypickup/gateway_pickup_controller_test.go +++ b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_controller_test.go @@ -26,8 +26,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" ravenv1beta1 "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" - "github.com/openyurtio/openyurt/pkg/controller/raven/gatewaypickup/config" - "github.com/openyurtio/openyurt/pkg/controller/raven/utils" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/gatewaypickup/config" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/utils" ) var ( diff --git a/pkg/controller/raven/gatewaypickup/gateway_pickup_enqueue_handlers.go b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_enqueue_handlers.go similarity index 98% rename from pkg/controller/raven/gatewaypickup/gateway_pickup_enqueue_handlers.go rename to pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_enqueue_handlers.go index 26c57e14847..4d628735257 100644 --- a/pkg/controller/raven/gatewaypickup/gateway_pickup_enqueue_handlers.go +++ b/pkg/yurtmanager/controller/raven/gatewaypickup/gateway_pickup_enqueue_handlers.go @@ -27,7 +27,7 @@ import ( "github.com/openyurtio/openyurt/pkg/apis/raven" ravenv1beta1 "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" - "github.com/openyurtio/openyurt/pkg/controller/raven/utils" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven/utils" ) type EnqueueGatewayForNode struct{} diff --git a/pkg/controller/raven/utils/utils.go b/pkg/yurtmanager/controller/raven/utils/utils.go similarity index 100% rename from pkg/controller/raven/utils/utils.go rename to pkg/yurtmanager/controller/raven/utils/utils.go diff --git a/pkg/controller/servicetopology/adapter/adapter.go b/pkg/yurtmanager/controller/servicetopology/adapter/adapter.go similarity index 100% rename from pkg/controller/servicetopology/adapter/adapter.go rename to pkg/yurtmanager/controller/servicetopology/adapter/adapter.go diff --git a/pkg/controller/servicetopology/adapter/endpoints_adapter.go b/pkg/yurtmanager/controller/servicetopology/adapter/endpoints_adapter.go similarity index 100% rename from pkg/controller/servicetopology/adapter/endpoints_adapter.go rename to pkg/yurtmanager/controller/servicetopology/adapter/endpoints_adapter.go diff --git a/pkg/controller/servicetopology/adapter/endpoints_adapter_test.go b/pkg/yurtmanager/controller/servicetopology/adapter/endpoints_adapter_test.go similarity index 100% rename from pkg/controller/servicetopology/adapter/endpoints_adapter_test.go rename to pkg/yurtmanager/controller/servicetopology/adapter/endpoints_adapter_test.go diff --git a/pkg/controller/servicetopology/adapter/endpointslicev1_adapter.go b/pkg/yurtmanager/controller/servicetopology/adapter/endpointslicev1_adapter.go similarity index 100% rename from pkg/controller/servicetopology/adapter/endpointslicev1_adapter.go rename to pkg/yurtmanager/controller/servicetopology/adapter/endpointslicev1_adapter.go diff --git a/pkg/controller/servicetopology/adapter/endpointslicev1_adapter_test.go b/pkg/yurtmanager/controller/servicetopology/adapter/endpointslicev1_adapter_test.go similarity index 100% rename from pkg/controller/servicetopology/adapter/endpointslicev1_adapter_test.go rename to pkg/yurtmanager/controller/servicetopology/adapter/endpointslicev1_adapter_test.go diff --git a/pkg/controller/servicetopology/adapter/endpointslicev1beta1_adapter.go b/pkg/yurtmanager/controller/servicetopology/adapter/endpointslicev1beta1_adapter.go similarity index 100% rename from pkg/controller/servicetopology/adapter/endpointslicev1beta1_adapter.go rename to pkg/yurtmanager/controller/servicetopology/adapter/endpointslicev1beta1_adapter.go diff --git a/pkg/controller/servicetopology/adapter/endpointslicev1beta1_adapter_test.go b/pkg/yurtmanager/controller/servicetopology/adapter/endpointslicev1beta1_adapter_test.go similarity index 100% rename from pkg/controller/servicetopology/adapter/endpointslicev1beta1_adapter_test.go rename to pkg/yurtmanager/controller/servicetopology/adapter/endpointslicev1beta1_adapter_test.go diff --git a/pkg/controller/servicetopology/common.go b/pkg/yurtmanager/controller/servicetopology/common.go similarity index 100% rename from pkg/controller/servicetopology/common.go rename to pkg/yurtmanager/controller/servicetopology/common.go diff --git a/pkg/controller/servicetopology/endpoints/endpoints_controller.go b/pkg/yurtmanager/controller/servicetopology/endpoints/endpoints_controller.go similarity index 96% rename from pkg/controller/servicetopology/endpoints/endpoints_controller.go rename to pkg/yurtmanager/controller/servicetopology/endpoints/endpoints_controller.go index e35426e662f..39adacb3088 100644 --- a/pkg/controller/servicetopology/endpoints/endpoints_controller.go +++ b/pkg/yurtmanager/controller/servicetopology/endpoints/endpoints_controller.go @@ -31,8 +31,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" - common "github.com/openyurtio/openyurt/pkg/controller/servicetopology" - "github.com/openyurtio/openyurt/pkg/controller/servicetopology/adapter" + common "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology/adapter" ) func init() { diff --git a/pkg/controller/servicetopology/endpoints/endpoints_enqueue_handlers.go b/pkg/yurtmanager/controller/servicetopology/endpoints/endpoints_enqueue_handlers.go similarity index 93% rename from pkg/controller/servicetopology/endpoints/endpoints_enqueue_handlers.go rename to pkg/yurtmanager/controller/servicetopology/endpoints/endpoints_enqueue_handlers.go index 2ac0c355fb6..febf3d35a51 100644 --- a/pkg/controller/servicetopology/endpoints/endpoints_enqueue_handlers.go +++ b/pkg/yurtmanager/controller/servicetopology/endpoints/endpoints_enqueue_handlers.go @@ -25,8 +25,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/openyurtio/openyurt/pkg/controller/servicetopology/adapter" - "github.com/openyurtio/openyurt/pkg/controller/servicetopology/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology/adapter" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology/util" ) type EnqueueEndpointsForService struct { diff --git a/pkg/controller/servicetopology/endpointslice/endpointslice_controller.go b/pkg/yurtmanager/controller/servicetopology/endpointslice/endpointslice_controller.go similarity index 96% rename from pkg/controller/servicetopology/endpointslice/endpointslice_controller.go rename to pkg/yurtmanager/controller/servicetopology/endpointslice/endpointslice_controller.go index 335b2ccbbd8..8a26925f55f 100644 --- a/pkg/controller/servicetopology/endpointslice/endpointslice_controller.go +++ b/pkg/yurtmanager/controller/servicetopology/endpointslice/endpointslice_controller.go @@ -34,8 +34,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" - common "github.com/openyurtio/openyurt/pkg/controller/servicetopology" - "github.com/openyurtio/openyurt/pkg/controller/servicetopology/adapter" + common "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology/adapter" ) func init() { diff --git a/pkg/controller/servicetopology/endpointslice/endpointslice_enqueue_handlers.go b/pkg/yurtmanager/controller/servicetopology/endpointslice/endpointslice_enqueue_handlers.go similarity index 93% rename from pkg/controller/servicetopology/endpointslice/endpointslice_enqueue_handlers.go rename to pkg/yurtmanager/controller/servicetopology/endpointslice/endpointslice_enqueue_handlers.go index 1d39a4c5f56..234af1bd95d 100644 --- a/pkg/controller/servicetopology/endpointslice/endpointslice_enqueue_handlers.go +++ b/pkg/yurtmanager/controller/servicetopology/endpointslice/endpointslice_enqueue_handlers.go @@ -25,8 +25,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/openyurtio/openyurt/pkg/controller/servicetopology/adapter" - "github.com/openyurtio/openyurt/pkg/controller/servicetopology/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology/adapter" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/servicetopology/util" ) type EnqueueEndpointsliceForService struct { diff --git a/pkg/controller/servicetopology/util/util.go b/pkg/yurtmanager/controller/servicetopology/util/util.go similarity index 100% rename from pkg/controller/servicetopology/util/util.go rename to pkg/yurtmanager/controller/servicetopology/util/util.go diff --git a/pkg/controller/util/controller_utils.go b/pkg/yurtmanager/controller/util/controller_utils.go similarity index 100% rename from pkg/controller/util/controller_utils.go rename to pkg/yurtmanager/controller/util/controller_utils.go diff --git a/pkg/controller/util/controller_utils_test.go b/pkg/yurtmanager/controller/util/controller_utils_test.go similarity index 100% rename from pkg/controller/util/controller_utils_test.go rename to pkg/yurtmanager/controller/util/controller_utils_test.go diff --git a/pkg/controller/util/node/controller_utils.go b/pkg/yurtmanager/controller/util/node/controller_utils.go similarity index 100% rename from pkg/controller/util/node/controller_utils.go rename to pkg/yurtmanager/controller/util/node/controller_utils.go diff --git a/pkg/controller/util/refmanager/ref_manager.go b/pkg/yurtmanager/controller/util/refmanager/ref_manager.go similarity index 100% rename from pkg/controller/util/refmanager/ref_manager.go rename to pkg/yurtmanager/controller/util/refmanager/ref_manager.go diff --git a/pkg/controller/util/refmanager/ref_manager_test.go b/pkg/yurtmanager/controller/util/refmanager/ref_manager_test.go similarity index 100% rename from pkg/controller/util/refmanager/ref_manager_test.go rename to pkg/yurtmanager/controller/util/refmanager/ref_manager_test.go diff --git a/pkg/controller/util/tools.go b/pkg/yurtmanager/controller/util/tools.go similarity index 100% rename from pkg/controller/util/tools.go rename to pkg/yurtmanager/controller/util/tools.go diff --git a/pkg/controller/util/tools_test.go b/pkg/yurtmanager/controller/util/tools_test.go similarity index 100% rename from pkg/controller/util/tools_test.go rename to pkg/yurtmanager/controller/util/tools_test.go diff --git a/pkg/controller/yurtappdaemon/config/types.go b/pkg/yurtmanager/controller/yurtappdaemon/config/types.go similarity index 100% rename from pkg/controller/yurtappdaemon/config/types.go rename to pkg/yurtmanager/controller/yurtappdaemon/config/types.go diff --git a/pkg/controller/yurtappdaemon/nodepool_enqueue_handlers.go b/pkg/yurtmanager/controller/yurtappdaemon/nodepool_enqueue_handlers.go similarity index 100% rename from pkg/controller/yurtappdaemon/nodepool_enqueue_handlers.go rename to pkg/yurtmanager/controller/yurtappdaemon/nodepool_enqueue_handlers.go diff --git a/pkg/controller/yurtappdaemon/nodepool_enqueue_handlers_test.go b/pkg/yurtmanager/controller/yurtappdaemon/nodepool_enqueue_handlers_test.go similarity index 100% rename from pkg/controller/yurtappdaemon/nodepool_enqueue_handlers_test.go rename to pkg/yurtmanager/controller/yurtappdaemon/nodepool_enqueue_handlers_test.go diff --git a/pkg/controller/yurtappdaemon/revision.go b/pkg/yurtmanager/controller/yurtappdaemon/revision.go similarity index 99% rename from pkg/controller/yurtappdaemon/revision.go rename to pkg/yurtmanager/controller/yurtappdaemon/revision.go index 6e1bb4b8e57..e05a990c8bf 100644 --- a/pkg/controller/yurtappdaemon/revision.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/revision.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/apiutil" appsalphav1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/controller/util/refmanager" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/refmanager" ) func (r *ReconcileYurtAppDaemon) controlledHistories(yad *appsalphav1.YurtAppDaemon) ([]*apps.ControllerRevision, error) { diff --git a/pkg/controller/yurtappdaemon/revision_test.go b/pkg/yurtmanager/controller/yurtappdaemon/revision_test.go similarity index 100% rename from pkg/controller/yurtappdaemon/revision_test.go rename to pkg/yurtmanager/controller/yurtappdaemon/revision_test.go diff --git a/pkg/controller/yurtappdaemon/util.go b/pkg/yurtmanager/controller/yurtappdaemon/util.go similarity index 100% rename from pkg/controller/yurtappdaemon/util.go rename to pkg/yurtmanager/controller/yurtappdaemon/util.go diff --git a/pkg/controller/yurtappdaemon/util_test.go b/pkg/yurtmanager/controller/yurtappdaemon/util_test.go similarity index 100% rename from pkg/controller/yurtappdaemon/util_test.go rename to pkg/yurtmanager/controller/yurtappdaemon/util_test.go diff --git a/pkg/controller/yurtappdaemon/workloadcontroller/controller.go b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/controller.go similarity index 100% rename from pkg/controller/yurtappdaemon/workloadcontroller/controller.go rename to pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/controller.go diff --git a/pkg/controller/yurtappdaemon/workloadcontroller/deployment_controller.go b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/deployment_controller.go similarity index 98% rename from pkg/controller/yurtappdaemon/workloadcontroller/deployment_controller.go rename to pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/deployment_controller.go index 0e3fb0dc27b..e2985991a4d 100644 --- a/pkg/controller/yurtappdaemon/workloadcontroller/deployment_controller.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/deployment_controller.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/controller/util/refmanager" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/refmanager" ) const updateRetries = 5 diff --git a/pkg/controller/yurtappdaemon/workloadcontroller/deployment_controller_test.go b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/deployment_controller_test.go similarity index 100% rename from pkg/controller/yurtappdaemon/workloadcontroller/deployment_controller_test.go rename to pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/deployment_controller_test.go diff --git a/pkg/controller/yurtappdaemon/workloadcontroller/statefulset_controller.go b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/statefulset_controller.go similarity index 100% rename from pkg/controller/yurtappdaemon/workloadcontroller/statefulset_controller.go rename to pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/statefulset_controller.go diff --git a/pkg/controller/yurtappdaemon/workloadcontroller/util.go b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/util.go similarity index 100% rename from pkg/controller/yurtappdaemon/workloadcontroller/util.go rename to pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/util.go diff --git a/pkg/controller/yurtappdaemon/workloadcontroller/util_test.go b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/util_test.go similarity index 100% rename from pkg/controller/yurtappdaemon/workloadcontroller/util_test.go rename to pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/util_test.go diff --git a/pkg/controller/yurtappdaemon/workloadcontroller/workload.go b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/workload.go similarity index 100% rename from pkg/controller/yurtappdaemon/workloadcontroller/workload.go rename to pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/workload.go diff --git a/pkg/controller/yurtappdaemon/workloadcontroller/workload_test.go b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/workload_test.go similarity index 100% rename from pkg/controller/yurtappdaemon/workloadcontroller/workload_test.go rename to pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/workload_test.go diff --git a/pkg/controller/yurtappdaemon/yurtappdaemon_controller.go b/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller.go similarity index 99% rename from pkg/controller/yurtappdaemon/yurtappdaemon_controller.go rename to pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller.go index 0434ec7b70f..097a092ef07 100644 --- a/pkg/controller/yurtappdaemon/yurtappdaemon_controller.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller.go @@ -39,8 +39,8 @@ import ( "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/controller/util" - "github.com/openyurtio/openyurt/pkg/controller/yurtappdaemon/workloadcontroller" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller" ) var ( diff --git a/pkg/controller/yurtappdaemon/yurtappdaemon_controller_test.go b/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller_test.go similarity index 99% rename from pkg/controller/yurtappdaemon/yurtappdaemon_controller_test.go rename to pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller_test.go index 0bc6a05c193..9a668ac6963 100644 --- a/pkg/controller/yurtappdaemon/yurtappdaemon_controller_test.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/controller/yurtappdaemon/workloadcontroller" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller" ) //func TestAdd(t *testing.T) { diff --git a/pkg/controller/yurtappset/adapter/adapter.go b/pkg/yurtmanager/controller/yurtappset/adapter/adapter.go similarity index 100% rename from pkg/controller/yurtappset/adapter/adapter.go rename to pkg/yurtmanager/controller/yurtappset/adapter/adapter.go diff --git a/pkg/controller/yurtappset/adapter/adapter_util.go b/pkg/yurtmanager/controller/yurtappset/adapter/adapter_util.go similarity index 100% rename from pkg/controller/yurtappset/adapter/adapter_util.go rename to pkg/yurtmanager/controller/yurtappset/adapter/adapter_util.go diff --git a/pkg/controller/yurtappset/adapter/adapter_util_test.go b/pkg/yurtmanager/controller/yurtappset/adapter/adapter_util_test.go similarity index 100% rename from pkg/controller/yurtappset/adapter/adapter_util_test.go rename to pkg/yurtmanager/controller/yurtappset/adapter/adapter_util_test.go diff --git a/pkg/controller/yurtappset/adapter/deployment_adapter.go b/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter.go similarity index 100% rename from pkg/controller/yurtappset/adapter/deployment_adapter.go rename to pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter.go diff --git a/pkg/controller/yurtappset/adapter/deployment_adapter_test.go b/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter_test.go similarity index 100% rename from pkg/controller/yurtappset/adapter/deployment_adapter_test.go rename to pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter_test.go diff --git a/pkg/controller/yurtappset/adapter/statefulset_adapter.go b/pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter.go similarity index 100% rename from pkg/controller/yurtappset/adapter/statefulset_adapter.go rename to pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter.go diff --git a/pkg/controller/yurtappset/adapter/statefulset_adapter_test.go b/pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter_test.go similarity index 100% rename from pkg/controller/yurtappset/adapter/statefulset_adapter_test.go rename to pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter_test.go diff --git a/pkg/controller/yurtappset/config/types.go b/pkg/yurtmanager/controller/yurtappset/config/types.go similarity index 100% rename from pkg/controller/yurtappset/config/types.go rename to pkg/yurtmanager/controller/yurtappset/config/types.go diff --git a/pkg/controller/yurtappset/pool.go b/pkg/yurtmanager/controller/yurtappset/pool.go similarity index 96% rename from pkg/controller/yurtappset/pool.go rename to pkg/yurtmanager/controller/yurtappset/pool.go index 4d0a7c0b1cf..a579a8f3a2f 100644 --- a/pkg/controller/yurtappset/pool.go +++ b/pkg/yurtmanager/controller/yurtappset/pool.go @@ -21,7 +21,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/controller/yurtappset/adapter" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/adapter" ) // Pool stores the details of a pool resource owned by one YurtAppSet. diff --git a/pkg/controller/yurtappset/pool_control.go b/pkg/yurtmanager/controller/yurtappset/pool_control.go similarity index 97% rename from pkg/controller/yurtappset/pool_control.go rename to pkg/yurtmanager/controller/yurtappset/pool_control.go index 98530e74f42..34791df33bf 100644 --- a/pkg/controller/yurtappset/pool_control.go +++ b/pkg/yurtmanager/controller/yurtappset/pool_control.go @@ -29,8 +29,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/controller/util/refmanager" - "github.com/openyurtio/openyurt/pkg/controller/yurtappset/adapter" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/refmanager" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/adapter" ) // PoolControl provides pool operations of MutableSet. diff --git a/pkg/controller/yurtappset/pool_controller_test.go b/pkg/yurtmanager/controller/yurtappset/pool_controller_test.go similarity index 98% rename from pkg/controller/yurtappset/pool_controller_test.go rename to pkg/yurtmanager/controller/yurtappset/pool_controller_test.go index 9a79e5e2278..da2c8e0ac5a 100644 --- a/pkg/controller/yurtappset/pool_controller_test.go +++ b/pkg/yurtmanager/controller/yurtappset/pool_controller_test.go @@ -29,7 +29,7 @@ import ( fakeclint "sigs.k8s.io/controller-runtime/pkg/client/fake" appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - adpt "github.com/openyurtio/openyurt/pkg/controller/yurtappset/adapter" + adpt "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/adapter" ) var ( diff --git a/pkg/controller/yurtappset/revision.go b/pkg/yurtmanager/controller/yurtappset/revision.go similarity index 99% rename from pkg/controller/yurtappset/revision.go rename to pkg/yurtmanager/controller/yurtappset/revision.go index 35b33f338cb..987e72e8231 100644 --- a/pkg/controller/yurtappset/revision.go +++ b/pkg/yurtmanager/controller/yurtappset/revision.go @@ -34,7 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/apiutil" appsalphav1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/controller/util/refmanager" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/refmanager" ) // ControllerRevisionHashLabel is the label used to indicate the hash value of a ControllerRevision's Data. diff --git a/pkg/controller/yurtappset/revision_test.go b/pkg/yurtmanager/controller/yurtappset/revision_test.go similarity index 100% rename from pkg/controller/yurtappset/revision_test.go rename to pkg/yurtmanager/controller/yurtappset/revision_test.go diff --git a/pkg/controller/yurtappset/yurtappset_controller.go b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller.go similarity index 99% rename from pkg/controller/yurtappset/yurtappset_controller.go rename to pkg/yurtmanager/controller/yurtappset/yurtappset_controller.go index eeeb4c79e49..2d3edaa2dd5 100644 --- a/pkg/controller/yurtappset/yurtappset_controller.go +++ b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller.go @@ -42,7 +42,7 @@ import ( "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/controller/yurtappset/adapter" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/adapter" ) func init() { diff --git a/pkg/controller/yurtappset/yurtappset_controller_statefulset_test.go b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller_statefulset_test.go similarity index 100% rename from pkg/controller/yurtappset/yurtappset_controller_statefulset_test.go rename to pkg/yurtmanager/controller/yurtappset/yurtappset_controller_statefulset_test.go diff --git a/pkg/controller/yurtappset/yurtappset_controller_suite_test.go b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller_suite_test.go similarity index 100% rename from pkg/controller/yurtappset/yurtappset_controller_suite_test.go rename to pkg/yurtmanager/controller/yurtappset/yurtappset_controller_suite_test.go diff --git a/pkg/controller/yurtappset/yurtappset_controller_test.go b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller_test.go similarity index 97% rename from pkg/controller/yurtappset/yurtappset_controller_test.go rename to pkg/yurtmanager/controller/yurtappset/yurtappset_controller_test.go index 3ebfbc26940..18a2a364a10 100644 --- a/pkg/controller/yurtappset/yurtappset_controller_test.go +++ b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller_test.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - adpt "github.com/openyurtio/openyurt/pkg/controller/yurtappset/adapter" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/adapter" ) var ( @@ -150,12 +150,12 @@ func TestReconcileYurtAppSet_Reconcile(t *testing.T) { appsv1alpha1.StatefulSetTemplateType: &PoolControl{ Client: fc, scheme: scheme, - adapter: &adpt.StatefulSetAdapter{Client: fc, Scheme: scheme}, + adapter: &adapter.StatefulSetAdapter{Client: fc, Scheme: scheme}, }, appsv1alpha1.DeploymentTemplateType: &PoolControl{ Client: fc, scheme: scheme, - adapter: &adpt.DeploymentAdapter{Client: fc, Scheme: scheme}, + adapter: &adapter.DeploymentAdapter{Client: fc, Scheme: scheme}, }, }, } diff --git a/pkg/controller/yurtappset/yurtappset_controller_utils.go b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller_utils.go similarity index 100% rename from pkg/controller/yurtappset/yurtappset_controller_utils.go rename to pkg/yurtmanager/controller/yurtappset/yurtappset_controller_utils.go diff --git a/pkg/controller/yurtappset/yurtappset_update.go b/pkg/yurtmanager/controller/yurtappset/yurtappset_update.go similarity index 99% rename from pkg/controller/yurtappset/yurtappset_update.go rename to pkg/yurtmanager/controller/yurtappset/yurtappset_update.go index acfaa4f4999..6a73afd915b 100644 --- a/pkg/controller/yurtappset/yurtappset_update.go +++ b/pkg/yurtmanager/controller/yurtappset/yurtappset_update.go @@ -32,7 +32,7 @@ import ( "k8s.io/klog/v2" unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/controller/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util" ) func (r *ReconcileYurtAppSet) managePools(yas *unitv1alpha1.YurtAppSet, diff --git a/pkg/controller/yurtcoordinator/cert/certificate.go b/pkg/yurtmanager/controller/yurtcoordinator/cert/certificate.go similarity index 100% rename from pkg/controller/yurtcoordinator/cert/certificate.go rename to pkg/yurtmanager/controller/yurtcoordinator/cert/certificate.go diff --git a/pkg/controller/yurtcoordinator/cert/certificate_test.go b/pkg/yurtmanager/controller/yurtcoordinator/cert/certificate_test.go similarity index 100% rename from pkg/controller/yurtcoordinator/cert/certificate_test.go rename to pkg/yurtmanager/controller/yurtcoordinator/cert/certificate_test.go diff --git a/pkg/controller/yurtcoordinator/cert/secret.go b/pkg/yurtmanager/controller/yurtcoordinator/cert/secret.go similarity index 100% rename from pkg/controller/yurtcoordinator/cert/secret.go rename to pkg/yurtmanager/controller/yurtcoordinator/cert/secret.go diff --git a/pkg/controller/yurtcoordinator/cert/secret_test.go b/pkg/yurtmanager/controller/yurtcoordinator/cert/secret_test.go similarity index 100% rename from pkg/controller/yurtcoordinator/cert/secret_test.go rename to pkg/yurtmanager/controller/yurtcoordinator/cert/secret_test.go diff --git a/pkg/controller/yurtcoordinator/cert/util.go b/pkg/yurtmanager/controller/yurtcoordinator/cert/util.go similarity index 100% rename from pkg/controller/yurtcoordinator/cert/util.go rename to pkg/yurtmanager/controller/yurtcoordinator/cert/util.go diff --git a/pkg/controller/yurtcoordinator/cert/util_test.go b/pkg/yurtmanager/controller/yurtcoordinator/cert/util_test.go similarity index 100% rename from pkg/controller/yurtcoordinator/cert/util_test.go rename to pkg/yurtmanager/controller/yurtcoordinator/cert/util_test.go diff --git a/pkg/controller/yurtcoordinator/cert/yurtcoordinatorcert_controller.go b/pkg/yurtmanager/controller/yurtcoordinator/cert/yurtcoordinatorcert_controller.go similarity index 100% rename from pkg/controller/yurtcoordinator/cert/yurtcoordinatorcert_controller.go rename to pkg/yurtmanager/controller/yurtcoordinator/cert/yurtcoordinatorcert_controller.go diff --git a/pkg/controller/yurtcoordinator/cert/yurtcoordinatorcert_controller_test.go b/pkg/yurtmanager/controller/yurtcoordinator/cert/yurtcoordinatorcert_controller_test.go similarity index 100% rename from pkg/controller/yurtcoordinator/cert/yurtcoordinatorcert_controller_test.go rename to pkg/yurtmanager/controller/yurtcoordinator/cert/yurtcoordinatorcert_controller_test.go diff --git a/pkg/controller/yurtcoordinator/constant/constant.go b/pkg/yurtmanager/controller/yurtcoordinator/constant/constant.go similarity index 100% rename from pkg/controller/yurtcoordinator/constant/constant.go rename to pkg/yurtmanager/controller/yurtcoordinator/constant/constant.go diff --git a/pkg/controller/yurtcoordinator/delegatelease/delegatelease_controller.go b/pkg/yurtmanager/controller/yurtcoordinator/delegatelease/delegatelease_controller.go similarity index 96% rename from pkg/controller/yurtcoordinator/delegatelease/delegatelease_controller.go rename to pkg/yurtmanager/controller/yurtcoordinator/delegatelease/delegatelease_controller.go index ea5f68f2886..7420f6007a1 100644 --- a/pkg/controller/yurtcoordinator/delegatelease/delegatelease_controller.go +++ b/pkg/yurtmanager/controller/yurtcoordinator/delegatelease/delegatelease_controller.go @@ -36,9 +36,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" - nodeutil "github.com/openyurtio/openyurt/pkg/controller/util/node" - "github.com/openyurtio/openyurt/pkg/controller/yurtcoordinator/constant" - "github.com/openyurtio/openyurt/pkg/controller/yurtcoordinator/utils" + nodeutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/node" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtcoordinator/constant" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtcoordinator/utils" ) func init() { diff --git a/pkg/controller/yurtcoordinator/delegatelease/delegatelease_controller_test.go b/pkg/yurtmanager/controller/yurtcoordinator/delegatelease/delegatelease_controller_test.go similarity index 93% rename from pkg/controller/yurtcoordinator/delegatelease/delegatelease_controller_test.go rename to pkg/yurtmanager/controller/yurtcoordinator/delegatelease/delegatelease_controller_test.go index 037fd9c6fb3..57ada1cf45e 100644 --- a/pkg/controller/yurtcoordinator/delegatelease/delegatelease_controller_test.go +++ b/pkg/yurtmanager/controller/yurtcoordinator/delegatelease/delegatelease_controller_test.go @@ -21,7 +21,7 @@ import ( corev1 "k8s.io/api/core/v1" - "github.com/openyurtio/openyurt/pkg/controller/yurtcoordinator/utils" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtcoordinator/utils" ) func TestTaintNode(t *testing.T) { diff --git a/pkg/controller/yurtcoordinator/podbinding/podbinding_controller.go b/pkg/yurtmanager/controller/yurtcoordinator/podbinding/podbinding_controller.go similarity index 98% rename from pkg/controller/yurtcoordinator/podbinding/podbinding_controller.go rename to pkg/yurtmanager/controller/yurtcoordinator/podbinding/podbinding_controller.go index 02cfd571e07..cdd02666f58 100644 --- a/pkg/controller/yurtcoordinator/podbinding/podbinding_controller.go +++ b/pkg/yurtmanager/controller/yurtcoordinator/podbinding/podbinding_controller.go @@ -33,8 +33,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" - "github.com/openyurtio/openyurt/pkg/controller/yurtcoordinator/constant" "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtcoordinator/constant" ) func init() { diff --git a/pkg/controller/yurtcoordinator/podbinding/podbinding_controller_test.go b/pkg/yurtmanager/controller/yurtcoordinator/podbinding/podbinding_controller_test.go similarity index 100% rename from pkg/controller/yurtcoordinator/podbinding/podbinding_controller_test.go rename to pkg/yurtmanager/controller/yurtcoordinator/podbinding/podbinding_controller_test.go diff --git a/pkg/controller/yurtcoordinator/utils/lease.go b/pkg/yurtmanager/controller/yurtcoordinator/utils/lease.go similarity index 94% rename from pkg/controller/yurtcoordinator/utils/lease.go rename to pkg/yurtmanager/controller/yurtcoordinator/utils/lease.go index 188fd1f49d7..2af5532d7d6 100644 --- a/pkg/controller/yurtcoordinator/utils/lease.go +++ b/pkg/yurtmanager/controller/yurtcoordinator/utils/lease.go @@ -20,7 +20,7 @@ package utils import ( "sync" - "github.com/openyurtio/openyurt/pkg/controller/yurtcoordinator/constant" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtcoordinator/constant" ) type LeaseDelegatedCounter struct { diff --git a/pkg/controller/yurtcoordinator/utils/lease_test.go b/pkg/yurtmanager/controller/yurtcoordinator/utils/lease_test.go similarity index 100% rename from pkg/controller/yurtcoordinator/utils/lease_test.go rename to pkg/yurtmanager/controller/yurtcoordinator/utils/lease_test.go diff --git a/pkg/controller/yurtcoordinator/utils/taints.go b/pkg/yurtmanager/controller/yurtcoordinator/utils/taints.go similarity index 100% rename from pkg/controller/yurtcoordinator/utils/taints.go rename to pkg/yurtmanager/controller/yurtcoordinator/utils/taints.go diff --git a/pkg/controller/yurtcoordinator/utils/taints_test.go b/pkg/yurtmanager/controller/yurtcoordinator/utils/taints_test.go similarity index 93% rename from pkg/controller/yurtcoordinator/utils/taints_test.go rename to pkg/yurtmanager/controller/yurtcoordinator/utils/taints_test.go index 46e15e740e1..4c052ba1110 100644 --- a/pkg/controller/yurtcoordinator/utils/taints_test.go +++ b/pkg/yurtmanager/controller/yurtcoordinator/utils/taints_test.go @@ -21,7 +21,7 @@ import ( v1 "k8s.io/api/core/v1" - "github.com/openyurtio/openyurt/pkg/controller/yurtcoordinator/constant" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtcoordinator/constant" ) func TestDeleteTaintsByKey(t *testing.T) { diff --git a/pkg/controller/yurtstaticset/config/types.go b/pkg/yurtmanager/controller/yurtstaticset/config/types.go similarity index 100% rename from pkg/controller/yurtstaticset/config/types.go rename to pkg/yurtmanager/controller/yurtstaticset/config/types.go diff --git a/pkg/controller/yurtstaticset/upgradeinfo/upgrade_info.go b/pkg/yurtmanager/controller/yurtstaticset/upgradeinfo/upgrade_info.go similarity index 99% rename from pkg/controller/yurtstaticset/upgradeinfo/upgrade_info.go rename to pkg/yurtmanager/controller/yurtstaticset/upgradeinfo/upgrade_info.go index be4d1476dbf..6027fe708c1 100644 --- a/pkg/controller/yurtstaticset/upgradeinfo/upgrade_info.go +++ b/pkg/yurtmanager/controller/yurtstaticset/upgradeinfo/upgrade_info.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/controller/yurtstaticset/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset/util" ) const ( diff --git a/pkg/controller/yurtstaticset/upgradeinfo/upgrade_info_test.go b/pkg/yurtmanager/controller/yurtstaticset/upgradeinfo/upgrade_info_test.go similarity index 100% rename from pkg/controller/yurtstaticset/upgradeinfo/upgrade_info_test.go rename to pkg/yurtmanager/controller/yurtstaticset/upgradeinfo/upgrade_info_test.go diff --git a/pkg/controller/yurtstaticset/util/util.go b/pkg/yurtmanager/controller/yurtstaticset/util/util.go similarity index 100% rename from pkg/controller/yurtstaticset/util/util.go rename to pkg/yurtmanager/controller/yurtstaticset/util/util.go diff --git a/pkg/controller/yurtstaticset/yurtstaticset_controller.go b/pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller.go similarity index 98% rename from pkg/controller/yurtstaticset/yurtstaticset_controller.go rename to pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller.go index 2047b64a051..16e76fb9092 100644 --- a/pkg/controller/yurtstaticset/yurtstaticset_controller.go +++ b/pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller.go @@ -42,9 +42,9 @@ import ( appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/controller/yurtstaticset/config" - "github.com/openyurtio/openyurt/pkg/controller/yurtstaticset/upgradeinfo" - "github.com/openyurtio/openyurt/pkg/controller/yurtstaticset/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset/config" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset/upgradeinfo" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset/util" ) func init() { diff --git a/pkg/controller/yurtstaticset/yurtstaticset_controller_test.go b/pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller_test.go similarity index 98% rename from pkg/controller/yurtstaticset/yurtstaticset_controller_test.go rename to pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller_test.go index 76eece0d502..4508aa78b43 100644 --- a/pkg/controller/yurtstaticset/yurtstaticset_controller_test.go +++ b/pkg/yurtmanager/controller/yurtstaticset/yurtstaticset_controller_test.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/controller/yurtstaticset/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset/util" ) const ( diff --git a/pkg/webhook/builder/defaulter_custom.go b/pkg/yurtmanager/webhook/builder/defaulter_custom.go similarity index 100% rename from pkg/webhook/builder/defaulter_custom.go rename to pkg/yurtmanager/webhook/builder/defaulter_custom.go diff --git a/pkg/webhook/builder/validator_custom.go b/pkg/yurtmanager/webhook/builder/validator_custom.go similarity index 100% rename from pkg/webhook/builder/validator_custom.go rename to pkg/yurtmanager/webhook/builder/validator_custom.go diff --git a/pkg/webhook/builder/webhook.go b/pkg/yurtmanager/webhook/builder/webhook.go similarity index 98% rename from pkg/webhook/builder/webhook.go rename to pkg/yurtmanager/webhook/builder/webhook.go index 240c7694617..a70bcbe4833 100644 --- a/pkg/webhook/builder/webhook.go +++ b/pkg/yurtmanager/webhook/builder/webhook.go @@ -30,7 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "sigs.k8s.io/controller-runtime/pkg/webhook/conversion" - "github.com/openyurtio/openyurt/pkg/webhook/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" ) // WebhookBuilder builds a Webhook. diff --git a/pkg/webhook/gateway/v1alpha1/gateway_default.go b/pkg/yurtmanager/webhook/gateway/v1alpha1/gateway_default.go similarity index 100% rename from pkg/webhook/gateway/v1alpha1/gateway_default.go rename to pkg/yurtmanager/webhook/gateway/v1alpha1/gateway_default.go diff --git a/pkg/webhook/gateway/v1alpha1/gateway_handler.go b/pkg/yurtmanager/webhook/gateway/v1alpha1/gateway_handler.go similarity index 96% rename from pkg/webhook/gateway/v1alpha1/gateway_handler.go rename to pkg/yurtmanager/webhook/gateway/v1alpha1/gateway_handler.go index 97573b424a4..d2f0b5272fc 100644 --- a/pkg/webhook/gateway/v1alpha1/gateway_handler.go +++ b/pkg/yurtmanager/webhook/gateway/v1alpha1/gateway_handler.go @@ -23,7 +23,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" "github.com/openyurtio/openyurt/pkg/apis/raven/v1alpha1" - "github.com/openyurtio/openyurt/pkg/webhook/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" ) // SetupWebhookWithManager sets up Cluster webhooks. mutate path, validatepath, error diff --git a/pkg/webhook/gateway/v1alpha1/gateway_validation.go b/pkg/yurtmanager/webhook/gateway/v1alpha1/gateway_validation.go similarity index 100% rename from pkg/webhook/gateway/v1alpha1/gateway_validation.go rename to pkg/yurtmanager/webhook/gateway/v1alpha1/gateway_validation.go diff --git a/pkg/webhook/gateway/v1beta1/gateway_default.go b/pkg/yurtmanager/webhook/gateway/v1beta1/gateway_default.go similarity index 100% rename from pkg/webhook/gateway/v1beta1/gateway_default.go rename to pkg/yurtmanager/webhook/gateway/v1beta1/gateway_default.go diff --git a/pkg/webhook/gateway/v1beta1/gateway_handler.go b/pkg/yurtmanager/webhook/gateway/v1beta1/gateway_handler.go similarity index 97% rename from pkg/webhook/gateway/v1beta1/gateway_handler.go rename to pkg/yurtmanager/webhook/gateway/v1beta1/gateway_handler.go index 1cd33b5540a..0193b8c9e6c 100644 --- a/pkg/webhook/gateway/v1beta1/gateway_handler.go +++ b/pkg/yurtmanager/webhook/gateway/v1beta1/gateway_handler.go @@ -23,7 +23,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" "github.com/openyurtio/openyurt/pkg/apis/raven/v1beta1" - "github.com/openyurtio/openyurt/pkg/webhook/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" ) // SetupWebhookWithManager sets up Cluster webhooks. mutate path, validatepath, error diff --git a/pkg/webhook/gateway/v1beta1/gateway_validation.go b/pkg/yurtmanager/webhook/gateway/v1beta1/gateway_validation.go similarity index 100% rename from pkg/webhook/gateway/v1beta1/gateway_validation.go rename to pkg/yurtmanager/webhook/gateway/v1beta1/gateway_validation.go diff --git a/pkg/webhook/node/v1/node_default.go b/pkg/yurtmanager/webhook/node/v1/node_default.go similarity index 100% rename from pkg/webhook/node/v1/node_default.go rename to pkg/yurtmanager/webhook/node/v1/node_default.go diff --git a/pkg/webhook/node/v1/node_default_test.go b/pkg/yurtmanager/webhook/node/v1/node_default_test.go similarity index 100% rename from pkg/webhook/node/v1/node_default_test.go rename to pkg/yurtmanager/webhook/node/v1/node_default_test.go diff --git a/pkg/webhook/node/v1/node_handler.go b/pkg/yurtmanager/webhook/node/v1/node_handler.go similarity index 93% rename from pkg/webhook/node/v1/node_handler.go rename to pkg/yurtmanager/webhook/node/v1/node_handler.go index f6cbc5b6aa0..e63bc1730a3 100644 --- a/pkg/webhook/node/v1/node_handler.go +++ b/pkg/yurtmanager/webhook/node/v1/node_handler.go @@ -22,8 +22,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - "github.com/openyurtio/openyurt/pkg/webhook/builder" - "github.com/openyurtio/openyurt/pkg/webhook/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/builder" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" ) const ( diff --git a/pkg/webhook/node/v1/node_validation.go b/pkg/yurtmanager/webhook/node/v1/node_validation.go similarity index 100% rename from pkg/webhook/node/v1/node_validation.go rename to pkg/yurtmanager/webhook/node/v1/node_validation.go diff --git a/pkg/webhook/node/v1/node_validation_test.go b/pkg/yurtmanager/webhook/node/v1/node_validation_test.go similarity index 100% rename from pkg/webhook/node/v1/node_validation_test.go rename to pkg/yurtmanager/webhook/node/v1/node_validation_test.go diff --git a/pkg/webhook/nodepool/v1beta1/nodepool_default.go b/pkg/yurtmanager/webhook/nodepool/v1beta1/nodepool_default.go similarity index 100% rename from pkg/webhook/nodepool/v1beta1/nodepool_default.go rename to pkg/yurtmanager/webhook/nodepool/v1beta1/nodepool_default.go diff --git a/pkg/webhook/nodepool/v1beta1/nodepool_default_test.go b/pkg/yurtmanager/webhook/nodepool/v1beta1/nodepool_default_test.go similarity index 100% rename from pkg/webhook/nodepool/v1beta1/nodepool_default_test.go rename to pkg/yurtmanager/webhook/nodepool/v1beta1/nodepool_default_test.go diff --git a/pkg/webhook/nodepool/v1beta1/nodepool_handler.go b/pkg/yurtmanager/webhook/nodepool/v1beta1/nodepool_handler.go similarity index 97% rename from pkg/webhook/nodepool/v1beta1/nodepool_handler.go rename to pkg/yurtmanager/webhook/nodepool/v1beta1/nodepool_handler.go index d1fa643d5f7..0bf0f3f876f 100644 --- a/pkg/webhook/nodepool/v1beta1/nodepool_handler.go +++ b/pkg/yurtmanager/webhook/nodepool/v1beta1/nodepool_handler.go @@ -23,7 +23,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" "github.com/openyurtio/openyurt/pkg/apis/apps/v1beta1" - "github.com/openyurtio/openyurt/pkg/webhook/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" ) // SetupWebhookWithManager sets up Cluster webhooks. mutate path, validatepath, error diff --git a/pkg/webhook/nodepool/v1beta1/nodepool_validation.go b/pkg/yurtmanager/webhook/nodepool/v1beta1/nodepool_validation.go similarity index 95% rename from pkg/webhook/nodepool/v1beta1/nodepool_validation.go rename to pkg/yurtmanager/webhook/nodepool/v1beta1/nodepool_validation.go index 81f7959d1a6..5c14370f339 100644 --- a/pkg/webhook/nodepool/v1beta1/nodepool_validation.go +++ b/pkg/yurtmanager/webhook/nodepool/v1beta1/nodepool_validation.go @@ -105,6 +105,11 @@ func validateNodePoolSpec(spec *appsv1beta1.NodePoolSpec) field.ErrorList { if spec.Type != appsv1beta1.Edge && spec.Type != appsv1beta1.Cloud { return []*field.Error{field.Invalid(field.NewPath("spec").Child("type"), spec.Type, "pool type should be Edge or Cloud")} } + + // Cloud NodePool can not set HostNetwork=true + if spec.Type == appsv1beta1.Cloud && spec.HostNetwork { + return []*field.Error{field.Invalid(field.NewPath("spec").Child("hostNetwork"), spec.HostNetwork, "Cloud NodePool cloud not support hostNetwork")} + } return nil } diff --git a/pkg/webhook/nodepool/v1beta1/nodepool_validation_test.go b/pkg/yurtmanager/webhook/nodepool/v1beta1/nodepool_validation_test.go similarity index 100% rename from pkg/webhook/nodepool/v1beta1/nodepool_validation_test.go rename to pkg/yurtmanager/webhook/nodepool/v1beta1/nodepool_validation_test.go diff --git a/pkg/webhook/platformadmin/v1alpha1/platformadmin_default.go b/pkg/yurtmanager/webhook/platformadmin/v1alpha1/platformadmin_default.go similarity index 100% rename from pkg/webhook/platformadmin/v1alpha1/platformadmin_default.go rename to pkg/yurtmanager/webhook/platformadmin/v1alpha1/platformadmin_default.go diff --git a/pkg/webhook/platformadmin/v1alpha1/platformadmin_handler.go b/pkg/yurtmanager/webhook/platformadmin/v1alpha1/platformadmin_handler.go similarity index 94% rename from pkg/webhook/platformadmin/v1alpha1/platformadmin_handler.go rename to pkg/yurtmanager/webhook/platformadmin/v1alpha1/platformadmin_handler.go index 83a7e4e3e92..7dc5b2e6170 100644 --- a/pkg/webhook/platformadmin/v1alpha1/platformadmin_handler.go +++ b/pkg/yurtmanager/webhook/platformadmin/v1alpha1/platformadmin_handler.go @@ -25,8 +25,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" - "github.com/openyurtio/openyurt/pkg/controller/platformadmin/config" - "github.com/openyurtio/openyurt/pkg/webhook/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/platformadmin/config" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" ) type Manifest struct { diff --git a/pkg/webhook/platformadmin/v1alpha1/platformadmin_validation.go b/pkg/yurtmanager/webhook/platformadmin/v1alpha1/platformadmin_validation.go similarity index 98% rename from pkg/webhook/platformadmin/v1alpha1/platformadmin_validation.go rename to pkg/yurtmanager/webhook/platformadmin/v1alpha1/platformadmin_validation.go index 538017c4628..fa4236bf794 100644 --- a/pkg/webhook/platformadmin/v1alpha1/platformadmin_validation.go +++ b/pkg/yurtmanager/webhook/platformadmin/v1alpha1/platformadmin_validation.go @@ -27,7 +27,7 @@ import ( unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" - util "github.com/openyurtio/openyurt/pkg/controller/platformadmin/utils" + util "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/platformadmin/utils" ) // ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. diff --git a/pkg/webhook/platformadmin/v1alpha2/platformadmin_default.go b/pkg/yurtmanager/webhook/platformadmin/v1alpha2/platformadmin_default.go similarity index 100% rename from pkg/webhook/platformadmin/v1alpha2/platformadmin_default.go rename to pkg/yurtmanager/webhook/platformadmin/v1alpha2/platformadmin_default.go diff --git a/pkg/webhook/platformadmin/v1alpha2/platformadmin_handler.go b/pkg/yurtmanager/webhook/platformadmin/v1alpha2/platformadmin_handler.go similarity index 95% rename from pkg/webhook/platformadmin/v1alpha2/platformadmin_handler.go rename to pkg/yurtmanager/webhook/platformadmin/v1alpha2/platformadmin_handler.go index a6b193a0b74..aa707e3e879 100644 --- a/pkg/webhook/platformadmin/v1alpha2/platformadmin_handler.go +++ b/pkg/yurtmanager/webhook/platformadmin/v1alpha2/platformadmin_handler.go @@ -25,8 +25,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha2" - "github.com/openyurtio/openyurt/pkg/controller/platformadmin/config" - "github.com/openyurtio/openyurt/pkg/webhook/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/platformadmin/config" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" ) type Manifest struct { diff --git a/pkg/webhook/platformadmin/v1alpha2/platformadmin_validation.go b/pkg/yurtmanager/webhook/platformadmin/v1alpha2/platformadmin_validation.go similarity index 98% rename from pkg/webhook/platformadmin/v1alpha2/platformadmin_validation.go rename to pkg/yurtmanager/webhook/platformadmin/v1alpha2/platformadmin_validation.go index 170c790244a..8525c13db6d 100644 --- a/pkg/webhook/platformadmin/v1alpha2/platformadmin_validation.go +++ b/pkg/yurtmanager/webhook/platformadmin/v1alpha2/platformadmin_validation.go @@ -28,7 +28,7 @@ import ( unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha2" - util "github.com/openyurtio/openyurt/pkg/controller/platformadmin/utils" + util "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/platformadmin/utils" ) // ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type. diff --git a/pkg/webhook/pod/v1/pod_handler.go b/pkg/yurtmanager/webhook/pod/v1/pod_handler.go similarity index 92% rename from pkg/webhook/pod/v1/pod_handler.go rename to pkg/yurtmanager/webhook/pod/v1/pod_handler.go index 6d5b3cc6291..cd7df17daea 100644 --- a/pkg/webhook/pod/v1/pod_handler.go +++ b/pkg/yurtmanager/webhook/pod/v1/pod_handler.go @@ -22,8 +22,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - "github.com/openyurtio/openyurt/pkg/webhook/builder" - "github.com/openyurtio/openyurt/pkg/webhook/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/builder" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" ) const ( diff --git a/pkg/webhook/pod/v1/pod_validation.go b/pkg/yurtmanager/webhook/pod/v1/pod_validation.go similarity index 98% rename from pkg/webhook/pod/v1/pod_validation.go rename to pkg/yurtmanager/webhook/pod/v1/pod_validation.go index 410e3687688..193bf7d180f 100644 --- a/pkg/webhook/pod/v1/pod_validation.go +++ b/pkg/yurtmanager/webhook/pod/v1/pod_validation.go @@ -31,7 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/controller/yurtcoordinator/constant" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtcoordinator/constant" ) const ( diff --git a/pkg/webhook/server.go b/pkg/yurtmanager/webhook/server.go similarity index 76% rename from pkg/webhook/server.go rename to pkg/yurtmanager/webhook/server.go index b72f5faad57..a57ed20f149 100644 --- a/pkg/webhook/server.go +++ b/pkg/yurtmanager/webhook/server.go @@ -27,24 +27,24 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" - "github.com/openyurtio/openyurt/pkg/controller/nodepool" - "github.com/openyurtio/openyurt/pkg/controller/platformadmin" - "github.com/openyurtio/openyurt/pkg/controller/raven" - ctrlutil "github.com/openyurtio/openyurt/pkg/controller/util" - "github.com/openyurtio/openyurt/pkg/controller/yurtappdaemon" - "github.com/openyurtio/openyurt/pkg/controller/yurtappset" - "github.com/openyurtio/openyurt/pkg/controller/yurtstaticset" - v1beta1gateway "github.com/openyurtio/openyurt/pkg/webhook/gateway/v1beta1" - v1node "github.com/openyurtio/openyurt/pkg/webhook/node/v1" - v1beta1nodepool "github.com/openyurtio/openyurt/pkg/webhook/nodepool/v1beta1" - v1alpha1platformadmin "github.com/openyurtio/openyurt/pkg/webhook/platformadmin/v1alpha1" - v1alpha2platformadmin "github.com/openyurtio/openyurt/pkg/webhook/platformadmin/v1alpha2" - v1pod "github.com/openyurtio/openyurt/pkg/webhook/pod/v1" - "github.com/openyurtio/openyurt/pkg/webhook/util" - webhookcontroller "github.com/openyurtio/openyurt/pkg/webhook/util/controller" - v1alpha1yurtappdaemon "github.com/openyurtio/openyurt/pkg/webhook/yurtappdaemon/v1alpha1" - v1alpha1yurtappset "github.com/openyurtio/openyurt/pkg/webhook/yurtappset/v1alpha1" - v1alpha1yurtstaticset "github.com/openyurtio/openyurt/pkg/webhook/yurtstaticset/v1alpha1" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/nodepool" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/platformadmin" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/raven" + ctrlutil "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappdaemon" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset" + "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtstaticset" + v1beta1gateway "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/gateway/v1beta1" + v1node "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/node/v1" + v1beta1nodepool "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/nodepool/v1beta1" + v1alpha1platformadmin "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/platformadmin/v1alpha1" + v1alpha2platformadmin "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/platformadmin/v1alpha2" + v1pod "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/pod/v1" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" + webhookcontroller "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util/controller" + v1alpha1yurtappdaemon "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/yurtappdaemon/v1alpha1" + v1alpha1yurtappset "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/yurtappset/v1alpha1" + v1alpha1yurtstaticset "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/yurtstaticset/v1alpha1" ) type SetupWebhookWithManager interface { diff --git a/pkg/webhook/util/configuration/configuration.go b/pkg/yurtmanager/webhook/util/configuration/configuration.go similarity index 98% rename from pkg/webhook/util/configuration/configuration.go rename to pkg/yurtmanager/webhook/util/configuration/configuration.go index 4f1b6a20a56..9332939adfc 100644 --- a/pkg/webhook/util/configuration/configuration.go +++ b/pkg/yurtmanager/webhook/util/configuration/configuration.go @@ -28,7 +28,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" - webhookutil "github.com/openyurtio/openyurt/pkg/webhook/util" + webhookutil "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" ) func Ensure(kubeClient clientset.Interface, handlers map[string]struct{}, caBundle []byte, webhookPort int) error { diff --git a/pkg/webhook/util/controller/webhook_controller.go b/pkg/yurtmanager/webhook/util/controller/webhook_controller.go similarity index 97% rename from pkg/webhook/util/controller/webhook_controller.go rename to pkg/yurtmanager/webhook/util/controller/webhook_controller.go index 2734dad20e9..11eed0df2ad 100644 --- a/pkg/webhook/util/controller/webhook_controller.go +++ b/pkg/yurtmanager/webhook/util/controller/webhook_controller.go @@ -43,10 +43,10 @@ import ( "k8s.io/klog/v2" "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" - webhookutil "github.com/openyurtio/openyurt/pkg/webhook/util" - "github.com/openyurtio/openyurt/pkg/webhook/util/configuration" - "github.com/openyurtio/openyurt/pkg/webhook/util/generator" - "github.com/openyurtio/openyurt/pkg/webhook/util/writer" + webhookutil "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util/configuration" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util/generator" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util/writer" ) const ( diff --git a/pkg/webhook/util/generator/certgenerator.go b/pkg/yurtmanager/webhook/util/generator/certgenerator.go similarity index 100% rename from pkg/webhook/util/generator/certgenerator.go rename to pkg/yurtmanager/webhook/util/generator/certgenerator.go diff --git a/pkg/webhook/util/generator/selfsigned.go b/pkg/yurtmanager/webhook/util/generator/selfsigned.go similarity index 100% rename from pkg/webhook/util/generator/selfsigned.go rename to pkg/yurtmanager/webhook/util/generator/selfsigned.go diff --git a/pkg/webhook/util/generator/util.go b/pkg/yurtmanager/webhook/util/generator/util.go similarity index 100% rename from pkg/webhook/util/generator/util.go rename to pkg/yurtmanager/webhook/util/generator/util.go diff --git a/pkg/webhook/util/util.go b/pkg/yurtmanager/webhook/util/util.go similarity index 100% rename from pkg/webhook/util/util.go rename to pkg/yurtmanager/webhook/util/util.go diff --git a/pkg/webhook/util/writer/atomic/atomic_writer.go b/pkg/yurtmanager/webhook/util/writer/atomic/atomic_writer.go similarity index 100% rename from pkg/webhook/util/writer/atomic/atomic_writer.go rename to pkg/yurtmanager/webhook/util/writer/atomic/atomic_writer.go diff --git a/pkg/webhook/util/writer/certwriter.go b/pkg/yurtmanager/webhook/util/writer/certwriter.go similarity index 97% rename from pkg/webhook/util/writer/certwriter.go rename to pkg/yurtmanager/webhook/util/writer/certwriter.go index da094400c71..1795ac57d0d 100644 --- a/pkg/webhook/util/writer/certwriter.go +++ b/pkg/yurtmanager/webhook/util/writer/certwriter.go @@ -22,7 +22,7 @@ import ( "k8s.io/klog/v2" - "github.com/openyurtio/openyurt/pkg/webhook/util/generator" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util/generator" ) const ( diff --git a/pkg/webhook/util/writer/error.go b/pkg/yurtmanager/webhook/util/writer/error.go similarity index 100% rename from pkg/webhook/util/writer/error.go rename to pkg/yurtmanager/webhook/util/writer/error.go diff --git a/pkg/webhook/util/writer/fs.go b/pkg/yurtmanager/webhook/util/writer/fs.go similarity index 97% rename from pkg/webhook/util/writer/fs.go rename to pkg/yurtmanager/webhook/util/writer/fs.go index 6107eb843cd..c5a9f23c5d7 100644 --- a/pkg/webhook/util/writer/fs.go +++ b/pkg/yurtmanager/webhook/util/writer/fs.go @@ -24,8 +24,8 @@ import ( "k8s.io/klog/v2" - "github.com/openyurtio/openyurt/pkg/webhook/util/generator" - "github.com/openyurtio/openyurt/pkg/webhook/util/writer/atomic" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util/generator" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util/writer/atomic" ) const ( diff --git a/pkg/webhook/util/writer/secret.go b/pkg/yurtmanager/webhook/util/writer/secret.go similarity index 98% rename from pkg/webhook/util/writer/secret.go rename to pkg/yurtmanager/webhook/util/writer/secret.go index 15616351fa9..781c8801f3d 100644 --- a/pkg/webhook/util/writer/secret.go +++ b/pkg/yurtmanager/webhook/util/writer/secret.go @@ -27,7 +27,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" - "github.com/openyurtio/openyurt/pkg/webhook/util/generator" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util/generator" ) const ( diff --git a/pkg/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_default.go b/pkg/yurtmanager/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_default.go similarity index 100% rename from pkg/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_default.go rename to pkg/yurtmanager/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_default.go diff --git a/pkg/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_handler.go b/pkg/yurtmanager/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_handler.go similarity index 97% rename from pkg/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_handler.go rename to pkg/yurtmanager/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_handler.go index 457905fe18b..a375e9869dd 100644 --- a/pkg/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_handler.go +++ b/pkg/yurtmanager/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_handler.go @@ -24,7 +24,7 @@ import ( "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/webhook/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" ) // SetupWebhookWithManager sets up Cluster webhooks. diff --git a/pkg/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_validation.go b/pkg/yurtmanager/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_validation.go similarity index 100% rename from pkg/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_validation.go rename to pkg/yurtmanager/webhook/yurtappdaemon/v1alpha1/yurtappdaemon_validation.go diff --git a/pkg/webhook/yurtappset/v1alpha1/validate.go b/pkg/yurtmanager/webhook/yurtappset/v1alpha1/validate.go similarity index 100% rename from pkg/webhook/yurtappset/v1alpha1/validate.go rename to pkg/yurtmanager/webhook/yurtappset/v1alpha1/validate.go diff --git a/pkg/webhook/yurtappset/v1alpha1/yurtappset_default.go b/pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_default.go similarity index 100% rename from pkg/webhook/yurtappset/v1alpha1/yurtappset_default.go rename to pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_default.go diff --git a/pkg/webhook/yurtappset/v1alpha1/yurtappset_handler.go b/pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_handler.go similarity index 97% rename from pkg/webhook/yurtappset/v1alpha1/yurtappset_handler.go rename to pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_handler.go index de90853e81f..9b0efc7085b 100644 --- a/pkg/webhook/yurtappset/v1alpha1/yurtappset_handler.go +++ b/pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_handler.go @@ -23,7 +23,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/webhook/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" ) // SetupWebhookWithManager sets up Cluster webhooks. diff --git a/pkg/webhook/yurtappset/v1alpha1/yurtappset_validation.go b/pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_validation.go similarity index 100% rename from pkg/webhook/yurtappset/v1alpha1/yurtappset_validation.go rename to pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_validation.go diff --git a/pkg/webhook/yurtappset/v1alpha1/yurtappset_webhook_test.go b/pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_webhook_test.go similarity index 100% rename from pkg/webhook/yurtappset/v1alpha1/yurtappset_webhook_test.go rename to pkg/yurtmanager/webhook/yurtappset/v1alpha1/yurtappset_webhook_test.go diff --git a/pkg/webhook/yurtstaticset/v1alpha1/yurtstaticset_default.go b/pkg/yurtmanager/webhook/yurtstaticset/v1alpha1/yurtstaticset_default.go similarity index 100% rename from pkg/webhook/yurtstaticset/v1alpha1/yurtstaticset_default.go rename to pkg/yurtmanager/webhook/yurtstaticset/v1alpha1/yurtstaticset_default.go diff --git a/pkg/webhook/yurtstaticset/v1alpha1/yurtstaticset_handler.go b/pkg/yurtmanager/webhook/yurtstaticset/v1alpha1/yurtstaticset_handler.go similarity index 97% rename from pkg/webhook/yurtstaticset/v1alpha1/yurtstaticset_handler.go rename to pkg/yurtmanager/webhook/yurtstaticset/v1alpha1/yurtstaticset_handler.go index 3ad19bd456e..69a64505a48 100644 --- a/pkg/webhook/yurtstaticset/v1alpha1/yurtstaticset_handler.go +++ b/pkg/yurtmanager/webhook/yurtstaticset/v1alpha1/yurtstaticset_handler.go @@ -22,7 +22,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook" "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" - "github.com/openyurtio/openyurt/pkg/webhook/util" + "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util" ) // SetupWebhookWithManager sets up Cluster webhooks. mutate path, validatepath, error diff --git a/pkg/webhook/yurtstaticset/v1alpha1/yurtstaticset_validation.go b/pkg/yurtmanager/webhook/yurtstaticset/v1alpha1/yurtstaticset_validation.go similarity index 100% rename from pkg/webhook/yurtstaticset/v1alpha1/yurtstaticset_validation.go rename to pkg/yurtmanager/webhook/yurtstaticset/v1alpha1/yurtstaticset_validation.go diff --git a/pkg/yurttunnel/util/util.go b/pkg/yurttunnel/util/util.go index f5adb8d7a03..b812c15bc66 100644 --- a/pkg/yurttunnel/util/util.go +++ b/pkg/yurttunnel/util/util.go @@ -32,8 +32,8 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" - "github.com/openyurtio/openyurt/pkg/profile" "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/openyurtio/openyurt/pkg/util/profile" ) const ( From efe75d58961ee60b33e822c3fbb8786a415d7718 Mon Sep 17 00:00:00 2001 From: rambohe Date: Fri, 18 Aug 2023 09:29:23 +0800 Subject: [PATCH 20/21] improve labels and annotations definition (#1663) Signed-off-by: rambohe-ch --- .../v1alpha1/well_known_labels_annotations.go | 64 ------------------- pkg/apis/apps/v1beta1/nodepool_types.go | 2 - .../apps/well_known_labels_annotations.go | 29 ++++----- pkg/util/json.go | 50 --------------- .../filter/nodeportisolation/filter_test.go | 2 +- .../platformadmin/platformadmin_controller.go | 5 +- .../nodepool_enqueue_handlers_test.go | 11 ++-- .../controller/yurtappdaemon/revision_test.go | 3 +- .../deployment_controller.go | 13 ++-- .../deployment_controller_test.go | 3 +- .../yurtappdaemon/workloadcontroller/util.go | 4 +- .../workloadcontroller/util_test.go | 4 +- .../workloadcontroller/workload.go | 2 +- .../workloadcontroller/workload_test.go | 2 +- .../yurtappdaemon_controller_test.go | 5 +- .../yurtappset/adapter/adapter_util.go | 9 +-- .../yurtappset/adapter/adapter_util_test.go | 2 +- .../yurtappset/adapter/deployment_adapter.go | 13 ++-- .../adapter/deployment_adapter_test.go | 21 +++--- .../yurtappset/adapter/statefulset_adapter.go | 13 ++-- .../adapter/statefulset_adapter_test.go | 21 +++--- .../controller/yurtappset/pool_control.go | 3 +- .../yurtappset/yurtappset_controller_utils.go | 7 +- .../webhook/pod/v1/pod_validation.go | 6 +- test/e2e/util/nodepool.go | 6 +- 25 files changed, 95 insertions(+), 205 deletions(-) delete mode 100644 pkg/apis/apps/v1alpha1/well_known_labels_annotations.go delete mode 100644 pkg/util/json.go diff --git a/pkg/apis/apps/v1alpha1/well_known_labels_annotations.go b/pkg/apis/apps/v1alpha1/well_known_labels_annotations.go deleted file mode 100644 index da0fe354b1c..00000000000 --- a/pkg/apis/apps/v1alpha1/well_known_labels_annotations.go +++ /dev/null @@ -1,64 +0,0 @@ -/* -Copyright 2021 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -@CHANGELOG -OpenYurt Authors: -change some const value -*/ - -package v1alpha1 - -// YurtAppSet & YurtAppDaemon related labels and annotations -const ( - // ControllerRevisionHashLabelKey is used to record the controller revision of current resource. - ControllerRevisionHashLabelKey = "apps.openyurt.io/controller-revision-hash" - - // PoolNameLabelKey is used to record the name of current pool. - PoolNameLabelKey = "apps.openyurt.io/pool-name" - - // SpecifiedDeleteKey indicates this object should be deleted, and the value could be the deletion option. - SpecifiedDeleteKey = "apps.openyurt.io/specified-delete" - - // AnnotationPatchKey indicates the patch for every sub pool - AnnotationPatchKey = "apps.openyurt.io/patch" - - AnnotationRefNodePool = "apps.openyurt.io/ref-nodepool" -) - -// NodePool related labels and annotations -const ( - // LabelDesiredNodePool indicates which nodepool the node want to join - LabelDesiredNodePool = "apps.openyurt.io/desired-nodepool" - - // LabelCurrentNodePool indicates which nodepool the node is currently - // belonging to - LabelCurrentNodePool = "apps.openyurt.io/nodepool" - - // LabelCurrentYurtAppDaemon indicates which service the yurtappdaemon is currently - // belonging to - LabelCurrentYurtAppDaemon = "apps.openyurt.io/yurtappdaemon" - - AnnotationPrevAttrs = "nodepool.openyurt.io/previous-attributes" - - // DefaultCloudNodePoolName defines the name of the default cloud nodepool - DefaultCloudNodePoolName = "default-nodepool" - - // DefaultEdgeNodePoolName defines the name of the default edge nodepool - DefaultEdgeNodePoolName = "default-edge-nodepool" - - // ServiceTopologyKey is the toplogy key that will be attached to node, - // the value will be the name of the nodepool - ServiceTopologyKey = "topology.kubernetes.io/zone" -) diff --git a/pkg/apis/apps/v1beta1/nodepool_types.go b/pkg/apis/apps/v1beta1/nodepool_types.go index 60ee2a08b6c..3a28b1d8f89 100644 --- a/pkg/apis/apps/v1beta1/nodepool_types.go +++ b/pkg/apis/apps/v1beta1/nodepool_types.go @@ -26,8 +26,6 @@ type NodePoolType string const ( Edge NodePoolType = "Edge" Cloud NodePoolType = "Cloud" - - NodePoolTypeLabelKey = "openyurt.io/node-pool-type" ) // NodePoolSpec defines the desired state of NodePool diff --git a/pkg/apis/apps/well_known_labels_annotations.go b/pkg/apis/apps/well_known_labels_annotations.go index 56f7abb77e4..619babbca07 100644 --- a/pkg/apis/apps/well_known_labels_annotations.go +++ b/pkg/apis/apps/well_known_labels_annotations.go @@ -1,21 +1,26 @@ /* -Copyright 2023 The OpenYurt Authors. +Copyright 2021 The OpenYurt Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +@CHANGELOG +OpenYurt Authors: +change some const value */ package apps +// YurtAppSet & YurtAppDaemon related labels and annotations const ( // ControllerRevisionHashLabelKey is used to record the controller revision of current resource. ControllerRevisionHashLabelKey = "apps.openyurt.io/controller-revision-hash" @@ -23,29 +28,17 @@ const ( // PoolNameLabelKey is used to record the name of current pool. PoolNameLabelKey = "apps.openyurt.io/pool-name" - // SpecifiedDeleteKey indicates this object should be deleted, and the value could be the deletion option. - SpecifiedDeleteKey = "apps.openyurt.io/specified-delete" - // AnnotationPatchKey indicates the patch for every sub pool AnnotationPatchKey = "apps.openyurt.io/patch" + + AnnotationRefNodePool = "apps.openyurt.io/ref-nodepool" ) // NodePool related labels and annotations const ( - NodePoolTypeLabelKey = "openyurt.io/node-pool-type" - - // LabelDesiredNodePool indicates which nodepool the node want to join - LabelDesiredNodePool = "apps.openyurt.io/desired-nodepool" - - // LabelCurrentNodePool indicates which nodepool the node is currently - // belonging to - LabelCurrentNodePool = "apps.openyurt.io/nodepool" - - AnnotationPrevAttrs = "nodepool.openyurt.io/previous-attributes" - + AnnotationPrevAttrs = "nodepool.openyurt.io/previous-attributes" NodePoolLabel = "apps.openyurt.io/nodepool" NodePoolTypeLabel = "nodepool.openyurt.io/type" NodePoolHostNetworkLabel = "nodepool.openyurt.io/hostnetwork" - - NodePoolChangedEvent = "NodePoolChanged" + NodePoolChangedEvent = "NodePoolChanged" ) diff --git a/pkg/util/json.go b/pkg/util/json.go deleted file mode 100644 index e2e91d8c69e..00000000000 --- a/pkg/util/json.go +++ /dev/null @@ -1,50 +0,0 @@ -/* -Copyright 2023 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "encoding/json" - "reflect" -) - -// DumpJSON returns the JSON encoding -func DumpJSON(o interface{}) string { - j, _ := json.Marshal(o) - return string(j) -} - -// IsJSONObjectEqual checks if two objects are equal after encoding json -func IsJSONObjectEqual(o1, o2 interface{}) bool { - if reflect.DeepEqual(o1, o2) { - return true - } - - oj1, _ := json.Marshal(o1) - oj2, _ := json.Marshal(o2) - os1 := string(oj1) - os2 := string(oj2) - if os1 == os2 { - return true - } - - om1 := make(map[string]interface{}) - om2 := make(map[string]interface{}) - _ = json.Unmarshal(oj1, &om1) - _ = json.Unmarshal(oj2, &om2) - - return reflect.DeepEqual(om1, om2) -} diff --git a/pkg/yurthub/filter/nodeportisolation/filter_test.go b/pkg/yurthub/filter/nodeportisolation/filter_test.go index 8ff333256ac..c00279050b9 100644 --- a/pkg/yurthub/filter/nodeportisolation/filter_test.go +++ b/pkg/yurthub/filter/nodeportisolation/filter_test.go @@ -92,7 +92,7 @@ func TestFilter(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "foo", Annotations: map[string]string{ - apps.LabelDesiredNodePool: nodePoolName, + apps.NodePoolLabel: nodePoolName, }, }, } diff --git a/pkg/yurtmanager/controller/platformadmin/platformadmin_controller.go b/pkg/yurtmanager/controller/platformadmin/platformadmin_controller.go index 2ee00160579..c75a161d878 100644 --- a/pkg/yurtmanager/controller/platformadmin/platformadmin_controller.go +++ b/pkg/yurtmanager/controller/platformadmin/platformadmin_controller.go @@ -44,6 +44,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" appconfig "github.com/openyurtio/openyurt/cmd/yurt-manager/app/config" + "github.com/openyurtio/openyurt/pkg/apis/apps" appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" iotv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha1" iotv1alpha2 "github.com/openyurtio/openyurt/pkg/apis/iot/v1alpha2" @@ -443,7 +444,7 @@ func (r *ReconcilePlatformAdmin) reconcileComponent(ctx context.Context, platfor } pool.NodeSelectorTerm.MatchExpressions = append(pool.NodeSelectorTerm.MatchExpressions, corev1.NodeSelectorRequirement{ - Key: appsv1alpha1.LabelCurrentNodePool, + Key: apps.NodePoolLabel, Operator: corev1.NodeSelectorOpIn, Values: []string{platformAdmin.Spec.PoolName}, }) @@ -561,7 +562,7 @@ func (r *ReconcilePlatformAdmin) handleYurtAppSet(ctx context.Context, platformA } pool.NodeSelectorTerm.MatchExpressions = append(pool.NodeSelectorTerm.MatchExpressions, corev1.NodeSelectorRequirement{ - Key: appsv1alpha1.LabelCurrentNodePool, + Key: apps.NodePoolLabel, Operator: corev1.NodeSelectorOpIn, Values: []string{platformAdmin.Spec.PoolName}, }) diff --git a/pkg/yurtmanager/controller/yurtappdaemon/nodepool_enqueue_handlers_test.go b/pkg/yurtmanager/controller/yurtappdaemon/nodepool_enqueue_handlers_test.go index 07c97717179..0eb99d4c9e1 100644 --- a/pkg/yurtmanager/controller/yurtappdaemon/nodepool_enqueue_handlers_test.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/nodepool_enqueue_handlers_test.go @@ -27,6 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/event" + "github.com/openyurtio/openyurt/pkg/apis/apps" appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" ) @@ -91,7 +92,7 @@ func TestCreate(t *testing.T) { Namespace: "kube-system", Name: "kube-proxy", Annotations: map[string]string{ - appsv1alpha1.AnnotationRefNodePool: "a", + apps.AnnotationRefNodePool: "a", }, }, Spec: v1.PodSpec{}, @@ -144,7 +145,7 @@ func TestUpdate(t *testing.T) { Namespace: "kube-system", Name: "kube-proxy", Annotations: map[string]string{ - appsv1alpha1.AnnotationRefNodePool: "a", + apps.AnnotationRefNodePool: "a", }, }, Spec: v1.PodSpec{}, @@ -154,7 +155,7 @@ func TestUpdate(t *testing.T) { Namespace: "kube-system", Name: "kube-proxy", Annotations: map[string]string{ - appsv1alpha1.AnnotationRefNodePool: "a", + apps.AnnotationRefNodePool: "a", }, }, Spec: v1.PodSpec{}, @@ -206,7 +207,7 @@ func TestDelete(t *testing.T) { Namespace: "kube-system", Name: "kube-proxy", Annotations: map[string]string{ - appsv1alpha1.AnnotationRefNodePool: "a", + apps.AnnotationRefNodePool: "a", }, }, Spec: v1.PodSpec{}, @@ -260,7 +261,7 @@ func TestGeneric(t *testing.T) { Namespace: "kube-system", Name: "kube-proxy", Annotations: map[string]string{ - appsv1alpha1.AnnotationRefNodePool: "a", + apps.AnnotationRefNodePool: "a", }, }, Spec: v1.PodSpec{}, diff --git a/pkg/yurtmanager/controller/yurtappdaemon/revision_test.go b/pkg/yurtmanager/controller/yurtappdaemon/revision_test.go index 1681ec919f6..aaa0e2163ff 100644 --- a/pkg/yurtmanager/controller/yurtappdaemon/revision_test.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/revision_test.go @@ -27,6 +27,7 @@ import ( "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client/fake" + yurtapps "github.com/openyurtio/openyurt/pkg/apis/apps" alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" ) @@ -81,7 +82,7 @@ func TestNewRevision(t *testing.T) { }, Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - alpha1.PoolNameLabelKey: "a", + yurtapps.PoolNameLabelKey: "a", }, }, }, diff --git a/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/deployment_controller.go b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/deployment_controller.go index e2985991a4d..a70ad98f03b 100644 --- a/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/deployment_controller.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/deployment_controller.go @@ -28,6 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "github.com/openyurtio/openyurt/pkg/apis/apps" "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/refmanager" ) @@ -67,8 +68,8 @@ func (d *DeploymentControllor) applyTemplate(scheme *runtime.Scheme, yad *v1alph for k, v := range yad.Spec.Selector.MatchLabels { set.Labels[k] = v } - set.Labels[v1alpha1.ControllerRevisionHashLabelKey] = revision - set.Labels[v1alpha1.PoolNameLabelKey] = nodepool.GetName() + set.Labels[apps.ControllerRevisionHashLabelKey] = revision + set.Labels[apps.PoolNameLabelKey] = nodepool.GetName() if set.Annotations == nil { set.Annotations = map[string]string{} @@ -76,13 +77,13 @@ func (d *DeploymentControllor) applyTemplate(scheme *runtime.Scheme, yad *v1alph for k, v := range yad.Spec.WorkloadTemplate.DeploymentTemplate.Annotations { set.Annotations[k] = v } - set.Annotations[v1alpha1.AnnotationRefNodePool] = nodepool.GetName() + set.Annotations[apps.AnnotationRefNodePool] = nodepool.GetName() set.Namespace = yad.GetNamespace() set.GenerateName = getWorkloadPrefix(yad.GetName(), nodepool.GetName()) set.Spec = *yad.Spec.WorkloadTemplate.DeploymentTemplate.Spec.DeepCopy() - set.Spec.Selector.MatchLabels[v1alpha1.PoolNameLabelKey] = nodepool.GetName() + set.Spec.Selector.MatchLabels[apps.PoolNameLabelKey] = nodepool.GetName() // set RequiredDuringSchedulingIgnoredDuringExecution nil if set.Spec.Template.Spec.Affinity != nil && set.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil { @@ -92,8 +93,8 @@ func (d *DeploymentControllor) applyTemplate(scheme *runtime.Scheme, yad *v1alph if set.Spec.Template.Labels == nil { set.Spec.Template.Labels = map[string]string{} } - set.Spec.Template.Labels[v1alpha1.PoolNameLabelKey] = nodepool.GetName() - set.Spec.Template.Labels[v1alpha1.ControllerRevisionHashLabelKey] = revision + set.Spec.Template.Labels[apps.PoolNameLabelKey] = nodepool.GetName() + set.Spec.Template.Labels[apps.ControllerRevisionHashLabelKey] = revision // use nodeSelector set.Spec.Template.Spec.NodeSelector = CreateNodeSelectorByNodepoolName(nodepool.GetName()) diff --git a/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/deployment_controller_test.go b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/deployment_controller_test.go index af3dbd67176..6077dc82ad3 100644 --- a/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/deployment_controller_test.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/deployment_controller_test.go @@ -29,6 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" fakeclint "sigs.k8s.io/controller-runtime/pkg/client/fake" + "github.com/openyurtio/openyurt/pkg/apis/apps" "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" ) @@ -129,7 +130,7 @@ func TestApplyTemplate(t *testing.T) { }, Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - v1alpha1.PoolNameLabelKey: "a", + apps.PoolNameLabelKey: "a", }, }, }, diff --git a/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/util.go b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/util.go index e76a9d01e3d..eb0bedd6948 100644 --- a/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/util.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/util.go @@ -22,7 +22,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/validation" - "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/pkg/apis/apps" ) func getWorkloadPrefix(controllerName, nodepoolName string) string { @@ -35,7 +35,7 @@ func getWorkloadPrefix(controllerName, nodepoolName string) string { func CreateNodeSelectorByNodepoolName(nodepool string) map[string]string { return map[string]string{ - v1alpha1.LabelCurrentNodePool: nodepool, + apps.NodePoolLabel: nodepool, } } diff --git a/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/util_test.go b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/util_test.go index f5591ea56ba..37ae917405d 100644 --- a/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/util_test.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/util_test.go @@ -22,7 +22,7 @@ import ( corev1 "k8s.io/api/core/v1" - "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + "github.com/openyurtio/openyurt/pkg/apis/apps" ) const ( @@ -70,7 +70,7 @@ func TestCreateNodeSelectorByNodepoolName(t *testing.T) { "normal", "a", map[string]string{ - v1alpha1.LabelCurrentNodePool: "a", + apps.NodePoolLabel: "a", }, }, } diff --git a/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/workload.go b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/workload.go index ff50f1f589e..0270f9f0bb0 100644 --- a/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/workload.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/workload.go @@ -20,7 +20,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps" ) type Workload struct { diff --git a/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/workload_test.go b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/workload_test.go index 9928707e662..d6e02df6819 100644 --- a/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/workload_test.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller/workload_test.go @@ -23,7 +23,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps" ) func TestGetRevision(t *testing.T) { diff --git a/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller_test.go b/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller_test.go index 9a668ac6963..bfdf9091125 100644 --- a/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller_test.go +++ b/pkg/yurtmanager/controller/yurtappdaemon/yurtappdaemon_controller_test.go @@ -25,6 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "github.com/openyurtio/openyurt/pkg/apis/apps" unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappdaemon/workloadcontroller" ) @@ -507,7 +508,7 @@ func TestGetTemplateControls(t *testing.T) { }, Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - unitv1alpha1.PoolNameLabelKey: "a", + apps.PoolNameLabelKey: "a", }, }, }, @@ -550,7 +551,7 @@ func TestGetTemplateControls(t *testing.T) { }, Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - unitv1alpha1.PoolNameLabelKey: "a", + apps.PoolNameLabelKey: "a", }, }, }, diff --git a/pkg/yurtmanager/controller/yurtappset/adapter/adapter_util.go b/pkg/yurtmanager/controller/yurtappset/adapter/adapter_util.go index 5808a0e9825..5b51077965f 100644 --- a/pkg/yurtmanager/controller/yurtappset/adapter/adapter_util.go +++ b/pkg/yurtmanager/controller/yurtappset/adapter/adapter_util.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/klog/v2" + "github.com/openyurtio/openyurt/pkg/apis/apps" appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" ) @@ -94,7 +95,7 @@ func getRevision(objMeta metav1.Object) string { if objMeta.GetLabels() == nil { return "" } - return objMeta.GetLabels()[appsv1alpha1.ControllerRevisionHashLabelKey] + return objMeta.GetLabels()[apps.ControllerRevisionHashLabelKey] } // getCurrentPartition calculates current partition by counting the pods not having the updated revision @@ -139,7 +140,7 @@ func PoolHasPatch(poolConfig *appsv1alpha1.Pool, set metav1.Object) bool { if poolConfig.Patch == nil { // If No Patches, Must Set patches annotation to "" if anno := set.GetAnnotations(); anno != nil { - anno[appsv1alpha1.AnnotationPatchKey] = "" + anno[apps.AnnotationPatchKey] = "" } return false } @@ -154,10 +155,10 @@ func CreateNewPatchedObject(patchInfo *runtime.RawExtension, set metav1.Object, if anno := newPatched.GetAnnotations(); anno == nil { newPatched.SetAnnotations(map[string]string{ - appsv1alpha1.AnnotationPatchKey: string(patchInfo.Raw), + apps.AnnotationPatchKey: string(patchInfo.Raw), }) } else { - anno[appsv1alpha1.AnnotationPatchKey] = string(patchInfo.Raw) + anno[apps.AnnotationPatchKey] = string(patchInfo.Raw) } return nil } diff --git a/pkg/yurtmanager/controller/yurtappset/adapter/adapter_util_test.go b/pkg/yurtmanager/controller/yurtappset/adapter/adapter_util_test.go index 4227f4e8225..614ed00ebe3 100644 --- a/pkg/yurtmanager/controller/yurtappset/adapter/adapter_util_test.go +++ b/pkg/yurtmanager/controller/yurtappset/adapter/adapter_util_test.go @@ -26,7 +26,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" + unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps" ) func TestGetCurrentPartitionForStrategyOnDelete(t *testing.T) { diff --git a/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter.go b/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter.go index 143ba979c03..72b7971b6cb 100644 --- a/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter.go +++ b/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter.go @@ -26,6 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "github.com/openyurtio/openyurt/pkg/apis/apps" alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" ) @@ -100,9 +101,9 @@ func (a *DeploymentAdapter) ApplyPoolTemplate(yas *alpha1.YurtAppSet, poolName, for k, v := range yas.Spec.Selector.MatchLabels { set.Labels[k] = v } - set.Labels[alpha1.ControllerRevisionHashLabelKey] = revision + set.Labels[apps.ControllerRevisionHashLabelKey] = revision // record the pool name as a label - set.Labels[alpha1.PoolNameLabelKey] = poolName + set.Labels[apps.PoolNameLabelKey] = poolName if set.Annotations == nil { set.Annotations = map[string]string{} @@ -114,7 +115,7 @@ func (a *DeploymentAdapter) ApplyPoolTemplate(yas *alpha1.YurtAppSet, poolName, set.GenerateName = getPoolPrefix(yas.Name, poolName) selectors := yas.Spec.Selector.DeepCopy() - selectors.MatchLabels[alpha1.PoolNameLabelKey] = poolName + selectors.MatchLabels[apps.PoolNameLabelKey] = poolName if err := controllerutil.SetControllerReference(yas, set, a.Scheme); err != nil { return err @@ -128,8 +129,8 @@ func (a *DeploymentAdapter) ApplyPoolTemplate(yas *alpha1.YurtAppSet, poolName, if set.Spec.Template.Labels == nil { set.Spec.Template.Labels = map[string]string{} } - set.Spec.Template.Labels[alpha1.PoolNameLabelKey] = poolName - set.Spec.Template.Labels[alpha1.ControllerRevisionHashLabelKey] = revision + set.Spec.Template.Labels[apps.PoolNameLabelKey] = poolName + set.Spec.Template.Labels[apps.ControllerRevisionHashLabelKey] = revision set.Spec.RevisionHistoryLimit = yas.Spec.RevisionHistoryLimit set.Spec.MinReadySeconds = yas.Spec.WorkloadTemplate.DeploymentTemplate.Spec.MinReadySeconds @@ -166,5 +167,5 @@ func (a *DeploymentAdapter) PostUpdate(yas *alpha1.YurtAppSet, obj runtime.Objec // IsExpected checks the pool is the expected revision or not. // The revision label can tell the current pool revision. func (a *DeploymentAdapter) IsExpected(obj metav1.Object, revision string) bool { - return obj.GetLabels()[alpha1.ControllerRevisionHashLabelKey] != revision + return obj.GetLabels()[apps.ControllerRevisionHashLabelKey] != revision } diff --git a/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter_test.go b/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter_test.go index b61c0af0f9e..c0f41234cc7 100644 --- a/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter_test.go +++ b/pkg/yurtmanager/controller/yurtappset/adapter/deployment_adapter_test.go @@ -27,6 +27,7 @@ import ( fakeclint "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "github.com/openyurtio/openyurt/pkg/apis/apps" appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" ) @@ -59,7 +60,7 @@ func TestDeploymentAdapter_ApplyPoolTemplate(t *testing.T) { DeploymentTemplate: &appsv1alpha1.DeploymentTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - appsv1alpha1.AnnotationPatchKey: "annotation-v", + apps.AnnotationPatchKey: "annotation-v", }, Labels: map[string]string{ "name": "foo", @@ -113,29 +114,29 @@ func TestDeploymentAdapter_ApplyPoolTemplate(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Labels: map[string]string{ - "name": "foo", - appsv1alpha1.ControllerRevisionHashLabelKey: "1", - appsv1alpha1.PoolNameLabelKey: "hangzhou", + "name": "foo", + apps.ControllerRevisionHashLabelKey: "1", + apps.PoolNameLabelKey: "hangzhou", }, Annotations: map[string]string{ - appsv1alpha1.AnnotationPatchKey: "", + apps.AnnotationPatchKey: "", }, GenerateName: "foo-hangzhou-", }, Spec: appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "name": "foo", - appsv1alpha1.PoolNameLabelKey: "hangzhou", + "name": "foo", + apps.PoolNameLabelKey: "hangzhou", }, }, Replicas: &one, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - "name": "foo", - appsv1alpha1.ControllerRevisionHashLabelKey: "1", - appsv1alpha1.PoolNameLabelKey: "hangzhou", + "name": "foo", + apps.ControllerRevisionHashLabelKey: "1", + apps.PoolNameLabelKey: "hangzhou", }, }, Spec: corev1.PodSpec{ diff --git a/pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter.go b/pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter.go index d9baf2f6566..be8a2710d4c 100644 --- a/pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter.go +++ b/pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter.go @@ -31,6 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "github.com/openyurtio/openyurt/pkg/apis/apps" alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" ) @@ -106,9 +107,9 @@ func (a *StatefulSetAdapter) ApplyPoolTemplate(yas *alpha1.YurtAppSet, poolName, for k, v := range yas.Spec.Selector.MatchLabels { set.Labels[k] = v } - set.Labels[alpha1.ControllerRevisionHashLabelKey] = revision + set.Labels[apps.ControllerRevisionHashLabelKey] = revision // record the pool name as a label - set.Labels[alpha1.PoolNameLabelKey] = poolName + set.Labels[apps.PoolNameLabelKey] = poolName if set.Annotations == nil { set.Annotations = map[string]string{} @@ -120,7 +121,7 @@ func (a *StatefulSetAdapter) ApplyPoolTemplate(yas *alpha1.YurtAppSet, poolName, set.GenerateName = getPoolPrefix(yas.Name, poolName) selectors := yas.Spec.Selector.DeepCopy() - selectors.MatchLabels[alpha1.PoolNameLabelKey] = poolName + selectors.MatchLabels[apps.PoolNameLabelKey] = poolName if err := controllerutil.SetControllerReference(yas, set, a.Scheme); err != nil { return err @@ -134,8 +135,8 @@ func (a *StatefulSetAdapter) ApplyPoolTemplate(yas *alpha1.YurtAppSet, poolName, if set.Spec.Template.Labels == nil { set.Spec.Template.Labels = map[string]string{} } - set.Spec.Template.Labels[alpha1.PoolNameLabelKey] = poolName - set.Spec.Template.Labels[alpha1.ControllerRevisionHashLabelKey] = revision + set.Spec.Template.Labels[apps.PoolNameLabelKey] = poolName + set.Spec.Template.Labels[apps.ControllerRevisionHashLabelKey] = revision set.Spec.RevisionHistoryLimit = yas.Spec.RevisionHistoryLimit set.Spec.PodManagementPolicy = yas.Spec.WorkloadTemplate.StatefulSetTemplate.Spec.PodManagementPolicy @@ -182,7 +183,7 @@ func (a *StatefulSetAdapter) PostUpdate(yas *alpha1.YurtAppSet, obj runtime.Obje // IsExpected checks the pool is the expected revision or not. // The revision label can tell the current pool revision. func (a *StatefulSetAdapter) IsExpected(obj metav1.Object, revision string) bool { - return obj.GetLabels()[alpha1.ControllerRevisionHashLabelKey] != revision + return obj.GetLabels()[apps.ControllerRevisionHashLabelKey] != revision } /* diff --git a/pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter_test.go b/pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter_test.go index de296193fa9..2eb70775a33 100644 --- a/pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter_test.go +++ b/pkg/yurtmanager/controller/yurtappset/adapter/statefulset_adapter_test.go @@ -27,6 +27,7 @@ import ( fakeclint "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "github.com/openyurtio/openyurt/pkg/apis/apps" appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" ) @@ -58,7 +59,7 @@ func TestStatefulSetAdapter_ApplyPoolTemplate(t *testing.T) { StatefulSetTemplate: &appsv1alpha1.StatefulSetTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - appsv1alpha1.AnnotationPatchKey: "annotation-v", + apps.AnnotationPatchKey: "annotation-v", }, Labels: map[string]string{ "name": "foo", @@ -112,29 +113,29 @@ func TestStatefulSetAdapter_ApplyPoolTemplate(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Labels: map[string]string{ - "name": "foo", - appsv1alpha1.ControllerRevisionHashLabelKey: "1", - appsv1alpha1.PoolNameLabelKey: "hangzhou", + "name": "foo", + apps.ControllerRevisionHashLabelKey: "1", + apps.PoolNameLabelKey: "hangzhou", }, Annotations: map[string]string{ - appsv1alpha1.AnnotationPatchKey: "", + apps.AnnotationPatchKey: "", }, GenerateName: "foo-hangzhou-", }, Spec: appsv1.StatefulSetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "name": "foo", - appsv1alpha1.PoolNameLabelKey: "hangzhou", + "name": "foo", + apps.PoolNameLabelKey: "hangzhou", }, }, Replicas: &one, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - "name": "foo", - appsv1alpha1.ControllerRevisionHashLabelKey: "1", - appsv1alpha1.PoolNameLabelKey: "hangzhou", + "name": "foo", + apps.ControllerRevisionHashLabelKey: "1", + apps.PoolNameLabelKey: "hangzhou", }, }, Spec: corev1.PodSpec{ diff --git a/pkg/yurtmanager/controller/yurtappset/pool_control.go b/pkg/yurtmanager/controller/yurtappset/pool_control.go index 34791df33bf..45f07095fb7 100644 --- a/pkg/yurtmanager/controller/yurtappset/pool_control.go +++ b/pkg/yurtmanager/controller/yurtappset/pool_control.go @@ -28,6 +28,7 @@ import ( "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/openyurtio/openyurt/pkg/apis/apps" alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/util/refmanager" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtappset/adapter" @@ -168,7 +169,7 @@ func (m *PoolControl) convertToPool(set metav1.Object) (*Pool, error) { ReplicasInfo: specReplicas, }, } - if data, ok := set.GetAnnotations()[alpha1.AnnotationPatchKey]; ok { + if data, ok := set.GetAnnotations()[apps.AnnotationPatchKey]; ok { pool.Status.PatchInfo = data } return pool, nil diff --git a/pkg/yurtmanager/controller/yurtappset/yurtappset_controller_utils.go b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller_utils.go index b96bbffddf4..3c7d22a560c 100644 --- a/pkg/yurtmanager/controller/yurtappset/yurtappset_controller_utils.go +++ b/pkg/yurtmanager/controller/yurtappset/yurtappset_controller_utils.go @@ -28,6 +28,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/openyurtio/openyurt/pkg/apis/apps" unitv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" ) @@ -39,13 +40,13 @@ type YurtAppSetPatches struct { } func getPoolNameFrom(metaObj metav1.Object) (string, error) { - name, exist := metaObj.GetLabels()[unitv1alpha1.PoolNameLabelKey] + name, exist := metaObj.GetLabels()[apps.PoolNameLabelKey] if !exist { - return "", fmt.Errorf("fail to get pool name from label of pool %s/%s: no label %s found", metaObj.GetNamespace(), metaObj.GetName(), unitv1alpha1.PoolNameLabelKey) + return "", fmt.Errorf("fail to get pool name from label of pool %s/%s: no label %s found", metaObj.GetNamespace(), metaObj.GetName(), apps.PoolNameLabelKey) } if len(name) == 0 { - return "", fmt.Errorf("fail to get pool name from label of pool %s/%s: label %s has an empty value", metaObj.GetNamespace(), metaObj.GetName(), unitv1alpha1.PoolNameLabelKey) + return "", fmt.Errorf("fail to get pool name from label of pool %s/%s: label %s has an empty value", metaObj.GetNamespace(), metaObj.GetName(), apps.PoolNameLabelKey) } return name, nil diff --git a/pkg/yurtmanager/webhook/pod/v1/pod_validation.go b/pkg/yurtmanager/webhook/pod/v1/pod_validation.go index 193bf7d180f..249429d12e5 100644 --- a/pkg/yurtmanager/webhook/pod/v1/pod_validation.go +++ b/pkg/yurtmanager/webhook/pod/v1/pod_validation.go @@ -30,13 +30,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + "github.com/openyurtio/openyurt/pkg/apis/apps" appsv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/apps/v1alpha1" "github.com/openyurtio/openyurt/pkg/yurtmanager/controller/yurtcoordinator/constant" ) const ( - LabelCurrentNodePool = "apps.openyurt.io/nodepool" - UserNodeController = "system:serviceaccount:kube-system:node-controller" + UserNodeController = "system:serviceaccount:kube-system:node-controller" NodeLeaseDurationSeconds = 40 DefaultPoolReadyNodeNumberRatioThreshold = 0.35 @@ -83,7 +83,7 @@ func validatePodDeletion(cli client.Client, pod *v1.Pod, req admission.Request) // only validate pod which in nodePool var nodePoolName string if node.Labels != nil { - if name, ok := node.Labels[LabelCurrentNodePool]; ok { + if name, ok := node.Labels[apps.NodePoolLabel]; ok { nodePoolName = name } } diff --git a/test/e2e/util/nodepool.go b/test/e2e/util/nodepool.go index 7afd6cb3173..3c3bfd1c789 100644 --- a/test/e2e/util/nodepool.go +++ b/test/e2e/util/nodepool.go @@ -52,8 +52,8 @@ func CleanupNodePoolLabel(ctx context.Context, k8sClient client.Client) error { newNode := originNode.DeepCopy() if newNode.Labels != nil { for k := range newNode.Labels { - if k == apps.LabelDesiredNodePool { - delete(newNode.Labels, apps.LabelDesiredNodePool) + if k == apps.NodePoolLabel { + delete(newNode.Labels, apps.NodePoolLabel) labelDeleted = true } } @@ -103,7 +103,7 @@ func InitNodeAndNodePool(ctx context.Context, k8sClient client.Client, poolToNod continue } - nodeLabels[apps.LabelDesiredNodePool] = nodeToPoolMap[originNode.Name] + nodeLabels[apps.NodePoolLabel] = nodeToPoolMap[originNode.Name] newNode.Labels = nodeLabels if err := k8sClient.Patch(ctx, newNode, client.MergeFrom(&originNode)); err != nil { return err From d15078f7bc19857e87952ba418fb1dd71d191bfa Mon Sep 17 00:00:00 2001 From: Zhen Zhao <413621396@qq.com> Date: Fri, 18 Aug 2023 11:05:24 +0800 Subject: [PATCH 21/21] fix node-servant convert not use yss template to deploy yurthub (#1633) * fix node-servant convert not use yss template to deploy yurthub * update --- pkg/node-servant/components/yurthub.go | 37 ++++--- pkg/node-servant/constant.go | 6 + test/e2e/cmd/init/constants/constants.go | 133 +++++++++++++++++++++++ test/e2e/cmd/init/converter.go | 43 +++++++- 4 files changed, 199 insertions(+), 20 deletions(-) diff --git a/pkg/node-servant/components/yurthub.go b/pkg/node-servant/components/yurthub.go index 94ff7957b59..ee20b5ed12e 100644 --- a/pkg/node-servant/components/yurthub.go +++ b/pkg/node-servant/components/yurthub.go @@ -23,7 +23,6 @@ import ( "net/url" "os" "path/filepath" - "strconv" "strings" "time" @@ -34,7 +33,7 @@ import ( "github.com/openyurtio/openyurt/pkg/projectinfo" kubeconfigutil "github.com/openyurtio/openyurt/pkg/util/kubeconfig" - "github.com/openyurtio/openyurt/pkg/util/templates" + tmplutil "github.com/openyurtio/openyurt/pkg/util/templates" "github.com/openyurtio/openyurt/pkg/yurtadm/constants" enutil "github.com/openyurtio/openyurt/pkg/yurtadm/util/edgenode" "github.com/openyurtio/openyurt/pkg/yurthub/storage/disk" @@ -46,6 +45,8 @@ const ( fileMode = 0666 DefaultRootDir = "/var/lib" DefaultCaPath = "/etc/kubernetes/pki/ca.crt" + yurthubYurtStaticSetName = "yurthub" + defaultConfigmapPath = "/data" ) type yurtHubOperator struct { @@ -77,20 +78,8 @@ func (op *yurtHubOperator) Install() error { // 1. put yurt-hub yaml into /etc/kubernetes/manifests klog.Infof("setting up yurthub on node") - // 1-1. replace variables in yaml file - klog.Infof("setting up yurthub apiServer addr") - yurthubTemplate, err := templates.SubsituteTemplate(constants.YurthubTemplate, map[string]string{ - "yurthubBindingAddr": constants.DefaultYurtHubServerAddr, - "kubernetesServerAddr": op.apiServerAddr, - "image": op.yurthubImage, - "bootstrapFile": constants.YurtHubBootstrapConfig, - "workingMode": string(op.workingMode), - "enableDummyIf": strconv.FormatBool(op.enableDummyIf), - "enableNodePool": strconv.FormatBool(op.enableNodePool), - }) - if err != nil { - return err - } + // 1-1. get configmap data path + configMapDataPath := filepath.Join(defaultConfigmapPath, yurthubYurtStaticSetName) // 1-2. create /var/lib/yurthub/bootstrap-hub.conf if err := enutil.EnsureDir(constants.YurtHubWorkdir); err != nil { @@ -106,10 +95,24 @@ func (op *yurtHubOperator) Install() error { if err := enutil.EnsureDir(podManifestPath); err != nil { return err } - if err := os.WriteFile(getYurthubYaml(podManifestPath), []byte(yurthubTemplate), fileMode); err != nil { + content, err := os.ReadFile(configMapDataPath) + if err != nil { + return fmt.Errorf("failed to read source file %s: %w", configMapDataPath, err) + } + klog.Infof("yurt-hub.yaml apiServerAddr: %+v", op.apiServerAddr) + yssYurtHub, err := tmplutil.SubsituteTemplate(string(content), map[string]string{ + "kubernetesServerAddr": op.apiServerAddr, + }) + if err != nil { return err } + if err = os.WriteFile(getYurthubYaml(podManifestPath), []byte(yssYurtHub), fileMode); err != nil { + return err + } + klog.Infof("create the %s/yurt-hub.yaml", podManifestPath) + klog.Infof("yurt-hub.yaml: %+v", configMapDataPath) + klog.Infof("yurt-hub.yaml content: %+v", yssYurtHub) // 2. wait yurthub pod to be ready return hubHealthcheck(op.yurthubHealthCheckTimeout) diff --git a/pkg/node-servant/constant.go b/pkg/node-servant/constant.go index b91d0b4ed0b..f71cf62dd92 100644 --- a/pkg/node-servant/constant.go +++ b/pkg/node-servant/constant.go @@ -42,6 +42,10 @@ spec: hostPath: path: / type: Directory + - name: configmap + configMap: + defaultMode: 420 + name: {{.configmap_name}} containers: - name: node-servant-servant image: {{.node_servant_image}} @@ -56,6 +60,8 @@ spec: volumeMounts: - mountPath: /openyurt name: host-root + - mountPath: /openyurt/data + name: configmap env: - name: NODE_NAME valueFrom: diff --git a/test/e2e/cmd/init/constants/constants.go b/test/e2e/cmd/init/constants/constants.go index 5d4300a9876..228bc16efad 100644 --- a/test/e2e/cmd/init/constants/constants.go +++ b/test/e2e/cmd/init/constants/constants.go @@ -187,4 +187,137 @@ data: discardcloudservice: "" masterservice: "" ` + + YurthubCloudYurtStaticSet = ` +apiVersion: apps.openyurt.io/v1alpha1 +kind: YurtStaticSet +metadata: + name: yurt-hub-cloud + namespace: "kube-system" +spec: + staticPodManifest: yurthub + template: + metadata: + labels: + k8s-app: yurt-hub-cloud + spec: + volumes: + - name: hub-dir + hostPath: + path: /var/lib/yurthub + type: DirectoryOrCreate + - name: kubernetes + hostPath: + path: /etc/kubernetes + type: Directory + containers: + - name: yurt-hub + image: {{.yurthub_image}} + imagePullPolicy: IfNotPresent + volumeMounts: + - name: hub-dir + mountPath: /var/lib/yurthub + - name: kubernetes + mountPath: /etc/kubernetes + command: + - yurthub + - --v=2 + - --bind-address=127.0.0.1 + - --server-addr={{.kubernetesServerAddr}} + - --node-name=$(NODE_NAME) + - --bootstrap-file=/var/lib/yurthub/bootstrap-hub.conf + - --working-mode=cloud + - --namespace="kube-system" + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /v1/healthz + port: 10267 + initialDelaySeconds: 300 + periodSeconds: 5 + failureThreshold: 3 + resources: + requests: + cpu: 150m + memory: 150Mi + limits: + memory: 300Mi + securityContext: + capabilities: + add: [ "NET_ADMIN", "NET_RAW" ] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + hostNetwork: true + priorityClassName: system-node-critical + priority: 2000001000 +` + YurthubYurtStaticSet = ` +apiVersion: apps.openyurt.io/v1alpha1 +kind: YurtStaticSet +metadata: + name: yurt-hub + namespace: "kube-system" +spec: + staticPodManifest: yurthub + template: + metadata: + labels: + k8s-app: yurt-hub + spec: + volumes: + - name: hub-dir + hostPath: + path: /var/lib/yurthub + type: DirectoryOrCreate + - name: kubernetes + hostPath: + path: /etc/kubernetes + type: Directory + containers: + - name: yurt-hub + image: {{.yurthub_image}} + imagePullPolicy: IfNotPresent + volumeMounts: + - name: hub-dir + mountPath: /var/lib/yurthub + - name: kubernetes + mountPath: /etc/kubernetes + command: + - yurthub + - --v=2 + - --bind-address=127.0.0.1 + - --server-addr={{.kubernetesServerAddr}} + - --node-name=$(NODE_NAME) + - --bootstrap-file=/var/lib/yurthub/bootstrap-hub.conf + - --working-mode=edge + - --namespace="kube-system" + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /v1/healthz + port: 10267 + initialDelaySeconds: 300 + periodSeconds: 5 + failureThreshold: 3 + resources: + requests: + cpu: 150m + memory: 150Mi + limits: + memory: 300Mi + securityContext: + capabilities: + add: [ "NET_ADMIN", "NET_RAW" ] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + hostNetwork: true + priorityClassName: system-node-critical + priority: 2000001000 +` ) diff --git a/test/e2e/cmd/init/converter.go b/test/e2e/cmd/init/converter.go index 9108248b90e..65adca57541 100644 --- a/test/e2e/cmd/init/converter.go +++ b/test/e2e/cmd/init/converter.go @@ -39,7 +39,9 @@ import ( nodeservant "github.com/openyurtio/openyurt/pkg/node-servant" kubeadmapi "github.com/openyurtio/openyurt/pkg/util/kubernetes/kubeadm/app/phases/bootstraptoken/clusterinfo" strutil "github.com/openyurtio/openyurt/pkg/util/strings" + tmplutil "github.com/openyurtio/openyurt/pkg/util/templates" "github.com/openyurtio/openyurt/pkg/yurthub/util" + "github.com/openyurtio/openyurt/test/e2e/cmd/init/constants" "github.com/openyurtio/openyurt/test/e2e/cmd/init/lock" kubeutil "github.com/openyurtio/openyurt/test/e2e/cmd/init/util/kubernetes" ) @@ -47,6 +49,8 @@ import ( const ( // defaultYurthubHealthCheckTimeout defines the default timeout for yurthub health check phase defaultYurthubHealthCheckTimeout = 2 * time.Minute + yssYurtHubCloudName = "yurt-static-set-yurt-hub-cloud" + yssYurtHubName = "yurt-static-set-yurt-hub" ) type ClusterConverter struct { @@ -124,14 +128,45 @@ func (c *ClusterConverter) deployYurthub() error { // The node-servant will detect the kubeadm_conf_path automatically // It will be either "/usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf" // or "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf". - "kubeadm_conf_path": "", - "working_mode": string(util.WorkingModeEdge), - "enable_dummy_if": strconv.FormatBool(c.EnableDummyIf), + "kubeadm_conf_path": "", + "working_mode": string(util.WorkingModeEdge), + "enable_dummy_if": strconv.FormatBool(c.EnableDummyIf), + "kubernetesServerAddr": "{{.kubernetesServerAddr}}", } if c.YurthubHealthCheckTimeout != defaultYurthubHealthCheckTimeout { convertCtx["yurthub_healthcheck_timeout"] = c.YurthubHealthCheckTimeout.String() } + // create the yurthub-cloud and yurthub yss + tempDir, err := os.MkdirTemp(c.RootDir, "yurt-hub") + if err != nil { + return err + } + defer os.RemoveAll(tempDir) + tempFile := filepath.Join(tempDir, "yurthub-cloud-yurtstaticset.yaml") + yssYurtHubCloud, err := tmplutil.SubsituteTemplate(constants.YurthubCloudYurtStaticSet, convertCtx) + if err != nil { + return err + } + if err = os.WriteFile(tempFile, []byte(yssYurtHubCloud), 0644); err != nil { + return err + } + if err = c.ComponentsBuilder.InstallComponents(tempFile, false); err != nil { + return err + } + + tempFile = filepath.Join(tempDir, "yurthub-yurtstaticset.yaml") + yssYurtHub, err := tmplutil.SubsituteTemplate(constants.YurthubYurtStaticSet, convertCtx) + if err != nil { + return err + } + if err = os.WriteFile(tempFile, []byte(yssYurtHub), 0644); err != nil { + return err + } + if err = c.ComponentsBuilder.InstallComponents(tempFile, false); err != nil { + return err + } + npExist, err := nodePoolResourceExists(c.ClientSet) if err != nil { return err @@ -141,6 +176,7 @@ func (c *ClusterConverter) deployYurthub() error { if len(c.EdgeNodes) != 0 { convertCtx["working_mode"] = string(util.WorkingModeEdge) + convertCtx["configmap_name"] = yssYurtHubName if err = kubeutil.RunServantJobs(c.ClientSet, c.WaitServantJobTimeout, func(nodeName string) (*batchv1.Job, error) { return nodeservant.RenderNodeServantJob("convert", convertCtx, nodeName) }, c.EdgeNodes, os.Stderr, false); err != nil { @@ -175,6 +211,7 @@ func (c *ClusterConverter) deployYurthub() error { // deploy yurt-hub and reset the kubelet service on cloud nodes convertCtx["working_mode"] = string(util.WorkingModeCloud) + convertCtx["configmap_name"] = yssYurtHubCloudName klog.Infof("convert context for cloud nodes(%q): %#+v", c.CloudNodes, convertCtx) if err = kubeutil.RunServantJobs(c.ClientSet, c.WaitServantJobTimeout, func(nodeName string) (*batchv1.Job, error) { return nodeservant.RenderNodeServantJob("convert", convertCtx, nodeName)