From d774a266df7eafcb2c88b72fecf148e44a95ae6e Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Thu, 7 Mar 2019 09:39:23 +0000 Subject: [PATCH] WIP --- pkg/addons/default/aws_node.go | 51 +++----- pkg/addons/default/helpers.go | 82 +++++++++++++ pkg/ctl/create/cluster.go | 6 +- pkg/ctl/utils/update_aws_node.go | 4 +- pkg/ctl/utils/write_kubeconfig.go | 4 +- pkg/eks/auth.go | 77 ------------- pkg/eks/client.go | 185 ++++++++++++++++++++++++++++++ 7 files changed, 287 insertions(+), 122 deletions(-) create mode 100644 pkg/eks/client.go diff --git a/pkg/addons/default/aws_node.go b/pkg/addons/default/aws_node.go index cb5ebcd89a8..ade923844f2 100644 --- a/pkg/addons/default/aws_node.go +++ b/pkg/addons/default/aws_node.go @@ -1,26 +1,20 @@ package defaultaddons import ( - "fmt" - "github.com/kris-nova/logger" "github.com/pkg/errors" + "github.com/weaveworks/eksctl/pkg/eks" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/cli-runtime/pkg/genericclioptions/resource" - "k8s.io/client-go/kubernetes" - // "k8s.io/apimachinery/pkg/api/meta" - // "k8s.io/cli-runtime/pkg/genericclioptions/resource" - // "k8s.io/client-go/kubernetes/scheme" ) const ( AWSNode = "aws-node" ) -func UpdateAWSNode(clientSet *kubernetes.Clientset, dryRun bool) error { - _, err := clientSet.Apps().DaemonSets(metav1.NamespaceSystem).Get(AWSNode, metav1.GetOptions{}) +func UpdateAWSNode(rawClient *eks.RawClient, dryRun bool) error { + _, err := rawClient.ClientSet.Apps().DaemonSets(metav1.NamespaceSystem).Get(AWSNode, metav1.GetOptions{}) if err != nil { if apierrs.IsNotFound(err) { logger.Warning("%q was not found", AWSNode) @@ -35,38 +29,19 @@ func UpdateAWSNode(clientSet *kubernetes.Clientset, dryRun bool) error { return err } - // client := clientSet.RESTClient().Put().Namespace(metav1.NamespaceSystem) - - // client := clientSet.RESTClient() - // mapper := meta.NewDefaultRESTMapper(scheme.Scheme) + // client, err := NewClientHelper(clientSet) + // if err != nil { + // return err + // } - for _, runtimeObj := range list.Items { - gvk := runtimeObj.Object.GetObjectKind().GroupVersionKind() - obj, ok := runtimeObj.Object.(metav1.Object) - if !ok { - return fmt.Errorf("cannot conver object of type %T to metav1.Object", runtimeObj) + for _, rawObj := range list.Items { + r, info, err := NewResourceClient(rawClient, rawObj) + if err != nil { + return err } - - logger.Info("%s:%s.%s/%s", obj.GetNamespace(), gvk.Kind, gvk.Group, obj.GetName()) - - // fqKind := schema.FromAPIVersionAndKind(gvk.Version, gvk.Kind) - // RESTMapping(fqKind.GroupKind(), fqKind.Version) - - r, err := resource.NewHelper(client, mapper).Get(obj.GetNamespace(), obj.GetName(), false) - logger.Debug("r = %#v", r) + res, err := r.Get(info.Namespace, info.Name, false) + logger.Debug("res = %#v", res) logger.Debug("err = %#v", err) - - // logger.Info("client.APIVersion = %#v", client.APIVersion()) - // req := client.Get(). - // NamespaceIfScoped(obj.GetNamespace(), obj.GetNamespace() != ""). - // //Resource(gvk.Kind + "." + gvk.Group). - // Resource(gvk.Kind). - // Name(obj.GetName()) - // // req := client.Resource().Body(obj.Object) - // logger.Debug("req = %#v", req) - // res := req.Do() - // logger.Debug("res = %#v", res) - // logger.Debug("res.Error = %#v", res.Error()) } return nil diff --git a/pkg/addons/default/helpers.go b/pkg/addons/default/helpers.go index 8d034c68568..fd778bf1c5d 100644 --- a/pkg/addons/default/helpers.go +++ b/pkg/addons/default/helpers.go @@ -2,15 +2,19 @@ package defaultaddons import ( "bytes" + "fmt" "io" "strings" + "github.com/kris-nova/logger" "github.com/pkg/errors" + "github.com/weaveworks/eksctl/pkg/eks" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/cli-runtime/pkg/genericclioptions/resource" "k8s.io/client-go/kubernetes/scheme" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" @@ -77,3 +81,81 @@ func listAppendFlattened(components *metav1.List, component runtime.RawExtension } return listAppendFlattened(components, runtime.RawExtension{Object: obj}) } + +// type ClientHelper struct { +// client restclient.Interface +// mapper meta.RESTMapper +// } + +// func NewClientHelper(clientSet *kubernetes.Clientset) (*ClientHelper, error) { +// // mapper := restmapper.NewDeferredDiscoveryRESTMapper(cacheddiscovery.NewMemCacheClient(clientSet.Discovery())) + +// apiGroupResources, err := restmapper.GetAPIGroupResources(clientSet.Discovery()) +// if err != nil { +// return nil, errors.Wrap(err, "getting list of API resources for REST client") +// } + +// for i, r := range apiGroupResources { +// logger.Debug("apiGroupResources[%d] = %#v", i, *r) +// } + +// c := &ClientHelper{ +// client: clientSet.RESTClient(), +// mapper: restmapper.NewDiscoveryRESTMapper(apiGroupResources), +// } + +// return c, nil +// } + +// func (c *ClientHelper) NewResource(rawObj runtime.RawExtension) (*resource.Helper, *resource.Info, error) { +// gvk := rawObj.Object.GetObjectKind().GroupVersionKind() +// // fqk := schema.FromAPIVersionAndKind(gvk.Version, gvk.Kind) +// // mapping, err := c.mapper.RESTMapping(fqk.GroupKind(), fqk.GroupVersion().Version) +// mapping, err := c.mapper.RESTMapping(gvk.GroupKind(), gvk.GroupVersion().Version) + +// if err != nil { +// return nil, nil, errors.Wrap(err, "constructing REST client mapping") +// } + +// obj, ok := rawObj.Object.(metav1.Object) +// if !ok { +// return nil, nil, fmt.Errorf("cannot conver object of type %T to metav1.Object", rawObj.Object) +// } + +// info := &resource.Info{ +// Client: c.client, +// Mapping: mapping, +// Name: obj.GetName(), +// Namespace: obj.GetNamespace(), +// Object: rawObj.Object, +// } +// logger.Info("%s:%s.%s/%s", info.Namespace, gvk.Kind, gvk.Group, info.Name) + +// return resource.NewHelper(c.client, mapping), info, nil +// } + +func NewResourceClient(rawClient *eks.RawClient, rawObj runtime.RawExtension) (*resource.Helper, *resource.Info, error) { + gvk := rawObj.Object.GetObjectKind().GroupVersionKind() + // fqk := schema.FromAPIVersionAndKind(gvk.Version, gvk.Kind) + // mapping, err := c.mapper.RESTMapping(fqk.GroupKind(), fqk.GroupVersion().Version) + helper, err := rawClient.NewFor(gvk) + if err != nil { + return nil, nil, err + } + + obj, ok := rawObj.Object.(metav1.Object) + if !ok { + return nil, nil, fmt.Errorf("cannot conver object of type %T to metav1.Object", rawObj.Object) + } + + info := &resource.Info{ + // Client: c.client, + // Mapping: mapping, + Name: obj.GetName(), + Namespace: obj.GetNamespace(), + Object: rawObj.Object, + } + logger.Info("%s:%s.%s/%s", info.Namespace, gvk.Kind, gvk.Group, info.Name) + + return helper, info, nil +} diff --git a/pkg/ctl/create/cluster.go b/pkg/ctl/create/cluster.go index a9f064a4fda..5f5257727d3 100644 --- a/pkg/ctl/create/cluster.go +++ b/pkg/ctl/create/cluster.go @@ -490,13 +490,13 @@ func doCreateCluster(p *api.ProviderConfig, cfg *api.ClusterConfig, nameArg stri var kubeconfigContextName string if writeKubeconfig { - clientConfig, err := ctl.NewClientConfig(cfg, false) + client, err := ctl.NewClient(cfg, false) if err != nil { return err } - kubeconfigContextName = clientConfig.ContextName + kubeconfigContextName = client.ContextName - kubeconfigPath, err = kubeconfig.Write(kubeconfigPath, *clientConfig.Client, setContext) + kubeconfigPath, err = kubeconfig.Write(kubeconfigPath, *client.Config, setContext) if err != nil { return errors.Wrap(err, "writing kubeconfig") } diff --git a/pkg/ctl/utils/update_aws_node.go b/pkg/ctl/utils/update_aws_node.go index 441c450459d..1307eef8669 100644 --- a/pkg/ctl/utils/update_aws_node.go +++ b/pkg/ctl/utils/update_aws_node.go @@ -69,10 +69,10 @@ func doUpdateAWSNode(p *api.ProviderConfig, cfg *api.ClusterConfig, nameArg stri return errors.Wrapf(err, "getting credentials for cluster %q", meta.Name) } - clientSet, err := ctl.NewStdClientSet(cfg) + rawClient, err := ctl.NewRawClient(cfg) if err != nil { return err } - return defaultaddons.UpdateAWSNode(clientSet, false) + return defaultaddons.UpdateAWSNode(rawClient, false) } diff --git a/pkg/ctl/utils/write_kubeconfig.go b/pkg/ctl/utils/write_kubeconfig.go index f89ea9751d3..7090a18eb9b 100644 --- a/pkg/ctl/utils/write_kubeconfig.go +++ b/pkg/ctl/utils/write_kubeconfig.go @@ -83,12 +83,12 @@ func doWriteKubeconfigCmd(p *api.ProviderConfig, cfg *api.ClusterConfig, nameArg return err } - config, err := ctl.NewClientConfig(cfg, false) + client, err := ctl.NewClient(cfg, false) if err != nil { return err } - filename, err := kubeconfig.Write(writeKubeconfigOutputPath, *config.Client, writeKubeconfigSetContext) + filename, err := kubeconfig.Write(writeKubeconfigOutputPath, *client.Config, writeKubeconfigSetContext) if err != nil { return errors.Wrap(err, "writing kubeconfig") } diff --git a/pkg/eks/auth.go b/pkg/eks/auth.go index d72eb0d22a6..e193c72262f 100644 --- a/pkg/eks/auth.go +++ b/pkg/eks/auth.go @@ -11,17 +11,11 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/sts" - "github.com/kubernetes-sigs/aws-iam-authenticator/pkg/token" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "k8s.io/kops/pkg/pki" api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha4" "github.com/weaveworks/eksctl/pkg/utils" - "github.com/weaveworks/eksctl/pkg/utils/kubeconfig" ) func (c *ClusterProvider) getKeyPairName(clusterName string, ng *api.NodeGroup, fingerprint *string) string { @@ -150,74 +144,3 @@ func (c *ClusterProvider) getUsername() string { } return "iam-root-account" } - -// ClientConfig stores information about the client config -type ClientConfig struct { - Client *clientcmdapi.Config - ContextName string -} - -// NewClientConfig creates a new client config, if withEmbeddedToken is true -// it will embed the STS token, otherwise it will use authenticator exec plugin -// and ensures that AWS_PROFILE environment variable gets set also -func (c *ClusterProvider) NewClientConfig(spec *api.ClusterConfig, withEmbeddedToken bool) (*ClientConfig, error) { - client, _, contextName := kubeconfig.New(spec, c.getUsername(), "") - - config := &ClientConfig{ - Client: client, - ContextName: contextName, - } - - if withEmbeddedToken { - if err := config.useEmbeddedToken(spec, c.Provider.STS().(*sts.STS)); err != nil { - return nil, err - } - } else { - kubeconfig.AppendAuthenticator(config.Client, spec, utils.DetectAuthenticator(), c.Provider.Profile()) - } - - return config, nil -} - -func (c *ClientConfig) useEmbeddedToken(spec *api.ClusterConfig, sts *sts.STS) error { - gen, err := token.NewGenerator(true) - if err != nil { - return errors.Wrap(err, "could not get token generator") - } - - tok, err := gen.GetWithSTS(spec.Metadata.Name, sts) - if err != nil { - return errors.Wrap(err, "could not get token") - } - - c.Client.AuthInfos[c.ContextName].Token = tok - return nil -} - -// NewClientSet creates a new API client -func (c *ClientConfig) NewClientSet() (*kubernetes.Clientset, error) { - clientConfig, err := clientcmd.NewDefaultClientConfig(*c.Client, &clientcmd.ConfigOverrides{}).ClientConfig() - if err != nil { - return nil, errors.Wrap(err, "failed to create API client configuration from client config") - } - - client, err := kubernetes.NewForConfig(clientConfig) - if err != nil { - return nil, errors.Wrap(err, "failed to create API client") - } - return client, nil -} - -// NewStdClientSet creates a new API client in one go with an embedded STS token, this is most commonly used option -func (c *ClusterProvider) NewStdClientSet(spec *api.ClusterConfig) (*kubernetes.Clientset, error) { - clientConfig, err := c.NewClientConfig(spec, true) - if err != nil { - return nil, errors.Wrap(err, "creating Kubernetes client config with embedded token") - } - - clientSet, err := clientConfig.NewClientSet() - if err != nil { - return nil, errors.Wrap(err, "creating Kubernetes client") - } - return clientSet, nil -} diff --git a/pkg/eks/client.go b/pkg/eks/client.go new file mode 100644 index 00000000000..0f7933f1c5b --- /dev/null +++ b/pkg/eks/client.go @@ -0,0 +1,185 @@ +package eks + +import ( + "github.com/kris-nova/logger" + "github.com/pkg/errors" + + "github.com/aws/aws-sdk-go/service/sts" + + "github.com/kubernetes-sigs/aws-iam-authenticator/pkg/token" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/cli-runtime/pkg/genericclioptions/resource" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + + api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha4" + "github.com/weaveworks/eksctl/pkg/utils" + "github.com/weaveworks/eksctl/pkg/utils/kubeconfig" +) + +// Client stores information about the client config +type Client struct { + Config *clientcmdapi.Config + ContextName string + + rawConfig *restclient.Config +} + +// RawClient stores information about the client config +type RawClient struct { + mapper meta.RESTMapper + config *restclient.Config + ClientSet kubernetes.Interface +} + +// NewClient creates a new client config, if withEmbeddedToken is true +// it will embed the STS token, otherwise it will use authenticator exec plugin +// and ensures that AWS_PROFILE environment variable gets set also +func (c *ClusterProvider) NewClient(spec *api.ClusterConfig, withEmbeddedToken bool) (*Client, error) { + clientConfig, _, contextName := kubeconfig.New(spec, c.getUsername(), "") + + config := &Client{ + Config: clientConfig, + ContextName: contextName, + } + + return config.new(spec, withEmbeddedToken, c.Provider.STS().(*sts.STS), c.Provider.Profile()) +} + +func (c *Client) new(spec *api.ClusterConfig, withEmbeddedToken bool, sts *sts.STS, profile string) (*Client, error) { + if withEmbeddedToken { + if err := c.useEmbeddedToken(spec, sts); err != nil { + return nil, err + } + } else { + kubeconfig.AppendAuthenticator(c.Config, spec, utils.DetectAuthenticator(), profile) + } + + rawConfig, err := clientcmd.NewDefaultClientConfig(*c.Config, &clientcmd.ConfigOverrides{}).ClientConfig() + if err != nil { + return nil, errors.Wrap(err, "failed to create API client configuration from client config") + } + c.rawConfig = rawConfig + + return c, nil +} + +func (c *Client) useEmbeddedToken(spec *api.ClusterConfig, sts *sts.STS) error { + gen, err := token.NewGenerator(true) + if err != nil { + return errors.Wrap(err, "could not get token generator") + } + + tok, err := gen.GetWithSTS(spec.Metadata.Name, sts) + if err != nil { + return errors.Wrap(err, "could not get token") + } + + c.Config.AuthInfos[c.ContextName].Token = tok + return nil +} + +// NewClientSet creates a new API client +func (c *Client) NewClientSet() (*kubernetes.Clientset, error) { + client, err := kubernetes.NewForConfig(c.rawConfig) + if err != nil { + return nil, errors.Wrap(err, "failed to create API client") + } + return client, nil +} + +func (c *ClusterProvider) newClientSetWithEmbeddedToken(spec *api.ClusterConfig) (*Client, *kubernetes.Clientset, error) { + client, err := c.NewClient(spec, true) + if err != nil { + return nil, nil, errors.Wrap(err, "creating Kubernetes client config with embedded token") + } + + clientSet, err := client.NewClientSet() + if err != nil { + return nil, nil, errors.Wrap(err, "creating Kubernetes client") + } + + return client, clientSet, nil +} + +// NewStdClientSet creates a new API client in one go with an embedded STS token, this is most commonly used option +func (c *ClusterProvider) NewStdClientSet(spec *api.ClusterConfig) (*kubernetes.Clientset, error) { + _, clientSet, err := c.newClientSetWithEmbeddedToken(spec) + if err != nil { + return nil, err + } + + return clientSet, nil +} + +// NewRawClient creates a new raw REST client in one go with an embedded STS token +func (c *ClusterProvider) NewRawClient(spec *api.ClusterConfig) (*RawClient, error) { + client, clientSet, err := c.newClientSetWithEmbeddedToken(spec) + if err != nil { + return nil, err + } + rawClient := &RawClient{ + config: client.rawConfig, + ClientSet: clientSet, + } + return rawClient.new() +} + +func (c *RawClient) new() (*RawClient, error) { + apiGroupResources, err := restmapper.GetAPIGroupResources(c.ClientSet.Discovery()) + if err != nil { + return nil, errors.Wrap(err, "getting list of API resources for raw REST client") + } + + for i, r := range apiGroupResources { + logger.Debug("apiGroupResources[%d] = %#v", i, *r) + } + + c.mapper = restmapper.NewDiscoveryRESTMapper(apiGroupResources) + + // c.config.GroupVersion = &schema.GroupVersion{Group: "", Version: "v1"} + if c.config.APIPath == "" { + c.config.APIPath = "/api" + } + if c.config.NegotiatedSerializer == nil { + c.config.NegotiatedSerializer = &serializer.DirectCodecFactory{CodecFactory: scheme.Codecs} + } + if err := restclient.SetKubernetesDefaults(c.config); err != nil { + return nil, errors.Wrap(err, "applying defaults for REST client") + } + return c, nil +} + +func (c *RawClient) NewFor(gvk schema.GroupVersionKind) (*resource.Helper, error) { + logger.Debug("gvk.GroupKind().Group = %s", gvk.GroupKind().Group) + logger.Debug("gvk.GroupKind().Kind = %s", gvk.GroupKind().Kind) + logger.Debug("gvk.GroupVersion().Version = %s", gvk.GroupVersion().Version) + mapping, err := c.mapper.RESTMapping(gvk.GroupKind(), gvk.GroupVersion().Version, "") + if err != nil { + return nil, errors.Wrapf(err, "constructing REST client mapping for %s", gvk.String()) + } + + // based on https: https://github.com/kubernetes/kubernetes/blob/bf4e93da5740c4e70eff390e9723bde76dedcb54/pkg/kubectl/cmd/util/factory_client_access.go#L108-L126 + switch gvk.Group { + case corev1.GroupName: + c.config.APIPath = "/api" + default: + c.config.APIPath = "/apis" + } + gv := gvk.GroupVersion() + c.config.GroupVersion = &gv + + client, err := restclient.RESTClientFor(c.config) + if err != nil { + return nil, errors.Wrapf(err, "constructing REST client for %s", gvk.String()) + } + + return resource.NewHelper(client, mapping), nil +}