From 50928a4c9c44b8b2e5adb86fc9e45c5e9dabc35e Mon Sep 17 00:00:00 2001 From: cbruno10 Date: Fri, 29 Sep 2023 13:19:01 -0400 Subject: [PATCH] Deprecate `source_type` config arg and add `source_types` config arg closes #166 (#167) Co-authored-by: Graza --- .gitignore | 1 + README.md | 13 ++-- config/kubernetes.spc | 13 ++-- docs/index.md | 19 +++++- kubernetes/connection_config.go | 1 + kubernetes/helm_template_render.go | 4 +- kubernetes/helm_utils.go | 31 ++++++---- kubernetes/utils.go | 98 +++++++++++++++++------------- 8 files changed, 108 insertions(+), 72 deletions(-) diff --git a/.gitignore b/.gitignore index 5f10583..006b118 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ # vendor/ .vscode/* +.idea diff --git a/README.md b/README.md index fae3f2a..205c4cb 100644 --- a/README.md +++ b/README.md @@ -54,13 +54,12 @@ connection "kubernetes" { # If no kubeconfig file can be found, the plugin will attempt to use the service account Kubernetes gives to pods. # This authentication method is intended for clients that expect to be running inside a pod running on Kubernetes. - # Specify the source of the resource. Possible values: `deployed`, `helm`, `manifest`, and `all`. - # Default set to `all`. Set the argument to override the default value. - # If the value is set to `deployed`, tables will show all the deployed resources. - # If set to `helm`, tables will only show resources from the configured helm charts. - # If set to `manifest`, tables will show all the resources from the kubernetes manifest. Make sure that the `manifest_file_paths` arg is set. - # If `all`, tables will show all the deployed and manifest resources. - # source_type = "all" + # Specify the source(s) of the resource(s). Possible values: `deployed`, `helm` and `manifest`. + # Defaults to all possible values. Set the argument to override the default value. + # If `deployed` is contained in the value, tables will show all the deployed resources. + # If `helm` is contained in the value, tables will show resources from the configured helm charts. + # If `manifest` is contained in the value, tables will show all the resources from the kubernetes manifest. Make sure that the `manifest_file_paths` arg is set. + # source_types = ["deployed", "helm", "manifest"] # Manifest File Configuration diff --git a/config/kubernetes.spc b/config/kubernetes.spc index ab5d5c8..e9763ad 100644 --- a/config/kubernetes.spc +++ b/config/kubernetes.spc @@ -28,13 +28,12 @@ connection "kubernetes" { # If no kubeconfig file can be found, the plugin will attempt to use the service account Kubernetes gives to pods. # This authentication method is intended for clients that expect to be running inside a pod running on Kubernetes. - # Specify the source of the resource. Possible values: `deployed`, `helm`, `manifest`, and `all`. - # Default set to `all`. Set the argument to override the default value. - # If the value is set to `deployed`, tables will show all the deployed resources. - # If set to `helm`, tables will only show resources from the configured helm charts. - # If set to `manifest`, tables will show all the resources from the kubernetes manifest. Make sure that the `manifest_file_paths` arg is set. - # If `all`, tables will show all the deployed and manifest resources. - # source_type = "all" + # Specify the source(s) of the resource(s). Possible values: `deployed`, `helm` and `manifest`. + # Defaults to all possible values. Set the argument to override the default value. + # If `deployed` is contained in the value, tables will show all the deployed resources. + # If `helm` is contained in the value, tables will show resources from the configured helm charts. + # If `manifest` is contained in the value, tables will show all the resources from the kubernetes manifest. Make sure that the `manifest_file_paths` arg is set. + # source_types = ["deployed", "helm", "manifest"] # Manifest File Configuration diff --git a/docs/index.md b/docs/index.md index 1eb588f..9125c1b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -43,6 +43,7 @@ connection "kubernetes" { plugin = "kubernetes" config_path = "~/.kube/config" config_context = "myCluster" + source_types = ["deployed"] } ``` @@ -113,12 +114,14 @@ connection "kubernetes_cluster_aks" { plugin = "kubernetes" config_path = "~/.kube/config" config_context = "myAKSCluster" + source_types = ["deployed"] } connection "kubernetes_cluster_eks" { plugin = "kubernetes" config_path = "~/.kube/config" config_context = "arn:aws:eks:us-east-1:123456789012:cluster/myEKSCluster" + source_types = ["deployed"] } ``` @@ -174,12 +177,14 @@ connection "kubernetes" { "github.com/GoogleCloudPlatform/microservices-demo//release//kubernetes-manifests.yaml", "s3::https://bucket.s3.us-east-1.amazonaws.com/test_folder//*.yml" ] + + source_types = ["manifest"] } ``` **Note**: If any path matches on `*` without `.yml` or `.yaml` or `.json`, all files (including non-Kubernetes manifest files) in the directory will be matched, which may cause errors if incompatible file types exist. -By default the plugin always lists the resources deployed in the current Kubernetes cluster context. If you want to restrict this behavior to read resource configurations from the configured manifest files only, add the `source_type` argument to the config and set the value to `manifest`. For example: +By default the plugin always lists the resources deployed in the current Kubernetes cluster context. If you want to restrict this behavior to read resource configurations from the configured manifest files only, add the `source_types` argument to the config and set the value to `manifest`. For example: ```hcl connection "kubernetes" { @@ -187,7 +192,7 @@ connection "kubernetes" { manifest_file_paths = [ ... ] - source_type = "manifest" + source_types = ["manifest"] } ``` @@ -209,6 +214,8 @@ connection "kubernetes" { plugin = "kubernetes" manifest_file_paths = [ "*.yml", "*.yaml", "*.json", "/path/to/dir/main.yml" ] + + source_types = ["manifest"] } ``` @@ -230,6 +237,8 @@ connection "kubernetes" { plugin = "kubernetes" manifest_file_paths = [ "bitbucket.org/atlassian/kubectl-run//test/kustomization//deploy.yml" ] + + source_types = ["manifest"] } ``` @@ -255,6 +264,8 @@ connection "kubernetes" { "s3::https://bucket-2.s3.us-east-1.amazonaws.com//*.yml?aws_profile=", "s3::https://bucket-2.s3.us-east-1.amazonaws.com/test_folder//*.yaml?aws_profile=" ] + + source_types = ["manifest"] } ``` @@ -299,6 +310,8 @@ connection "kubernetes" { "s3::https://bucket-1.s3.us-east-1.amazonaws.com/test_folder//*.yml", "s3::https://bucket-2.s3.us-east-1.amazonaws.com/test_folder//**/*.yaml" ] + + source_types = ["manifest"] } ``` @@ -332,6 +345,8 @@ connection "kubernetes" { values_file_paths = [] # works with values from chart's default values.yaml file } } + + source_types = ["helm"] } ``` diff --git a/kubernetes/connection_config.go b/kubernetes/connection_config.go index d14781b..9236826 100644 --- a/kubernetes/connection_config.go +++ b/kubernetes/connection_config.go @@ -11,6 +11,7 @@ type kubernetesConfig struct { CustomResourceTables []string `hcl:"custom_resource_tables,optional"` ManifestFilePaths []string `hcl:"manifest_file_paths,optional" steampipe:"watch"` SourceType *string `hcl:"source_type,optional"` + SourceTypes []string `hcl:"source_types,optional"` HelmRenderedCharts map[string]chartConfig `hcl:"helm_rendered_charts,optional"` } diff --git a/kubernetes/helm_template_render.go b/kubernetes/helm_template_render.go index be0666a..b164221 100644 --- a/kubernetes/helm_template_render.go +++ b/kubernetes/helm_template_render.go @@ -77,7 +77,7 @@ func getHelmRenderedTemplatesUncached(ctx context.Context, d *plugin.QueryData, if err != nil { return nil, err } - helmConfig := GetConfig(d.Connection) + kubernetesConfig := GetConfig(d.Connection) var renderedTemplates []HelmRenderedTemplate for _, chart := range charts { @@ -89,7 +89,7 @@ func getHelmRenderedTemplatesUncached(ctx context.Context, d *plugin.QueryData, } var processedHelmConfigs []string - for name, c := range helmConfig.HelmRenderedCharts { + for name, c := range kubernetesConfig.HelmRenderedCharts { if c.ChartPath == chart.Path && !helpers.StringSliceContains(processedHelmConfigs, name) { // Add the processed Helm render configs into processedHelmConfigs to avoid duplicate entries diff --git a/kubernetes/helm_utils.go b/kubernetes/helm_utils.go index 2c89b7c..9abd477 100644 --- a/kubernetes/helm_utils.go +++ b/kubernetes/helm_utils.go @@ -44,23 +44,32 @@ var parsedHelmChartCached = plugin.HydrateFunc(parsedHelmChartUncached).Memoize( // getParsedHelmChart instead. func parsedHelmChartUncached(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (any, error) { // Read the config - helmConfig := GetConfig(d.Connection) + kubernetesConfig := GetConfig(d.Connection) - // Return nil, if the source_type is set to other than "all" or "helm" - if helmConfig.SourceType != nil && - !helpers.StringSliceContains([]string{"all", "helm"}, *helmConfig.SourceType) { + // Check for the sourceTypes argument in the config. + // Default set to include values. + var sources = All.ToSourceTypes() + if kubernetesConfig.SourceTypes != nil { + sources = kubernetesConfig.SourceTypes + } + // TODO: Remove once `SourceType` is obsolete + if kubernetesConfig.SourceTypes == nil && kubernetesConfig.SourceType != nil { + if *kubernetesConfig.SourceType != "all" { // if is all, sources is already set by default + sources = []string{*kubernetesConfig.SourceType} + } + } + + if !helpers.StringSliceContains(sources, "helm") { return nil, nil } var charts []*parsedHelmChart - for _, v := range helmConfig.HelmRenderedCharts { - // Return error if source_tpe arg is explicitly set to "helm" in the config, but + for _, v := range kubernetesConfig.HelmRenderedCharts { + // Return error if source_types arg includes "helm" in the config, but // helm_chart_dir arg is not set. - if helmConfig.SourceType != nil && - *helmConfig.SourceType == "helm" && - v.ChartPath == "" { - return nil, errors.New("helm_chart_dir must be set in the config while the source_type is 'helm'") + if v.ChartPath == "" { + return nil, errors.New("helm_chart_dir must be set in the config while source_types includes 'helm'") } // Return empty parsedHelmChart object if no Helm chart directory path provided in the config @@ -124,7 +133,7 @@ func getUniqueValueFilesFromConfig(ctx context.Context, d *plugin.QueryData) []s // getHelmClient creates the client for Helm func getHelmClient(ctx context.Context, namespace string) (helmClient.Client, error) { - // Return nil, if no namespace provided + // Return nil if no namespace provided if namespace == "" { return nil, nil } diff --git a/kubernetes/utils.go b/kubernetes/utils.go index e644eb4..32658b7 100644 --- a/kubernetes/utils.go +++ b/kubernetes/utils.go @@ -58,6 +58,15 @@ func (sourceType SourceType) String() string { return string(sourceType) } +// ToSourceTypes is used to convert SourceType to []string +func (sourceType SourceType) ToSourceTypes() []string { + if sourceType == All { + return []string{Deployed.String(), Helm.String(), Manifest.String()} + } else { + return []string{sourceType.String()} + } +} + // GetNewClientset :: gets client for querying k8s apis for the provided context func GetNewClientset(ctx context.Context, d *plugin.QueryData) (*kubernetes.Clientset, error) { logger := plugin.Logger(ctx) @@ -75,7 +84,7 @@ func GetNewClientset(ctx context.Context, d *plugin.QueryData) (*kubernetes.Clie return nil, err } - // Return nil, if the config is set only to list the manifest resources. + // Return nil if deployed resources should not be included if kubeconfig == nil { return nil, nil } @@ -142,7 +151,7 @@ func GetNewClientCRD(ctx context.Context, d *plugin.QueryData) (*apiextension.Cl return nil, err } - // Return nil, if the config is set to only list the manifest resources. + // Return nil if deployed resources should not be included if kubeconfig == nil { return nil, nil } @@ -196,7 +205,7 @@ func GetNewClientCRDRaw(ctx context.Context, cc *connection.ConnectionCache, c * return nil, err } - // Return nil, if the config is set to only list the manifest resources. + // Return nil if deployed resources should not be included if kubeconfig == nil { return nil, nil } @@ -336,24 +345,21 @@ func getK8Config(ctx context.Context, d *plugin.QueryData) (clientcmd.ClientConf // get kubernetes config info kubernetesConfig := GetConfig(d.Connection) - // Check for the sourceType argument in the config. Valid values are: "deployed", "manifest" and "all". - // Default set to "all". - var source SourceType = "all" - if kubernetesConfig.SourceType != nil { - source = SourceType(*kubernetesConfig.SourceType) - if err := source.IsValid(); err != nil { - plugin.Logger(ctx).Debug("getK8Config", "invalid_source_type_error", "connection", d.Connection.Name, "error", err) - return nil, err + // Check for the sourceTypes argument in the config + // Default set to include values + var sources = All.ToSourceTypes() + if kubernetesConfig.SourceTypes != nil { + sources = kubernetesConfig.SourceTypes + } + // TODO: Remove once `SourceType` is obsolete + if kubernetesConfig.SourceTypes == nil && kubernetesConfig.SourceType != nil { + if *kubernetesConfig.SourceType != "all" { // if is all, sources is already set by default + sources = []string{*kubernetesConfig.SourceType} } } - // By default source type is set to "all", which indicates querying the table will return both deployed, helm and manifest resources. - // If the source type is explicitly set, other plugin will list resources based on that. For example: - // If set to "manifest", the table will only return the manifest resources. - // If set to "helm", the table will return the resources after rendering the templates defined in the configured chart. - // Similarly, setting the value as "deployed" will return all the deployed resources. - if source.String() == "manifest" || source.String() == "helm" { - plugin.Logger(ctx).Debug("getK8Config", "Returning nil for API server client.", "Source type", source.String(), "connection", d.Connection.Name) + if !helpers.StringSliceContains(sources, "deployed") { + plugin.Logger(ctx).Debug("getK8Config", "Returning nil for API server client.", "source_types", sources, "connection", d.Connection.Name) return nil, nil } @@ -427,24 +433,21 @@ func getK8ConfigRaw(ctx context.Context, cc *connection.ConnectionCache, c *plug // get kubernetes config info kubernetesConfig := GetConfig(c) - // Check for the sourceType argument in the config. Valid values are: "deployed", "manifest" and "all". - // Default set to "all". - var source SourceType = "all" - if kubernetesConfig.SourceType != nil { - source = SourceType(*kubernetesConfig.SourceType) - if err := source.IsValid(); err != nil { - plugin.Logger(ctx).Debug("getK8ConfigRaw", "invalid_source_type_error", "connection", c.Name, "error", err) - return nil, err + // Check for the sourceTypes argument in the config. + // Default set to include values. + var sources = All.ToSourceTypes() + if kubernetesConfig.SourceTypes != nil { + sources = kubernetesConfig.SourceTypes + } + // TODO: Remove once `SourceType` is obsolete + if kubernetesConfig.SourceTypes == nil && kubernetesConfig.SourceType != nil { + if *kubernetesConfig.SourceType != "all" { // if is all, sources is already set by default + sources = []string{*kubernetesConfig.SourceType} } } - // By default source type is set to "all", which indicates querying the table will return all the deployed, helm and manifest resources. - // If the source type is explicitly set, other plugin will list resources based on that. For example: - // If set to "manifest", the table will only return the manifest resources. - // If set to "helm", the table will return the resources after rendering the templates defined in the configured chart. - // Similarly, setting the value as "deployed" will return all the deployed resources. - if source.String() == "manifest" || source.String() == "helm" { - plugin.Logger(ctx).Debug("getK8ConfigRaw", "Returning nil for API server client.", "Source type", source.String(), "connection", c.Name) + if !helpers.StringSliceContains(sources, "deployed") { + plugin.Logger(ctx).Debug("getK8ConfigRaw", "Returning nil for API server client.", "source_types", sources, "connection", c.Name) return nil, nil } @@ -794,25 +797,34 @@ func parsedManifestFileContentUncached(ctx context.Context, d *plugin.QueryData, // Returns the list of file paths/glob patterns after resolving all the given manifest file paths. func resolveManifestFilePaths(ctx context.Context, d *plugin.QueryData) ([]string, error) { // Read the config - k8sConfig := GetConfig(d.Connection) + kubernetesConfig := GetConfig(d.Connection) + + // Check for the sourceTypes argument in the config. Valid values are: "deployed", "manifest" and "helm". + // Default set to include values. + var sources = All.ToSourceTypes() + if kubernetesConfig.SourceTypes != nil { + sources = kubernetesConfig.SourceTypes + } + // TODO: Remove once `SourceType` is obsolete + if kubernetesConfig.SourceTypes == nil && kubernetesConfig.SourceType != nil { + if *kubernetesConfig.SourceType != "all" { // if is all, sources is already set by default + sources = []string{*kubernetesConfig.SourceType} + } + } - // Return nil, if the source_type is set other than "manifest", or "all" - if k8sConfig.SourceType != nil && - !helpers.StringSliceContains([]string{"all", "manifest"}, *k8sConfig.SourceType) { + if !helpers.StringSliceContains(sources, "manifest") { return nil, nil } - // Return error if source_tpe arg is explicitly set to "manifest" in the config, but + // Return error if source_types arg includes "manifest" in the config, but // manifest_file_paths arg is not set. - if k8sConfig.SourceType != nil && - *k8sConfig.SourceType == "manifest" && - k8sConfig.ManifestFilePaths == nil { - return nil, errors.New("manifest_file_paths must be set in the config while the source_type is 'manifest'") + if kubernetesConfig.ManifestFilePaths == nil { + return nil, errors.New("manifest_file_paths must be set in the config while the source_types includes 'manifest'") } // Gather file path matches for the glob var matches, resolvedPaths []string - paths := k8sConfig.ManifestFilePaths + paths := kubernetesConfig.ManifestFilePaths for _, i := range paths { // List the files in the given source directory