feat: support --page-size and --page-buffer-size for snapshot exp…

…ort and save (#619) * feat: support -page-size and -page-buffer-size for snapshot save with k8s and snapshot export Signed-off-by: AhmedGrati <ahmedgrati1999@gmail.com> * feat: support -page-size and -page-buffer-size for snapshot save with k8s and snapshot export Signed-off-by: AhmedGrati <ahmedgrati1999@gmail.com> * docs: add new flags to the docs of kowkctl snapshot export and save Signed-off-by: AhmedGrati <ahmedgrati1999@gmail.com> --------- Signed-off-by: AhmedGrati <ahmedgrati1999@gmail.com>
kubernetes-sigs · May 31, 2023 · 4340fd4 · 4340fd4
1 parent 936cfef
commit 4340fd4
Show file tree

Hide file tree

Showing 4 changed files with 55 additions and 10 deletions.
diff --git a/pkg/kwokctl/cmd/snapshot/export/export.go b/pkg/kwokctl/cmd/snapshot/export/export.go
@@ -34,6 +34,8 @@ type flagpole struct {
 	Filters           []string
 	ImpersonateUser   string
 	ImpersonateGroups []string
+	PageSize          int64
+	PageBufferSize    int32
 }
 
 // NewCommand returns a new cobra.Command for cluster exporting.
@@ -53,6 +55,8 @@ func NewCommand(ctx context.Context) *cobra.Command {
 	cmd.Flags().StringSliceVar(&flags.Filters, "filter", snapshot.Resources, "Filter the resources to export")
 	cmd.Flags().StringVar(&flags.ImpersonateUser, "as", "", "Username to impersonate for the operation. User could be a regular user or a service account in a namespace.")
 	cmd.Flags().StringSliceVar(&flags.ImpersonateGroups, "as-group", nil, "Group to impersonate for the operation, this flag can be repeated to specify multiple groups.")
+	cmd.Flags().Int64Var(&flags.PageSize, "page-size", 500, "Define the page size")
+	cmd.Flags().Int32Var(&flags.PageBufferSize, "page-buffer-size", 10, "Define the number of pages to buffer")
 	return cmd
 }
 
@@ -79,7 +83,18 @@ func runE(ctx context.Context, flags *flagpole) error {
 		UserName: flags.ImpersonateUser,
 		Groups:   flags.ImpersonateGroups,
 	}
-	err = snapshot.Save(ctx, flags.Kubeconfig, file, flags.Filters, impersonateConfig)
+
+	pagerConfig := &snapshot.PagerConfig{
+		PageSize:       flags.PageSize,
+		PageBufferSize: flags.PageBufferSize,
+	}
+
+	snapshotSaveConfig := snapshot.SaveConfig{
+		PagerConfig:         pagerConfig,
+		ImpersonationConfig: impersonateConfig,
+	}
+
+	err = snapshot.Save(ctx, flags.Kubeconfig, file, flags.Filters, snapshotSaveConfig)
 	if err != nil {
 		return err
 	}

diff --git a/pkg/kwokctl/runtime/cluster_snapshot.go b/pkg/kwokctl/runtime/cluster_snapshot.go
@@ -38,7 +38,10 @@ func (c *Cluster) SnapshotSaveWithYAML(ctx context.Context, path string, filters
 	kubeconfigPath := c.GetWorkdirPath(InHostKubeconfigName)
 	// In most cases, the user should have full privileges on the clusters created by kwokctl,
 	// so no need to expose impersonation args to "snapshot save" command.
-	return snapshot.Save(ctx, kubeconfigPath, file, filters, rest.ImpersonationConfig{})
+	snapshotSaveConfig := snapshot.SaveConfig{
+		ImpersonationConfig: rest.ImpersonationConfig{},
+	}
+	return snapshot.Save(ctx, kubeconfigPath, file, filters, snapshotSaveConfig)
 }
 
 // SnapshotRestoreWithYAML restore the snapshot of cluster

diff --git a/pkg/kwokctl/snapshot/save.go b/pkg/kwokctl/snapshot/save.go
@@ -36,10 +36,24 @@ import (
 	"sigs.k8s.io/kwok/pkg/utils/yaml"
 )
 
+// PagerConfig is the configuration of the list pager.
+// It defines the page size and the page buffer size of the list pager.
+type PagerConfig struct {
+	PageSize       int64
+	PageBufferSize int32
+}
+
+// SaveConfig is the a combination of the impersonation config
+// and the PagerConfig.
+type SaveConfig struct {
+	PagerConfig         *PagerConfig
+	ImpersonationConfig rest.ImpersonationConfig
+}
+
 // Save saves the snapshot of cluster
-func Save(ctx context.Context, kubeconfigPath string, w io.Writer, resources []string, impersonateConfig rest.ImpersonationConfig) error {
+func Save(ctx context.Context, kubeconfigPath string, w io.Writer, resources []string, saveConfig SaveConfig) error {
 	clientset, err := client.NewClientset("", kubeconfigPath,
-		client.WithImpersonate(impersonateConfig),
+		client.WithImpersonate(saveConfig.ImpersonationConfig),
 	)
 	if err != nil {
 		return fmt.Errorf("failed to create clientset: %w", err)
@@ -89,6 +103,17 @@ func Save(ctx context.Context, kubeconfigPath string, w io.Writer, resources []s
 			return list, err
 		})
 
+		pagerConfig := saveConfig.PagerConfig
+
+		if pagerConfig != nil {
+			if pagerConfig.PageSize > 0 {
+				listPager.PageSize = pagerConfig.PageSize
+			}
+			if pagerConfig.PageBufferSize > 0 {
+				listPager.PageBufferSize = pagerConfig.PageBufferSize
+			}
+		}
+
 		count := 0
 		if err := listPager.EachListItem(ctx, metav1.ListOptions{}, func(obj runtime.Object) error {
 			if o, ok := obj.(metav1.Object); ok {

diff --git a/site/content/en/docs/generated/kwokctl_snapshot_export.md b/site/content/en/docs/generated/kwokctl_snapshot_export.md
@@ -9,12 +9,14 @@ kwokctl snapshot export [flags]
 ### Options
 
 ```
-      --as string           Username to impersonate for the operation. User could be a regular user or a service account in a namespace.
-      --as-group strings    Group to impersonate for the operation, this flag can be repeated to specify multiple groups.
-      --filter strings      Filter the resources to export (default [namespace,node,serviceaccount,configmap,secret,limitrange,runtimeclass.node.k8s.io,priorityclass.scheduling.k8s.io,clusterrolebindings.rbac.authorization.k8s.io,clusterroles.rbac.authorization.k8s.io,rolebindings.rbac.authorization.k8s.io,roles.rbac.authorization.k8s.io,daemonset.apps,deployment.apps,replicaset.apps,statefulset.apps,cronjob.batch,job.batch,persistentvolumeclaim,persistentvolume,pod,service,endpoints])
-  -h, --help                help for export
-      --kubeconfig string   Path to the kubeconfig file to use
-      --path string         Path to the snapshot
+      --as string                Username to impersonate for the operation. User could be a regular user or a service account in a namespace.
+      --as-group strings         Group to impersonate for the operation, this flag can be repeated to specify multiple groups.
+      --filter strings           Filter the resources to export (default [namespace,node,serviceaccount,configmap,secret,limitrange,runtimeclass.node.k8s.io,priorityclass.scheduling.k8s.io,clusterrolebindings.rbac.authorization.k8s.io,clusterroles.rbac.authorization.k8s.io,rolebindings.rbac.authorization.k8s.io,roles.rbac.authorization.k8s.io,daemonset.apps,deployment.apps,replicaset.apps,statefulset.apps,cronjob.batch,job.batch,persistentvolumeclaim,persistentvolume,pod,service,endpoints])
+  -h, --help                     help for export
+      --kubeconfig string        Path to the kubeconfig file to use
+      --page-buffer-size int32   Define the number of pages to buffer (default 10)
+      --page-size int            Define the page size (default 500)
+      --path string              Path to the snapshot
 ```
 
 ### Options inherited from parent commands