Skip to content

Commit

Permalink
upgrade mpb to v8 (#425)
Browse files Browse the repository at this point in the history
upgrade mpb to v8
  • Loading branch information
bakito authored Jul 22, 2024
1 parent ee32968 commit 01f4260
Show file tree
Hide file tree
Showing 5 changed files with 91 additions and 70 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ require (
github.com/spf13/cobra v1.8.1
github.com/spf13/pflag v1.0.5
github.com/vardius/worker-pool/v2 v2.1.0
github.com/vbauerster/mpb/v6 v6.0.4
github.com/vbauerster/mpb/v8 v8.7.4
go.uber.org/mock v0.4.0
golang.org/x/term v0.22.0
k8s.io/api v0.30.3
Expand Down
7 changes: 2 additions & 5 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,6 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
Expand Down Expand Up @@ -129,7 +128,6 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
Expand All @@ -154,8 +152,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/vardius/worker-pool/v2 v2.1.0 h1:3EICiE8FleVdHAWSPrJCmEA1Ikb1BT3blb32hcaDO2I=
github.com/vardius/worker-pool/v2 v2.1.0/go.mod h1:mWRxoOWVI4OgnOWv1Fj0U+JqCNO4xUBd2H6JoUIE58o=
github.com/vbauerster/mpb/v6 v6.0.4 h1:h6J5zM/2wimP5Hj00unQuV8qbo5EPcj6wbkCqgj7KcY=
github.com/vbauerster/mpb/v6 v6.0.4/go.mod h1:a/+JT57gqh6Du0Ay5jSR+uBMfXGdlR7VQlGP52fJxLM=
github.com/vbauerster/mpb/v8 v8.7.4 h1:p4f16iMfUt3PkAC73SCzAtgtSf8TYDqEbJUT3odPrPo=
github.com/vbauerster/mpb/v8 v8.7.4/go.mod h1:r1B5k2Ljj5KJFCekfihbiqyV4VaaRTANYmvWA2btufI=
github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=
github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
Expand Down Expand Up @@ -187,7 +185,6 @@ golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
Expand Down
6 changes: 3 additions & 3 deletions pkg/export/export.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ import (
"github.com/bakito/kubexporter/pkg/render"
"github.com/bakito/kubexporter/pkg/types"
"github.com/bakito/kubexporter/version"
"github.com/vbauerster/mpb/v6"
"github.com/vbauerster/mpb/v6/decor"
"github.com/vbauerster/mpb/v8"
"github.com/vbauerster/mpb/v8/decor"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
)
Expand Down Expand Up @@ -71,7 +71,7 @@ func (e *exporter) Export() error {
mainBar = prog.AddBar(int64(len(resources)),
mpb.PrependDecorators(
// display our name with one space on the right
decor.Name("Resources", decor.WC{W: len("Resources") + 1, C: decor.DidentRight}),
decor.Name("Resources", decor.WC{W: len("Resources") + 1, C: decor.DindentRight}),
decor.Elapsed(decor.ET_STYLE_GO),
),
mpb.AppendDecorators(
Expand Down
69 changes: 69 additions & 0 deletions pkg/export/worker/bar.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package worker

import (
"fmt"

"github.com/vbauerster/mpb/v8"
"github.com/vbauerster/mpb/v8/decor"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)

func (w *worker) newSearchBar() {
if w.prog != nil {
prevBar := w.resourceBar
w.resourceBar = w.prog.AddBar(1,
mpb.PrependDecorators(
w.preDecoratorSearch(),
),
mpb.AppendDecorators(
w.postDecorator(),
),
mpb.BarQueueAfter(prevBar),
)
}
}

func (w *worker) newExportBar(ul *unstructured.UnstructuredList) {
if w.resourceBar != nil && len(ul.Items) > 0 {
prevBar := w.resourceBar
w.resourceBar = w.prog.AddBar(int64(len(ul.Items)),
mpb.PrependDecorators(
w.preDecoratorExport(),
),
mpb.AppendDecorators(
w.postDecorator(),
),
mpb.BarQueueAfter(prevBar),
)
}
}

func (w *worker) preDecoratorSearch() decor.Decorator {
return decor.Any(func(s decor.Statistics) string {
page := ""
if w.config.QueryPageSize > 0 {
page = fmt.Sprintf(" (page %d)", w.currentPage)
}
return fmt.Sprintf("🔍 %2d: %s%s ", w.id, w.currentKind, page)
})
}

func (w *worker) preDecoratorExport() decor.Decorator {
return decor.Any(func(s decor.Statistics) string {
page := ""
if w.config.QueryPageSize > 0 {
page = fmt.Sprintf(" (page %d)", w.currentPage)
}
d, _ := w.elapsedDecorator.Decor(s)
return fmt.Sprintf("👷 %2d: %s%s %s", w.id, w.currentKind, page, d)
})
}

func (w *worker) postDecorator() decor.Decorator {
return decor.Any(func(s decor.Statistics) string {
d1, _ := decor.CurrentNoUnit("").Decor(s)
d2, _ := decor.TotalNoUnit("").Decor(s)
d3, _ := decor.Percentage().Decor(s)
return fmt.Sprintf("%s / %s %s", d1, d2, d3)
})
}
77 changes: 16 additions & 61 deletions pkg/export/worker/worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,10 @@ import (
"time"

"github.com/bakito/kubexporter/pkg/client"
"github.com/bakito/kubexporter/pkg/log"
"github.com/bakito/kubexporter/pkg/types"
"github.com/bakito/kubexporter/pkg/utils"
"github.com/vbauerster/mpb/v6"
"github.com/vbauerster/mpb/v6/decor"
"github.com/vbauerster/mpb/v8"
"github.com/vbauerster/mpb/v8/decor"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -34,7 +33,8 @@ type worker struct {
id int
config *types.Config
mainBar *mpb.Bar
recBar *mpb.Bar
resourceBar *mpb.Bar
prog *mpb.Progress
currentKind string
currentPage int
elapsedDecorator decor.Decorator
Expand Down Expand Up @@ -90,26 +90,14 @@ func New(id int, config *types.Config, ac *client.ApiClient, prog *mpb.Progress,
config: config,
ac: ac,
elapsedDecorator: decor.NewElapsed(decor.ET_STYLE_GO, time.Now()),
prog: prog,
}

if prog != nil {
w.recBar = prog.AddBar(1,
mpb.PrependDecorators(
w.preDecorator(),
),
mpb.AppendDecorators(
w.postDecorator(),
),
)
}
return w
}

// Stop end worker
func (w *worker) Stop() Stats {
if w.recBar != nil {
w.recBar.SetTotal(100, true)
}
return w.stats
}

Expand All @@ -130,43 +118,12 @@ func (w *worker) list(ctx context.Context, group, version, kind string, continue
}
opts := metav1.ListOptions{Continue: continueValue}
if !w.config.AsLists {
// for lists we do no pagination
// for lists, we do no pagination
opts.Limit = int64(w.config.QueryPageSize)
}
return dr.List(ctx, opts)
}

func (w *worker) preDecorator() decor.Decorator {
return decor.Any(func(s decor.Statistics) string {
if s.Completed {
return fmt.Sprintf("👷 %2d:", w.id)
}
if w.queryFinished && s.Total == 0 {
return fmt.Sprintf("\U0001F971 %2d: idle", w.id)
}
page := ""
if w.config.QueryPageSize > 0 {
page = fmt.Sprintf(" (page %d)", w.currentPage)
}
if !w.queryFinished {
return fmt.Sprintf("🔍 %2d: %s%s", w.id, w.currentKind, page)
}
return fmt.Sprintf("👷 %2d: %s%s %s", w.id, w.currentKind, page, w.elapsedDecorator.Decor(s))
})
}

func (w *worker) postDecorator() decor.Decorator {
return decor.Any(func(s decor.Statistics) string {
if s.Completed {
return log.Check
}
return fmt.Sprintf("%s / %s %s",
decor.CurrentNoUnit("").Decor(s),
decor.TotalNoUnit("").Decor(s),
decor.Percentage().Decor(s))
})
}

// GenerateWork generate the work function
func (w *worker) GenerateWork(wg *sync.WaitGroup, out chan *types.GroupResource) func(resource *types.GroupResource) {
return func(res *types.GroupResource) {
Expand Down Expand Up @@ -200,13 +157,14 @@ func (w *worker) GenerateWork(wg *sync.WaitGroup, out chan *types.GroupResource)

func (w *worker) listResources(ctx context.Context, res *types.GroupResource, hasMorePages string) string {
w.currentPage = res.Pages + 1
if w.recBar != nil {
w.recBar.SetCurrent(0)
w.recBar.SetTotal(0, false)
}
w.newSearchBar()
start := time.Now()
ul, err := w.list(ctx, res.APIGroup, res.APIVersion, res.APIResource.Kind, hasMorePages)

if w.resourceBar != nil {
w.resourceBar.IncrBy(1)
}

res.QueryDuration += time.Since(start)
w.queryFinished = true
start = time.Now()
Expand All @@ -221,9 +179,7 @@ func (w *worker) listResources(ctx context.Context, res *types.GroupResource, ha
res.Error = "Error:" + err.Error()
}
} else {
if w.recBar != nil {
w.recBar.SetTotal(int64(len(ul.Items)), false)
}
w.newExportBar(ul)
if w.config.AsLists {
res.ExportedInstances += w.exportLists(res, ul)
} else {
Expand Down Expand Up @@ -288,8 +244,8 @@ func (w *worker) exportLists(res *types.GroupResource, ul *unstructured.Unstruct
}
closeIgnoreError(f)()

if w.recBar != nil {
w.recBar.IncrBy(len(usl.Items))
if w.resourceBar != nil {
w.resourceBar.IncrBy(len(usl.Items))
}
cnt += len(usl.Items)
}
Expand All @@ -305,7 +261,6 @@ func (w *worker) exportSingleResources(res *types.GroupResource, ul *unstructure
for _, u := range ul.Items {
if !w.config.IsInstanceExcluded(res, u) {
cnt++
time.Sleep(time.Millisecond * 100)
w.stats.addNamespace(u.GetNamespace())
w.config.FilterFields(res, u)
w.config.MaskFields(res, u)
Expand Down Expand Up @@ -341,8 +296,8 @@ func (w *worker) exportSingleResources(res *types.GroupResource, ul *unstructure
closeIgnoreError(f)
}

if w.recBar != nil {
w.recBar.Increment()
if w.resourceBar != nil {
w.resourceBar.Increment()
}
}
return cnt
Expand Down

0 comments on commit 01f4260

Please sign in to comment.