From 3c7dd800bf750f4c637b053e245d7afd421c9f48 Mon Sep 17 00:00:00 2001 From: Shinto C V Date: Sat, 25 May 2024 15:26:55 +0530 Subject: [PATCH] 706 feat: Add progress add color to progress add info at the end --- backup/backup.go | 52 +++++++++++++++++++++++++++++++++++++++++++++--- go.mod | 9 +++++++-- go.sum | 16 +++++++++++++++ zfs/zfs.go | 9 +++++++++ 4 files changed, 81 insertions(+), 5 deletions(-) diff --git a/backup/backup.go b/backup/backup.go index fc07f78..265f73b 100644 --- a/backup/backup.go +++ b/backup/backup.go @@ -38,8 +38,10 @@ import ( "github.com/cenkalti/backoff" "github.com/dustin/go-humanize" + "github.com/k0kubun/go-ansi" "github.com/miolini/datacounter" "github.com/nightlyone/lockfile" + "github.com/schollz/progressbar/v3" "golang.org/x/sync/errgroup" "github.com/someone1/zfsbackup-go/backends" @@ -320,6 +322,7 @@ func Backup(pctx context.Context, jobInfo *files.JobInfo) error { // Prepare backends and setup plumbing for _, destination := range jobInfo.Destinations { + log.AppLogger.Infof("Initializing backend for destination %s", destination) backend, berr := prepareBackend(ctx, jobInfo, destination, uploadBuffer) if berr != nil { log.AppLogger.Errorf("Could not initialize backend due to error - %v.", berr) @@ -416,16 +419,19 @@ func Backup(pctx context.Context, jobInfo *files.JobInfo) error { } else { fmt.Fprintf( config.Stdout, - "Done.\n\tTotal ZFS Stream Bytes: %d (%s)\n\tTotal Bytes Written: %d (%s)\n\tElapsed Time: %v\n\tTotal Files Uploaded: %d\n", + "Done.\n\tTotal ZFS Stream Bytes: %d (%s)\n\tTotal Bytes Written: %d (%s)\n\tElapsed Time: %v\n\tTotal Files Uploaded: %d\n\tAverage Upload Rate: %s\n", jobInfo.ZFSStreamBytes, humanize.IBytes(jobInfo.ZFSStreamBytes), totalWrittenBytes, humanize.IBytes(totalWrittenBytes), time.Since(jobInfo.StartTime), len(jobInfo.Volumes)+1, + fmt.Sprintf("%.2f TB/hr", float64(totalWrittenBytes)/1e12/time.Since(jobInfo.StartTime).Hours()), ) } + fmt.Printf("Backup of %s completed successfully.\n", jobInfo.VolumeName) + log.AppLogger.Debugf("Cleaning up resources...") for _, backend := range usedBackends { @@ -478,6 +484,7 @@ func saveManifest(ctx context.Context, j *files.JobInfo, final bool) (*files.Vol } // nolint:funlen,gocyclo // Difficult to break this apart + func sendStream(ctx context.Context, j *files.JobInfo, c chan<- *files.VolumeInfo, buffer <-chan bool) error { var group *errgroup.Group group, ctx = errgroup.WithContext(ctx) @@ -493,6 +500,39 @@ func sendStream(ctx context.Context, j *files.JobInfo, c chan<- *files.VolumeInf usingPipe = true } + // Get total dataset size for progress tracking + totalSize, err := zfs.GetDatasetSize(ctx, j.VolumeName) + if err != nil { + return err + } + + // Initialize progress bar + bar := progressbar.NewOptions64(int64(totalSize), + progressbar.OptionSetWriter(ansi.NewAnsiStdout()), + progressbar.OptionEnableColorCodes(true), + progressbar.OptionShowBytes(true), + progressbar.OptionSetWidth(50), + progressbar.OptionThrottle(65*time.Millisecond), + progressbar.OptionShowCount(), + progressbar.OptionSetDescription("[cyan]Backing up...[reset]"), + progressbar.OptionOnCompletion(func() { + fmt.Fprint(ansi.NewAnsiStdout(), "\n") + }), + progressbar.OptionSpinnerType(14), + progressbar.OptionFullWidth(), + progressbar.OptionSetTheme(progressbar.Theme{ + Saucer: "[green]=[reset]", + SaucerHead: "[green]>[reset]", + SaucerPadding: " ", + BarStart: "[", + BarEnd: "]", + }), + ) + + // Initialize chunk tracking variables + totalChunks := int(totalSize / (j.VolumeSize * humanize.MiByte)) + var processedChunks int + group.Go(func() error { var lastTotalBytes uint64 defer close(c) @@ -527,6 +567,8 @@ func sendStream(ctx context.Context, j *files.JobInfo, c chan<- *files.VolumeInf if !usingPipe { c <- volume } + processedChunks++ + bar.Describe(fmt.Sprintf("Backing up... (%d/%d chunks)", processedChunks, totalChunks)) } <-buffer volume, err = files.CreateBackupVolume(ctx, j, volNum) @@ -542,7 +584,7 @@ func sendStream(ctx context.Context, j *files.JobInfo, c chan<- *files.VolumeInf } // Write a little at a time and break the output between volumes as needed - _, ierr := io.CopyN(volume, counter, files.BufferSize*2) + bytesWritten, ierr := io.CopyN(volume, counter, files.BufferSize*2) if ierr == io.EOF { // We are done! log.AppLogger.Debugf("Finished creating volume %s", volume.ObjectName) @@ -554,17 +596,21 @@ func sendStream(ctx context.Context, j *files.JobInfo, c chan<- *files.VolumeInf if !usingPipe { c <- volume } + processedChunks++ + bar.Describe(fmt.Sprintf("Backing up... (%d/%d chunks)", processedChunks, totalChunks)) return nil } else if ierr != nil { log.AppLogger.Errorf("Error while trying to read from the zfs stream for volume %s - %v", volume.ObjectName, ierr) return ierr } + // Update progress bar + bar.Add64(int64(bytesWritten)) } }) // Start the zfs send command log.AppLogger.Infof("Starting zfs send command: %s", strings.Join(cmd.Args, " ")) - err := cmd.Start() + err = cmd.Start() if err != nil { log.AppLogger.Errorf("Error starting zfs command - %v", err) return err diff --git a/go.mod b/go.mod index 5e3421d..71cb059 100644 --- a/go.mod +++ b/go.mod @@ -47,15 +47,20 @@ require ( github.com/googleapis/gax-go/v2 v2.5.1 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect github.com/klauspost/compress v1.15.10 // indirect github.com/kr/fs v0.1.0 // indirect github.com/mattn/go-ieproxy v0.0.9 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/schollz/progressbar/v3 v3.14.3 // indirect github.com/spf13/pflag v1.0.5 // indirect go.opencensus.io v0.23.0 // indirect golang.org/x/net v0.0.0-20220921203646-d300de134e69 // indirect golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 // indirect - golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 // indirect - golang.org/x/term v0.0.0-20220919170432-7a66f970e087 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/term v0.20.0 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index c928266..c6ab3d8 100644 --- a/go.sum +++ b/go.sum @@ -252,6 +252,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI= github.com/juju/ratelimit v1.0.2/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.10 h1:Ai8UzuomSCDw90e1qNMtb15msBXsNpH6gzkkENQNcJo= github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= @@ -268,8 +270,12 @@ github.com/kurin/blazer v0.5.3/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-ieproxy v0.0.9 h1:RvVbLiMv/Hbjf1gRaC2AQyzwbdVhdId7D2vPnXIml4k= github.com/mattn/go-ieproxy v0.0.9/go.mod h1:eF30/rfdQUO9EnzNIZQr0r9HiLMlZNCpJkHbmMuOAE0= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/miolini/datacounter v1.0.3 h1:tanOZPVblGXQl7/bSZWoEM8l4KK83q24qwQLMrO/HOA= github.com/miolini/datacounter v1.0.3/go.mod h1:C45dc2hBumHjDpEU64IqPwR6TDyPVpzOqqRTN7zmBUA= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= @@ -284,15 +290,20 @@ github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfx github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/schollz/progressbar/v3 v3.14.3 h1:oOuWW19ka12wxYU1XblR4n16wF/2Y1dBLMarMo6p4xU= +github.com/schollz/progressbar/v3 v3.14.3/go.mod h1:aT3UQ7yGm+2ZjeXPqsjTenwL3ddUiuZ0kfQ/2tHlyNI= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -506,10 +517,15 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 h1:h+EGohizhe9XlX18rfpa8k8RAc5XyaeamM+0VHRd4lc= golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220919170432-7a66f970e087 h1:tPwmk4vmvVCMdr98VgL4JH+qZxPL8fqlUOHnyOM8N3w= golang.org/x/term v0.0.0-20220919170432-7a66f970e087/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/zfs/zfs.go b/zfs/zfs.go index cf7a224..9f00199 100644 --- a/zfs/zfs.go +++ b/zfs/zfs.go @@ -110,6 +110,15 @@ func GetZFSProperty(ctx context.Context, prop, target string) (string, error) { return strings.TrimSpace(b.String()), nil } +// GetDatasetSize returns the size of the given ZFS dataset. +func GetDatasetSize(ctx context.Context, dataset string) (uint64, error) { + sizeStr, err := GetZFSProperty(ctx, "used", dataset) + if err != nil { + return 0, err + } + return strconv.ParseUint(sizeStr, 10, 64) +} + // GetZFSSendCommand will return the send command to use for the given JobInfo func GetZFSSendCommand(ctx context.Context, j *files.JobInfo) *exec.Cmd { // Prepare the zfs send command