-
Notifications
You must be signed in to change notification settings - Fork 152
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: Parse kopia snapshot restore
progress output.
#2776
Changes from 2 commits
f7fad5f
7e4a04c
466fc65
f932ade
fc19368
cfefd78
222b34b
2547a8d
92c4a3a
a9c5dfb
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
|
@@ -39,6 +39,7 @@ const ( | |||||
|
||||||
//nolint:lll | ||||||
snapshotCreateOutputRegEx = `(?P<spinner>[|/\-\\\*]).+[^\d](?P<numHashed>\d+) hashed \((?P<hashedSize>[^\)]+)\), (?P<numCached>\d+) cached \((?P<cachedSize>[^\)]+)\), uploaded (?P<uploadedSize>[^\)]+), (?:estimating...|estimated (?P<estimatedSize>[^\)]+) \((?P<estimatedProgress>[^\)]+)\%\).+)` | ||||||
snapshotRestoreOutputRegEx = `Processed (?P<processedCount>\d+) \((?P<processedSize>.*)\) of (?P<totalCount>\d+) \((?P<totalSize>.*)\) (?P<dataRate>.*) \((?P<percentage>.*)%\) remaining (?P<remainingTime>.*)\.` | ||||||
extractSnapshotIDRegEx = `Created snapshot with root ([^\s]+) and ID ([^\s]+).*$` | ||||||
repoTotalSizeFromBlobStatsRegEx = `Total: (\d+)$` | ||||||
repoCountFromBlobStatsRegEx = `Count: (\d+)$` | ||||||
|
@@ -205,7 +206,10 @@ type SnapshotCreateStats struct { | |||||
ProgressPercent int64 | ||||||
} | ||||||
|
||||||
var kopiaProgressPattern = regexp.MustCompile(snapshotCreateOutputRegEx) //nolint:lll | ||||||
var ( | ||||||
kopiaProgressPattern = regexp.MustCompile(snapshotCreateOutputRegEx) | ||||||
kopiaSnapshotRestorePattern = regexp.MustCompile(snapshotRestoreOutputRegEx) | ||||||
) | ||||||
|
||||||
// SnapshotStatsFromSnapshotCreate parses the output of a kopia snapshot | ||||||
// create execution for a log of the stats for that execution. | ||||||
|
@@ -327,6 +331,93 @@ func parseKopiaProgressLine(line string, matchOnlyFinished bool) (stats *Snapsho | |||||
} | ||||||
} | ||||||
|
||||||
// SnapshotRestoreStats is a container for stats parsed from the output of a | ||||||
// `kopia snapshot restore` command. | ||||||
type SnapshotRestoreStats struct { | ||||||
FilesProcessed int64 | ||||||
SizeProcessedB int64 | ||||||
FilesTotal int64 | ||||||
SizeTotalB int64 | ||||||
ProgressPercent int64 | ||||||
} | ||||||
|
||||||
// SnapshotStatsFromSnapshotRestore parses the output of a `kopia snapshot | ||||||
// restore` execution for a log of the stats for that execution. | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. sorry, I was not able to understand this comment properly, especially the part There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The comment was done in same styling (wording) as in similar function: In both cases it means that we have output of some command, we are parsing it and trying to find stats logged by command. Possible rephrasing (for both cases) : WDYT ? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. sounds good. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Replaced for both |
||||||
func SnapshotStatsFromSnapshotRestore(snapRestoreStderrOutput string) (stats *SnapshotRestoreStats) { | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
maybe this or similar? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I've renamed it to |
||||||
if snapRestoreStderrOutput == "" { | ||||||
return nil | ||||||
} | ||||||
logs := regexp.MustCompile("[\r\n]").Split(snapRestoreStderrOutput, -1) | ||||||
|
||||||
for _, l := range logs { | ||||||
lineStats := parseKopiaSnapshotRestoreProgressLine(l) | ||||||
if lineStats != nil { | ||||||
stats = lineStats | ||||||
} | ||||||
} | ||||||
|
||||||
return stats | ||||||
} | ||||||
|
||||||
func parseKopiaSnapshotRestoreProgressLine(line string) (stats *SnapshotRestoreStats) { | ||||||
match := kopiaSnapshotRestorePattern.FindStringSubmatch(line) | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think it would be a good idea if we can also share (in comments) how exactly this line looks like. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done. |
||||||
if len(match) < 8 { | ||||||
return nil | ||||||
} | ||||||
|
||||||
groups := make(map[string]string) | ||||||
for i, name := range kopiaSnapshotRestorePattern.SubexpNames() { | ||||||
if i != 0 && name != "" { | ||||||
groups[name] = match[i] | ||||||
} | ||||||
} | ||||||
|
||||||
processedCount, err := strconv.Atoi(groups["processedCount"]) | ||||||
if err != nil { | ||||||
log.WithError(err).Print("Skipping entry due to inability to parse number of processed files", field.M{"processedCount": groups["processedCount"]}) | ||||||
return nil | ||||||
} | ||||||
|
||||||
processedSize, err := humanize.ParseBytes(groups["processedSize"]) | ||||||
if err != nil { | ||||||
log.WithError(err).Print("Skipping entry due to inability to parse amount of processed bytes", field.M{"processedSize": groups["processedSize"]}) | ||||||
return nil | ||||||
} | ||||||
|
||||||
totalCount, err := strconv.Atoi(groups["totalCount"]) | ||||||
if err != nil { | ||||||
log.WithError(err).Print("Skipping entry due to inability to parse expected number of files", field.M{"totalCount": groups["totalCount"]}) | ||||||
return nil | ||||||
} | ||||||
|
||||||
totalSize, err := humanize.ParseBytes(groups["totalSize"]) | ||||||
if err != nil { | ||||||
log.WithError(err).Print("Skipping entry due to inability to parse expected amount of bytes", field.M{"totalSize": groups["totalSize"]}) | ||||||
return nil | ||||||
} | ||||||
|
||||||
progressPercent, err := strconv.ParseFloat(groups["percentage"], 64) | ||||||
if err != nil { | ||||||
log.WithError(err).Print("Skipping entry due to inability to parse progress percent string", field.M{"progressPercent": groups["progressPercent"]}) | ||||||
return nil | ||||||
} | ||||||
|
||||||
if progressPercent >= 100 { | ||||||
// It may happen that kopia reports progress of 100 or higher without actual | ||||||
e-sumin marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||
// completing the task. This can occur due to inaccurate estimation. | ||||||
// In such case, we will return the progress as 99% to avoid confusion. | ||||||
e-sumin marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||
progressPercent = 99 | ||||||
} | ||||||
|
||||||
return &SnapshotRestoreStats{ | ||||||
FilesProcessed: int64(processedCount), | ||||||
SizeProcessedB: int64(processedSize), | ||||||
FilesTotal: int64(totalCount), | ||||||
SizeTotalB: int64(totalSize), | ||||||
ProgressPercent: int64(progressPercent), | ||||||
} | ||||||
} | ||||||
|
||||||
// RepoSizeStatsFromBlobStatsRaw takes a string as input, interprets it as a kopia blob stats | ||||||
// output in an expected format (Contains the line "Total: <size>"), and returns the integer | ||||||
// size in bytes or an error if parsing is unsuccessful. | ||||||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
can it just be called
RestoreStats
? i quickly checked what kopia restore command is called and itskopia restore
so I think justRestoreStats
should also be ok.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
But please feel free to disagree.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Actually I was always using
kopia snapshot restore
but I've just checked documentation, and you are right. Seems thatsnapshot restore
is an alias torestore
. So your suggestion makes sense.