From 18735188d224b2452651489878a9dfb0e3b1c515 Mon Sep 17 00:00:00 2001 From: Cenk Alti Date: Fri, 26 May 2023 17:06:50 -0400 Subject: [PATCH] cli: Add etcdutl snapshot hashkv command Signed-off-by: Cenk Alti --- etcdutl/README.md | 38 +++++++++++++++++++++++++++++ etcdutl/etcdutl/printer.go | 12 +++++++++ etcdutl/etcdutl/printer_fields.go | 6 +++++ etcdutl/etcdutl/printer_json.go | 1 + etcdutl/etcdutl/printer_simple.go | 7 ++++++ etcdutl/etcdutl/printer_table.go | 11 +++++++++ etcdutl/etcdutl/snapshot_command.go | 31 +++++++++++++++++++++++ etcdutl/snapshot/v3_snapshot.go | 28 +++++++++++++++++++++ server/storage/mvcc/hash.go | 2 +- server/storage/mvcc/hash_test.go | 4 +-- server/storage/mvcc/kvstore.go | 2 +- server/storage/mvcc/kvstore_test.go | 2 +- 12 files changed, 139 insertions(+), 5 deletions(-) diff --git a/etcdutl/README.md b/etcdutl/README.md index 6943a047858a..c4ce0680ee98 100644 --- a/etcdutl/README.md +++ b/etcdutl/README.md @@ -115,6 +115,44 @@ Prints a line of JSON encoding the database hash, revision, total keys, and size +----------+----------+------------+------------+ ``` +### SNAPSHOT HASHKV [options] \ + +SNAPSHOT HASHKV prints hash of keys and values up to given revision. + +#### Options + +- rev -- Revision number. Default is 0 which means the latest revision. + +#### Output + +##### Simple format + +Prints a humanized table of the KV hash, hash revision and compact revision. + +##### JSON format + +Prints a line of JSON encoding the KV hash, hash revision and compact revision. + +#### Examples +```bash +./etcdutl snapshot hashkv file.db +# 35c86e9b, 214, 150 +``` + +```bash +./etcdutl --write-out=json snapshot hashkv file.db +# {"hash":902327963,"hashRevision":214,"compactRevision":150} +``` + +```bash +./etcdutl --write-out=table snapshot hashkv file.db ++----------+---------------+------------------+ +| HASH | HASH REVISION | COMPACT REVISION | ++----------+---------------+------------------+ +| 35c86e9b | 214 | 150 | ++----------+---------------+------------------+ +``` + ### VERSION Prints the version of etcdutl. diff --git a/etcdutl/etcdutl/printer.go b/etcdutl/etcdutl/printer.go index 7d65366065ff..dfdc723062f5 100644 --- a/etcdutl/etcdutl/printer.go +++ b/etcdutl/etcdutl/printer.go @@ -32,6 +32,7 @@ var ( type printer interface { DBStatus(snapshot.Status) + DBHashKV(snapshot.HashKV) } func NewPrinter(printerType string) printer { @@ -65,6 +66,7 @@ func newPrinterUnsupported(n string) printer { } func (p *printerUnsupported) DBStatus(snapshot.Status) { p.p(nil) } +func (p *printerUnsupported) DBHashKV(snapshot.HashKV) { p.p(nil) } func makeDBStatusTable(ds snapshot.Status) (hdr []string, rows [][]string) { hdr = []string{"hash", "revision", "total keys", "total size", "version"} @@ -78,6 +80,16 @@ func makeDBStatusTable(ds snapshot.Status) (hdr []string, rows [][]string) { return hdr, rows } +func makeDBHashKVTable(ds snapshot.HashKV) (hdr []string, rows [][]string) { + hdr = []string{"hash", "hash revision", "compact revision"} + rows = append(rows, []string{ + fmt.Sprintf("%x", ds.Hash), + fmt.Sprint(ds.HashRevision), + fmt.Sprint(ds.CompactRevision), + }) + return hdr, rows +} + func initPrinterFromCmd(cmd *cobra.Command) (p printer) { outputType, err := cmd.Flags().GetString("write-out") if err != nil { diff --git a/etcdutl/etcdutl/printer_fields.go b/etcdutl/etcdutl/printer_fields.go index d534e396ffe6..46c4f3e5c718 100644 --- a/etcdutl/etcdutl/printer_fields.go +++ b/etcdutl/etcdutl/printer_fields.go @@ -29,3 +29,9 @@ func (p *fieldsPrinter) DBStatus(r snapshot.Status) { fmt.Println(`"Size" :`, r.TotalSize) fmt.Println(`"Version" :`, r.Version) } + +func (p *fieldsPrinter) DBHashKV(r snapshot.HashKV) { + fmt.Println(`"Hash" :`, r.Hash) + fmt.Println(`"Hash revision" :`, r.HashRevision) + fmt.Println(`"Compact revision" :`, r.CompactRevision) +} diff --git a/etcdutl/etcdutl/printer_json.go b/etcdutl/etcdutl/printer_json.go index 38fe3e4548e3..6fee1a97fc0d 100644 --- a/etcdutl/etcdutl/printer_json.go +++ b/etcdutl/etcdutl/printer_json.go @@ -33,6 +33,7 @@ func newJSONPrinter() printer { } func (p *jsonPrinter) DBStatus(r snapshot.Status) { printJSON(r) } +func (p *jsonPrinter) DBHashKV(r snapshot.HashKV) { printJSON(r) } // !!! Share ?? func printJSON(v interface{}) { diff --git a/etcdutl/etcdutl/printer_simple.go b/etcdutl/etcdutl/printer_simple.go index 306ebf0c7f35..eeb8d84dd64a 100644 --- a/etcdutl/etcdutl/printer_simple.go +++ b/etcdutl/etcdutl/printer_simple.go @@ -30,3 +30,10 @@ func (s *simplePrinter) DBStatus(ds snapshot.Status) { fmt.Println(strings.Join(row, ", ")) } } + +func (s *simplePrinter) DBHashKV(ds snapshot.HashKV) { + _, rows := makeDBHashKVTable(ds) + for _, row := range rows { + fmt.Println(strings.Join(row, ", ")) + } +} diff --git a/etcdutl/etcdutl/printer_table.go b/etcdutl/etcdutl/printer_table.go index 2f8f81d4e6a7..78a389115d58 100644 --- a/etcdutl/etcdutl/printer_table.go +++ b/etcdutl/etcdutl/printer_table.go @@ -34,3 +34,14 @@ func (tp *tablePrinter) DBStatus(r snapshot.Status) { table.SetAlignment(tablewriter.ALIGN_RIGHT) table.Render() } + +func (tp *tablePrinter) DBHashKV(r snapshot.HashKV) { + hdr, rows := makeDBHashKVTable(r) + table := tablewriter.NewWriter(os.Stdout) + table.SetHeader(hdr) + for _, row := range rows { + table.Append(row) + } + table.SetAlignment(tablewriter.ALIGN_RIGHT) + table.Render() +} diff --git a/etcdutl/etcdutl/snapshot_command.go b/etcdutl/etcdutl/snapshot_command.go index 28df31f8dd02..b631ae873418 100644 --- a/etcdutl/etcdutl/snapshot_command.go +++ b/etcdutl/etcdutl/snapshot_command.go @@ -38,6 +38,7 @@ var ( restorePeerURLs string restoreName string skipHashCheck bool + hashKVRevision int64 ) // NewSnapshotCommand returns the cobra command for "snapshot". @@ -48,6 +49,7 @@ func NewSnapshotCommand() *cobra.Command { } cmd.AddCommand(NewSnapshotRestoreCommand()) cmd.AddCommand(newSnapshotStatusCommand()) + cmd.AddCommand(newSnapshotHashKVCommand()) return cmd } @@ -62,6 +64,19 @@ The items in the lists are hash, revision, total keys, total size. } } +func newSnapshotHashKVCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "hashkv ", + Short: "Prints the KV history hash of a given file", + Long: `When --write-out is set to simple, this command prints out comma-separated status lists for each endpoint. +The items in the lists are hash, hash revision, compact revision. +`, + Run: SnapshotHashKVCommandFunc, + } + cmd.Flags().Int64Var(&hashKVRevision, "rev", 0, "maximum revision to hash (default: all revisions)") + return cmd +} + func NewSnapshotRestoreCommand() *cobra.Command { cmd := &cobra.Command{ Use: "restore --data-dir {output dir} [options]", @@ -98,6 +113,22 @@ func SnapshotStatusCommandFunc(cmd *cobra.Command, args []string) { printer.DBStatus(ds) } +func SnapshotHashKVCommandFunc(cmd *cobra.Command, args []string) { + if len(args) != 1 { + err := fmt.Errorf("snapshot hashkv requires exactly one argument") + cobrautl.ExitWithError(cobrautl.ExitBadArgs, err) + } + printer := initPrinterFromCmd(cmd) + + lg := GetLogger() + sp := snapshot.NewV3(lg) + ds, err := sp.HashKV(args[0], hashKVRevision) + if err != nil { + cobrautl.ExitWithError(cobrautl.ExitError, err) + } + printer.DBHashKV(ds) +} + func snapshotRestoreCommandFunc(_ *cobra.Command, args []string) { SnapshotRestoreCommandFunc(restoreCluster, restoreClusterToken, restoreDataDir, restoreWalDir, restorePeerURLs, restoreName, skipHashCheck, args) diff --git a/etcdutl/snapshot/v3_snapshot.go b/etcdutl/snapshot/v3_snapshot.go index 8958ba80da13..e8058b811bc7 100644 --- a/etcdutl/snapshot/v3_snapshot.go +++ b/etcdutl/snapshot/v3_snapshot.go @@ -41,6 +41,7 @@ import ( "go.etcd.io/etcd/server/v3/etcdserver/api/v2store" "go.etcd.io/etcd/server/v3/etcdserver/cindex" "go.etcd.io/etcd/server/v3/storage/backend" + "go.etcd.io/etcd/server/v3/storage/mvcc" "go.etcd.io/etcd/server/v3/storage/schema" "go.etcd.io/etcd/server/v3/storage/wal" "go.etcd.io/etcd/server/v3/storage/wal/walpb" @@ -63,6 +64,9 @@ type Manager interface { // Status returns the snapshot file information. Status(dbPath string) (Status, error) + // HashKV returns the hash of keys and values up to specified revision (0 means all). + HashKV(dbPath string, rev int64) (HashKV, error) + // Restore restores a new etcd data directory from given snapshot // file. It returns an error if specified data directory already // exists, to prevent unintended data directory overwrites. @@ -110,6 +114,30 @@ type Status struct { Version string `json:"version"` } +type HashKV struct { + Hash uint32 `json:"hash"` + HashRevision int64 `json:"hashRevision"` + CompactRevision int64 `json:"compactRevision"` +} + +func (s *v3Manager) HashKV(dbPath string, rev int64) (ds HashKV, err error) { + cfg := backend.DefaultBackendConfig(zap.NewNop()) + cfg.Path = dbPath + b := backend.New(cfg) + st := mvcc.NewStore(zap.NewNop(), b, nil, mvcc.StoreConfig{}) + hst := mvcc.NewHashStorage(zap.NewNop(), st) + + h, _, err := hst.HashByRev(rev) + if err != nil { + return HashKV{}, err + } + return HashKV{ + Hash: h.Hash, + HashRevision: h.Revision, + CompactRevision: h.CompactRevision, + }, nil +} + // Status returns the snapshot file information. func (s *v3Manager) Status(dbPath string) (ds Status, err error) { if _, err = os.Stat(dbPath); err != nil { diff --git a/server/storage/mvcc/hash.go b/server/storage/mvcc/hash.go index 385d0c97966f..4456e575f5e7 100644 --- a/server/storage/mvcc/hash.go +++ b/server/storage/mvcc/hash.go @@ -111,7 +111,7 @@ type hashStorage struct { lg *zap.Logger } -func newHashStorage(lg *zap.Logger, s *store) *hashStorage { +func NewHashStorage(lg *zap.Logger, s *store) *hashStorage { return &hashStorage{ store: s, lg: lg, diff --git a/server/storage/mvcc/hash_test.go b/server/storage/mvcc/hash_test.go index d906d41f17ad..4d38ee49da72 100644 --- a/server/storage/mvcc/hash_test.go +++ b/server/storage/mvcc/hash_test.go @@ -178,7 +178,7 @@ func (tc hashTestCase) Compact(ctx context.Context, rev int64) error { func TestHasherStore(t *testing.T) { lg := zaptest.NewLogger(t) - s := newHashStorage(lg, newFakeStore(lg)) + s := NewHashStorage(lg, newFakeStore(lg)) defer s.store.Close() var hashes []KeyValueHash @@ -207,7 +207,7 @@ func TestHasherStore(t *testing.T) { func TestHasherStoreFull(t *testing.T) { lg := zaptest.NewLogger(t) - s := newHashStorage(lg, newFakeStore(lg)) + s := NewHashStorage(lg, newFakeStore(lg)) defer s.store.Close() var minRevision int64 = 100 diff --git a/server/storage/mvcc/kvstore.go b/server/storage/mvcc/kvstore.go index 8bc1b07d9978..f2cd3332c094 100644 --- a/server/storage/mvcc/kvstore.go +++ b/server/storage/mvcc/kvstore.go @@ -114,7 +114,7 @@ func NewStore(lg *zap.Logger, b backend.Backend, le lease.Lessor, cfg StoreConfi lg: lg, } - s.hashes = newHashStorage(lg, s) + s.hashes = NewHashStorage(lg, s) s.ReadView = &readView{s} s.WriteView = &writeView{s} if s.le != nil { diff --git a/server/storage/mvcc/kvstore_test.go b/server/storage/mvcc/kvstore_test.go index a3bdaac771ba..b35e8c0fa066 100644 --- a/server/storage/mvcc/kvstore_test.go +++ b/server/storage/mvcc/kvstore_test.go @@ -914,7 +914,7 @@ func newFakeStore(lg *zap.Logger) *store { lg: lg, } s.ReadView, s.WriteView = &readView{s}, &writeView{s} - s.hashes = newHashStorage(lg, s) + s.hashes = NewHashStorage(lg, s) return s }