-
Notifications
You must be signed in to change notification settings - Fork 480
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This commit adds a `remotecat dump` command to the pebble tool. This shows the contents (creator ID and objects) of a remote object catalog. If the `--verbose` flag is passed, it also shows each record in the catalog file. Fixes #2721.
- Loading branch information
1 parent
7ef7553
commit 7a4ed3c
Showing
10 changed files
with
289 additions
and
25 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use | ||
| // of this source code is governed by a BSD-style license that can be found in | ||
| // the LICENSE file. | ||
|
|
||
| //go:build make_test_remotecat | ||
| // +build make_test_remotecat | ||
|
|
||
| // Run using: go run -tags make_test_remotecat ./tool/make_test_remotecat.go | ||
| package main | ||
|
|
||
| import ( | ||
| "log" | ||
| "os" | ||
| "path/filepath" | ||
|
|
||
| "github.com/cockroachdb/pebble/internal/base" | ||
| "github.com/cockroachdb/pebble/objstorage" | ||
| "github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat" | ||
| "github.com/cockroachdb/pebble/vfs" | ||
| ) | ||
|
|
||
| func main() { | ||
| dir, err := os.MkdirTemp("", "make-test-remotecat") | ||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| catalog, _, err := remoteobjcat.Open(vfs.Default, dir) | ||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| if err := catalog.SetCreatorID(3); err != nil { | ||
| log.Fatal(err) | ||
| } | ||
|
|
||
| var b remoteobjcat.Batch | ||
| b.AddObject(remoteobjcat.RemoteObjectMetadata{ | ||
| FileNum: base.FileNum(1).DiskFileNum(), | ||
| FileType: base.FileTypeTable, | ||
| CreatorID: 3, | ||
| CreatorFileNum: base.FileNum(1).DiskFileNum(), | ||
| CleanupMethod: objstorage.SharedRefTracking, | ||
| Locator: "foo", | ||
| }) | ||
| if err := catalog.ApplyBatch(b); err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| b.Reset() | ||
| b.AddObject(remoteobjcat.RemoteObjectMetadata{ | ||
| FileNum: base.FileNum(2).DiskFileNum(), | ||
| FileType: base.FileTypeTable, | ||
| CreatorID: 5, | ||
| CreatorFileNum: base.FileNum(10).DiskFileNum(), | ||
| CleanupMethod: objstorage.SharedRefTracking, | ||
| Locator: "foo", | ||
| }) | ||
| b.DeleteObject(base.FileNum(1).DiskFileNum()) | ||
| b.AddObject(remoteobjcat.RemoteObjectMetadata{ | ||
| FileNum: base.FileNum(3).DiskFileNum(), | ||
| FileType: base.FileTypeTable, | ||
| CleanupMethod: objstorage.SharedRefTracking, | ||
| Locator: "bar", | ||
| CustomObjectName: "external.sst", | ||
| }) | ||
| if err := catalog.ApplyBatch(b); err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| if err := catalog.Close(); err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| contents, err := os.ReadFile(filepath.Join(dir, "REMOTE-OBJ-CATALOG-000001")) | ||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| if err := os.WriteFile("tool/testdata/REMOTE-OBJ-CATALOG", contents, 0666); err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,132 @@ | ||
| // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use | ||
| // of this source code is governed by a BSD-style license that can be found in | ||
| // the LICENSE file. | ||
|
|
||
| package tool | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io" | ||
| "sort" | ||
|
|
||
| "github.com/cockroachdb/pebble" | ||
| "github.com/cockroachdb/pebble/internal/base" | ||
| "github.com/cockroachdb/pebble/objstorage" | ||
| "github.com/cockroachdb/pebble/objstorage/objstorageprovider/remoteobjcat" | ||
| "github.com/cockroachdb/pebble/record" | ||
| "github.com/spf13/cobra" | ||
| ) | ||
|
|
||
| // remoteCatalogT implements tools for the remote object catalog. | ||
| type remoteCatalogT struct { | ||
| Root *cobra.Command | ||
| Dump *cobra.Command | ||
|
|
||
| verbose bool | ||
| opts *pebble.Options | ||
| } | ||
|
|
||
| func newRemoteCatalog(opts *pebble.Options) *remoteCatalogT { | ||
| m := &remoteCatalogT{ | ||
| opts: opts, | ||
| } | ||
|
|
||
| m.Root = &cobra.Command{ | ||
| Use: "remotecat", | ||
| Short: "remote object catalog introspection tools", | ||
| } | ||
|
|
||
| // Add dump command | ||
| m.Dump = &cobra.Command{ | ||
| Use: "dump <remote-catalog-files>", | ||
| Short: "print remote object catalog contents", | ||
| Long: ` | ||
| Print the contents of the REMOTE-OBJ-CATALOG files. | ||
| `, | ||
| Args: cobra.MinimumNArgs(1), | ||
| Run: m.runDump, | ||
| } | ||
| m.Dump.Flags().BoolVarP(&m.verbose, "verbose", "v", false, "show each record in the catalog") | ||
| m.Root.AddCommand(m.Dump) | ||
|
|
||
| return m | ||
| } | ||
|
|
||
| func (m *remoteCatalogT) runDump(cmd *cobra.Command, args []string) { | ||
| for _, arg := range args { | ||
| err := m.runDumpOne(cmd.OutOrStdout(), arg) | ||
| if err != nil { | ||
| fmt.Fprintf(cmd.OutOrStderr(), "%s\n", err) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func (m *remoteCatalogT) runDumpOne(stdout io.Writer, filename string) error { | ||
| f, err := m.opts.FS.Open(filename) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| var creatorID objstorage.CreatorID | ||
| objects := make(map[base.DiskFileNum]remoteobjcat.RemoteObjectMetadata) | ||
|
|
||
| fmt.Fprintf(stdout, "%s\n", filename) | ||
| var editIdx int | ||
| rr := record.NewReader(f, 0 /* logNum */) | ||
| for { | ||
| offset := rr.Offset() | ||
| r, err := rr.Next() | ||
| if err == io.EOF { | ||
| break | ||
| } else if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| var ve remoteobjcat.VersionEdit | ||
| err = ve.Decode(r) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if m.verbose { | ||
| fmt.Fprintf(stdout, "%d/%d\n", offset, editIdx) | ||
| if ve.CreatorID.IsSet() { | ||
| fmt.Fprintf(stdout, " CreatorID: %s\n", ve.CreatorID) | ||
| } | ||
| if len(ve.NewObjects) > 0 { | ||
| fmt.Fprintf(stdout, " NewObjects:\n") | ||
| for _, m := range ve.NewObjects { | ||
| fmt.Fprintf( | ||
| stdout, " %s CreatorID: %s CreatorFileNum: %s Locator: %q CustomObjectName: %q\n", | ||
| m.FileNum, m.CreatorID, m.CreatorFileNum, m.Locator, m.CustomObjectName, | ||
| ) | ||
| } | ||
| } | ||
| if len(ve.DeletedObjects) > 0 { | ||
| fmt.Fprintf(stdout, " DeletedObjects:\n") | ||
| for _, n := range ve.DeletedObjects { | ||
| fmt.Fprintf(stdout, " %s\n", n) | ||
| } | ||
| } | ||
| } | ||
| editIdx++ | ||
| ve.Apply(&creatorID, objects) | ||
| } | ||
| fmt.Fprintf(stdout, "CreatorID: %v\n", creatorID) | ||
| var filenums []base.DiskFileNum | ||
| for n := range objects { | ||
| filenums = append(filenums, n) | ||
| } | ||
| sort.Slice(filenums, func(i, j int) bool { | ||
| return filenums[i].FileNum() < filenums[j].FileNum() | ||
| }) | ||
| fmt.Fprintf(stdout, "Objects:\n") | ||
| for _, n := range filenums { | ||
| m := objects[n] | ||
| fmt.Fprintf( | ||
| stdout, " %s CreatorID: %s CreatorFileNum: %s Locator: %q CustomObjectName: %q\n", | ||
| n, m.CreatorID, m.CreatorFileNum, m.Locator, m.CustomObjectName, | ||
| ) | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use | ||
| // of this source code is governed by a BSD-style license that can be found in | ||
| // the LICENSE file. | ||
|
|
||
| package tool | ||
|
|
||
| import "testing" | ||
|
|
||
| func TestRemotecat(t *testing.T) { | ||
| runTests(t, "testdata/remotecat") | ||
| } |
Binary file not shown.
Oops, something went wrong.