Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Pebble local datastore #2484

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Pebble local datastore #2484

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b59347e
Use pebble for internal data store
gammazero Jan 17, 2024
c492fc1
update pebble
gammazero Jan 18, 2024
c8171dc
update deps
gammazero Mar 5, 2024
796e5d5
mod tidy
gammazero Mar 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions command/daemon.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -590,9 +590,8 @@ func loadConfig(filePath string) (*config.Config, error) {
if cfg.Version != config.Version {
log.Warn("Configuration file out-of-date. Upgrade by running: storetheindex init --upgrade")
}

if cfg.Datastore.Type != "levelds" {
return nil, fmt.Errorf("only levelds datastore type supported, %q not supported", cfg.Datastore.Type)
if err = checkDatastoreType(cfg.Datastore.Type); err != nil {
return nil, err
}

return cfg, nil
Expand Down
22 changes: 19 additions & 3 deletions command/datastore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/query"
leveldb "github.com/ipfs/go-ds-leveldb"
pebbledb "github.com/ipfs/go-ds-pebble"
"github.com/ipni/storetheindex/config"
"github.com/ipni/storetheindex/fsutil"
)
Expand All @@ -25,9 +26,18 @@ const (
updateBatchSize = 500000
)

func checkDatastoreType(dsType string) error {
switch dsType {
case "levelds", "pebble":
return nil
}
return fmt.Errorf("only levelds and pebble datastore types supported, %q not supported", dsType)
}

func createDatastore(ctx context.Context, dir, dsType string, rmExisting bool) (datastore.Batching, string, error) {
if dsType != "levelds" {
return nil, "", fmt.Errorf("only levelds datastore type supported, %q not supported", dsType)
err := checkDatastoreType(dsType)
if err != nil {
return nil, "", err
}
dataStorePath, err := config.Path("", dir)
if err != nil {
Expand All @@ -41,7 +51,13 @@ func createDatastore(ctx context.Context, dir, dsType string, rmExisting bool) (
if err = fsutil.DirWritable(dataStorePath); err != nil {
return nil, "", err
}
ds, err := leveldb.NewDatastore(dataStorePath, nil)
var ds datastore.Batching
switch dsType {
case "levelds":
ds, err = leveldb.NewDatastore(dataStorePath, nil)
case "pebble":
ds, err = pebbledb.NewDatastore(dataStorePath, nil)
}
if err != nil {
return nil, "", err
}
Expand Down
4 changes: 2 additions & 2 deletions config/datastore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@ type Datastore struct {
func NewDatastore() Datastore {
return Datastore{
Dir: "datastore",
Type: "levelds",
Type: "pebble",
TmpDir: "tmpstore",
TmpType: "levelds",
TmpType: "pebble",
}
}

Expand Down
4 changes: 3 additions & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,11 @@ require (
github.com/gammazero/channelqueue v0.2.1
github.com/gammazero/deque v0.2.1
github.com/gammazero/targz v0.0.3
github.com/ipfs/boxo v0.17.0
github.com/ipfs/boxo v0.18.0
github.com/ipfs/go-cid v0.4.1
github.com/ipfs/go-datastore v0.6.0
github.com/ipfs/go-ds-leveldb v0.5.0
github.com/ipfs/go-ds-pebble v0.3.1
github.com/ipfs/go-log/v2 v2.5.1
github.com/ipld/go-car/v2 v2.13.1
github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0
Expand Down Expand Up @@ -203,6 +204,7 @@ require (
go.uber.org/fx v1.20.1 // indirect
go.uber.org/mock v0.4.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
golang.org/x/crypto v0.19.0 // indirect
golang.org/x/exp v0.0.0-20240213143201-ec583247a57a // indirect
golang.org/x/mod v0.15.0 // indirect
Expand Down
12 changes: 9 additions & 3 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -421,8 +421,8 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/boxo v0.17.0 h1:fVXAb12dNbraCX1Cdid5BB6Kl62gVLNVA+e0EYMqAU0=
github.com/ipfs/boxo v0.17.0/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80=
github.com/ipfs/boxo v0.18.0 h1:MOL9/AgoV3e7jlVMInicaSdbgralfqSsbkc31dZ9tmw=
github.com/ipfs/boxo v0.18.0/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80=
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0=
Expand Down Expand Up @@ -475,6 +475,8 @@ github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1
github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s=
github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUNumo=
github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q=
github.com/ipfs/go-ds-pebble v0.3.1 h1:Jyad1qy+d0NZNisaSGUlBSt3dZNHAPl+JThyYe9Rziw=
github.com/ipfs/go-ds-pebble v0.3.1/go.mod h1:XYnWtulwJvHVOr2B0WVA/UC3dvRgFevjp8Pn9a3E1xo=
github.com/ipfs/go-graphsync v0.16.0 h1:0BX7whXlV13Y9FZ/jRg+xaGHaGYbtGxGppKD6tncw6k=
github.com/ipfs/go-graphsync v0.16.0/go.mod h1:WfbMW3hhmX5GQEQ+KJxsFzVJVBKgC5szfrYK7Zc7xIM=
github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08=
Expand Down Expand Up @@ -1247,8 +1249,9 @@ go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
go4.org v0.0.0-20200411211856-f5505b9728dd h1:BNJlw5kRTzdmyfh5U8F93HA2OwkP7ZGwA51eJ/0wKOU=
go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg=
go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc=
go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU=
golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
Expand Down Expand Up @@ -1368,6 +1371,7 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
Expand Down Expand Up @@ -1479,6 +1483,7 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand All @@ -1487,6 +1492,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down
8 changes: 4 additions & 4 deletions internal/registry/registry_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (

"github.com/ipfs/go-cid"
"github.com/ipfs/go-datastore"
leveldb "github.com/ipfs/go-ds-leveldb"
pebbledb "github.com/ipfs/go-ds-pebble"
"github.com/ipni/go-libipni/find/model"
"github.com/ipni/go-libipni/test"
"github.com/ipni/storetheindex/config"
Expand Down Expand Up @@ -142,7 +142,7 @@ func TestDatastore(t *testing.T) {
}

// Create datastore
dstore, err := leveldb.NewDatastore(dataStorePath, nil)
dstore, err := pebbledb.NewDatastore(dataStorePath, nil)
require.NoError(t, err)

r, err := New(ctx, discoveryCfg, dstore)
Expand Down Expand Up @@ -172,7 +172,7 @@ func TestDatastore(t *testing.T) {
require.NoError(t, dstore.Close())

// Create datastore
dstore, err = leveldb.NewDatastore(dataStorePath, nil)
dstore, err = pebbledb.NewDatastore(dataStorePath, nil)
require.NoError(t, err)

r, err = New(ctx, discoveryCfg, dstore)
Expand Down Expand Up @@ -222,7 +222,7 @@ func TestDatastore(t *testing.T) {
t.Log("Converted registry to work with assigner service")
discoveryCfg.UseAssigner = true
discoveryCfg.UnassignedPublishers = true
dstore, err = leveldb.NewDatastore(dataStorePath, nil)
dstore, err = pebbledb.NewDatastore(dataStorePath, nil)
require.NoError(t, err)
r, err = New(ctx, discoveryCfg, dstore)
require.NoError(t, err)
Expand Down
Loading