Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use newer flatfs sharding scheme #3608

Merged
merged 1 commit into from
Jan 20, 2017
Merged

Use newer flatfs sharding scheme #3608

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,9 +213,9 @@
},
{
"author": "whyrusleeping",
"hash": "Qmbx2KUs8mUbDUiiESzC1ms7mdmh4pRu8X1V1tffC46M4n",
"hash": "Qmcdc2Str4Dkc8yVCzzUFpnxmkS9AZXnpfo6jbhjjtXXkB",
"name": "go-ds-flatfs",
"version": "1.0.1"
"version": "1.1.0"
},
{
"author": "whyrusleeping",
Expand Down
9 changes: 5 additions & 4 deletions repo/fsrepo/defaultds.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import (
levelds "gx/ipfs/QmaHHmfEozrrotyhyN44omJouyuEtx6ahddqV6W5yRaUSQ/go-ds-leveldb"
ldbopts "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt"
measure "gx/ipfs/QmbUSMTQtK9GRrUbD4ngqJwSzHsquUc8nyDubRWp4vPybH/go-ds-measure"
"gx/ipfs/Qmbx2KUs8mUbDUiiESzC1ms7mdmh4pRu8X1V1tffC46M4n/go-ds-flatfs"
"gx/ipfs/Qmcdc2Str4Dkc8yVCzzUFpnxmkS9AZXnpfo6jbhjjtXXkB/go-ds-flatfs"
)

const (
Expand All @@ -33,9 +33,10 @@ func openDefaultDatastore(r *FSRepo) (repo.Datastore, error) {
}

syncfs := !r.config.Datastore.NoSync
// 5 bytes of prefix gives us 25 bits of freedom, 16 of which are taken by
// by the Qm prefix. Leaving us with 9 bits, or 512 way sharding
blocksDS, err := flatfs.New(path.Join(r.path, flatfsDirectory), 5, syncfs)

// 2 characters of base32 suffix gives us 10 bits of freedom.
// Leaving us with 10 bits, or 1024 way sharding
blocksDS, err := flatfs.CreateOrOpen(path.Join(r.path, flatfsDirectory), flatfs.NextToLast(2), syncfs)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The comment needs to be updated as well. (again)
If I understand correctly, that this is the last two base32 characters, it's 10 bits of sharding.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mateon1 good catch on the comment. Yeah, its the second to last two base32 characters (since the last character doesnt have 10 full bits of entropy)

if err != nil {
return nil, fmt.Errorf("unable to open flatfs datastore: %v", err)
}
Expand Down
2 changes: 1 addition & 1 deletion repo/fsrepo/fsrepo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import (
var log = logging.Logger("fsrepo")

// version number that we are currently expecting to see
var RepoVersion = 4
var RepoVersion = 5

var migrationInstructions = `See https://github.com/ipfs/fs-repo-migrations/blob/master/run.md
Sorry for the inconvenience. In the future, these will run automatically.`
Expand Down
3 changes: 3 additions & 0 deletions repo/fsrepo/migrations/migrations.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ func migrationsBinName() string {
}

func RunMigration(newv int) error {
if newv == 5 {
return fmt.Errorf("No automatic migrations yet for repo version 5. You will have to build them from source.\nSee https://github.com/ipfs/fs-repo-migrations")
}
migrateBin := migrationsBinName()

fmt.Println(" => Looking for suitable fs-repo-migrations binary.")
Expand Down
5 changes: 3 additions & 2 deletions test/sharness/t0066-migration.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,9 @@ test_expect_success "ipfs daemon --migrate=true runs migration" '
'

test_expect_success "output looks good" '
grep "Running: " true_out > /dev/null &&
grep "Success: fs-repo has been migrated to version 4." true_out > /dev/null
#grep "Running: " true_out > /dev/null &&
#grep "Success: fs-repo has been migrated to version 4." true_out > /dev/null
grep "No automatic migrations yet" true_out > /dev/null
'

test_expect_success "'ipfs daemon' prompts to auto migrate" '
Expand Down
4 changes: 2 additions & 2 deletions test/sharness/t0084-repo-read-rehash.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ test_init_ipfs
H_BLOCK1=$(echo "Block 1" | ipfs add -q)
H_BLOCK2=$(echo "Block 2" | ipfs add -q)

BS_BLOCK1="CIQPD/CIQPDDQH5PDJTF4QSNMPFC45FQZH5MBSWCX2W254P7L7HGNHW5MQXZA.data"
BS_BLOCK2="CIQNY/CIQNYWBOKHY7TCY7FUOBXKVJ66YRMARDT3KC7PPY6UWWPZR4YA67CKQ.data"
BS_BLOCK1="XZ/CIQPDDQH5PDJTF4QSNMPFC45FQZH5MBSWCX2W254P7L7HGNHW5MQXZA.data"
BS_BLOCK2="CK/CIQNYWBOKHY7TCY7FUOBXKVJ66YRMARDT3KC7PPY6UWWPZR4YA67CKQ.data"


test_expect_success 'blocks are swapped' '
Expand Down