-
Notifications
You must be signed in to change notification settings - Fork 15
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix: add sharding to s3 blockstore (#202)
To avoid non-obvious performance footguns shard by default
- Loading branch information
1 parent
405250f
commit e1324a1
Showing
3 changed files
with
131 additions
and
83 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| import { CID } from 'multiformats/cid' | ||
| import { base32upper } from 'multiformats/bases/base32' | ||
| import type { MultibaseCodec } from 'multiformats/bases/interface' | ||
|
|
||
| export interface ShardingStrategy { | ||
| extension: string | ||
| encode: (cid: CID) => string | ||
| decode: (path: string) => CID | ||
| } | ||
|
|
||
| export interface NextToLastInit { | ||
| /** | ||
| * The file extension to use. default: '.data' | ||
| */ | ||
| extension?: string | ||
|
|
||
| /** | ||
| * How many characters to take from the end of the CID. default: 2 | ||
| */ | ||
| prefixLength?: number | ||
|
|
||
| /** | ||
| * The multibase codec to use - nb. should be case insensitive. | ||
| * default: base32upper | ||
| */ | ||
| base?: MultibaseCodec<string> | ||
| } | ||
|
|
||
| /** | ||
| * A sharding strategy that takes the last few characters of a multibase encoded | ||
| * CID and uses them as the directory to store the block in. This prevents | ||
| * storing all blocks in a single directory which would overwhelm most | ||
| * filesystems. | ||
| */ | ||
| export class NextToLast implements ShardingStrategy { | ||
| public extension: string | ||
| private readonly prefixLength: number | ||
| private readonly base: MultibaseCodec<string> | ||
|
|
||
| constructor (init: NextToLastInit = {}) { | ||
| this.extension = init.extension ?? '.data' | ||
| this.prefixLength = init.prefixLength ?? 2 | ||
| this.base = init.base ?? base32upper | ||
| } | ||
|
|
||
| encode (cid: CID): string { | ||
| const str = this.base.encoder.encode(cid.multihash.bytes) | ||
| const prefix = str.substring(str.length - this.prefixLength) | ||
|
|
||
| return `${prefix}/${str}${this.extension}` | ||
| } | ||
|
|
||
| decode (str: string): CID { | ||
| let fileName = str.split('/').pop() | ||
|
|
||
| if (fileName == null) { | ||
| throw new Error('Invalid path') | ||
| } | ||
|
|
||
| if (fileName.endsWith(this.extension)) { | ||
| fileName = fileName.substring(0, fileName.length - this.extension.length) | ||
| } | ||
|
|
||
| return CID.decode(this.base.decoder.decode(fileName)) | ||
| } | ||
| } | ||
|
|
||
| export interface FlatDirectoryInit { | ||
| /** | ||
| * The file extension to use. default: '.data' | ||
| */ | ||
| extension?: string | ||
|
|
||
| /** | ||
| * How many characters to take from the end of the CID. default: 2 | ||
| */ | ||
| prefixLength?: number | ||
|
|
||
| /** | ||
| * The multibase codec to use - nb. should be case insensitive. | ||
| * default: base32padupper | ||
| */ | ||
| base?: MultibaseCodec<string> | ||
| } | ||
|
|
||
| /** | ||
| * A sharding strategy that does not do any sharding and stores all files | ||
| * in one directory. Only for testing, do not use in production. | ||
| */ | ||
| export class FlatDirectory implements ShardingStrategy { | ||
| public extension: string | ||
| private readonly base: MultibaseCodec<string> | ||
|
|
||
| constructor (init: NextToLastInit = {}) { | ||
| this.extension = init.extension ?? '.data' | ||
| this.base = init.base ?? base32upper | ||
| } | ||
|
|
||
| encode (cid: CID): string { | ||
| const str = this.base.encoder.encode(cid.multihash.bytes) | ||
|
|
||
| return `${str}${this.extension}` | ||
| } | ||
|
|
||
| decode (str: string): CID { | ||
| let fileName = str.split('/').pop() | ||
|
|
||
| if (fileName == null) { | ||
| throw new Error('Invalid path') | ||
| } | ||
|
|
||
| if (fileName.endsWith(this.extension)) { | ||
| fileName = fileName.substring(0, fileName.length - this.extension.length) | ||
| } | ||
|
|
||
| return CID.decode(this.base.decoder.decode(fileName)) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters