From 7a93e18d6cce302688e62927f47e660000b27674 Mon Sep 17 00:00:00 2001 From: Devin Rousso Date: Mon, 6 Mar 2023 20:55:03 -0700 Subject: [PATCH] #244 - add a `createLevelDatabase` to allow customizing the `Level` used by `BlockstoreLevel` this will also be passed down when creating a `MessageStoreLevel` and `DataStoreLevel` --- README.md | 2 +- package-lock.json | 1 + package.json | 1 + src/index.ts | 1 + src/store/blockstore-level.ts | 20 +++++++++++++++---- src/store/create-level.ts | 11 ++++++++++ src/store/data-store-level.ts | 20 +++++++++++++++++-- src/store/message-store-level.ts | 9 +++++++-- tests/dwn.spec.ts | 4 +++- .../handlers/protocols-configure.spec.ts | 4 +++- .../handlers/protocols-query.spec.ts | 4 +++- .../records/handlers/records-delete.spec.ts | 4 +++- .../records/handlers/records-query.spec.ts | 4 +++- .../records/handlers/records-read.spec.ts | 4 +++- .../records/handlers/records-write.spec.ts | 4 +++- tests/store/message-store.spec.ts | 19 ++++++++++++++++++ 16 files changed, 96 insertions(+), 16 deletions(-) create mode 100644 src/store/create-level.ts diff --git a/README.md b/README.md index 528840026..22081b3d8 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # Decentralized Web Node (DWN) SDK Code Coverage -![Statements](https://img.shields.io/badge/statements-94.77%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-94.04%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-92.68%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-94.77%25-brightgreen.svg?style=flat) +![Statements](https://img.shields.io/badge/statements-94.83%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-94.05%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-92.71%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-94.83%25-brightgreen.svg?style=flat) ## Introduction diff --git a/package-lock.json b/package-lock.json index 5041f6994..f4cff028e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "@types/node": "^18.13.0", "@types/readable-stream": "^2.3.15", "@types/search-index": "3.2.0", + "abstract-level": "1.0.3", "ajv": "8.11.0", "blockstore-core": "3.0.0", "cross-fetch": "3.1.5", diff --git a/package.json b/package.json index ad5d7995b..24b831841 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "@types/node": "^18.13.0", "@types/readable-stream": "^2.3.15", "@types/search-index": "3.2.0", + "abstract-level": "1.0.3", "ajv": "8.11.0", "blockstore-core": "3.0.0", "cross-fetch": "3.1.5", diff --git a/src/index.ts b/src/index.ts index 69289fca0..14ed6bb62 100644 --- a/src/index.ts +++ b/src/index.ts @@ -17,6 +17,7 @@ export type { RecordsDeleteMessage, RecordsQueryMessage, RecordsWriteMessage } f export { AllowAllTenantGate, TenantGate } from './core/tenant-gate.js'; export { Cid } from './utils/cid.js'; export { DataStore } from './store/data-store.js'; +export { DataStoreLevel } from './store/data-store-level.js'; export { DateSort } from './interfaces/records/messages/records-query.js'; export { DataStream } from './utils/data-stream.js'; export { DidKeyResolver } from './did/did-key-resolver.js'; diff --git a/src/store/blockstore-level.ts b/src/store/blockstore-level.ts index 968463fcf..c2ebf8a00 100644 --- a/src/store/blockstore-level.ts +++ b/src/store/blockstore-level.ts @@ -1,9 +1,10 @@ +import type { LevelDatabase } from './create-level.js'; import type { AwaitIterable, Batch, KeyQuery, Pair, Query } from 'interface-store'; import type { Blockstore, Options } from 'interface-blockstore'; import { abortOr } from '../utils/abort.js'; import { CID } from 'multiformats'; -import { Level } from 'level'; +import { createLevelDatabase } from './create-level.js'; import { sleep } from '../utils/time.js'; // `level` works in Node.js 12+ and Electron 5+ on Linux, Mac OS, Windows and @@ -11,7 +12,9 @@ import { sleep } from '../utils/time.js'; // platforms like Raspberry Pi and Android, as well as in Chrome, Firefox, Edge, Safari, iOS Safari // and Chrome for Android. export class BlockstoreLevel implements Blockstore { - db: Level; + config: BlockstoreLevelConfig; + + db: LevelDatabase; /** * @param location - must be a directory path (relative or absolute) where LevelDB will store its @@ -19,8 +22,13 @@ export class BlockstoreLevel implements Blockstore { * the {@link https://developer.mozilla.org/en-US/docs/Web/API/IDBDatabase IDBDatabase} * to be opened. */ - constructor(location: string) { - this.db = new Level(location, { keyEncoding: 'utf8', valueEncoding: 'binary' }); + constructor(location: string, config: BlockstoreLevelConfig = {}) { + this.config = { + createLevelDatabase, + ...config + }; + + this.db = this.config.createLevelDatabase(location, { keyEncoding: 'utf8', valueEncoding: 'binary' }); } async open(): Promise { @@ -130,3 +138,7 @@ export class BlockstoreLevel implements Blockstore { throw new Error('not implemented'); } } + +type BlockstoreLevelConfig = { + createLevelDatabase?: typeof createLevelDatabase, +}; \ No newline at end of file diff --git a/src/store/create-level.ts b/src/store/create-level.ts new file mode 100644 index 000000000..81f5428dc --- /dev/null +++ b/src/store/create-level.ts @@ -0,0 +1,11 @@ +import type { AbstractDatabaseOptions, AbstractLevel } from 'abstract-level'; + +import { Level } from 'level'; + +export type LevelDatabase = AbstractLevel; + +export type LevelDatabaseOptions = AbstractDatabaseOptions; + +export function createLevelDatabase(location: string, options?: LevelDatabaseOptions): LevelDatabase { + return new Level(location, options); +} diff --git a/src/store/data-store-level.ts b/src/store/data-store-level.ts index 6e33f90c3..fd256ca6b 100644 --- a/src/store/data-store-level.ts +++ b/src/store/data-store-level.ts @@ -1,5 +1,6 @@ import { BlockstoreLevel } from './blockstore-level.js'; import { CID } from 'multiformats/cid'; +import { createLevelDatabase } from './create-level.js'; import { DataStore } from './data-store.js'; import { exporter } from 'ipfs-unixfs-exporter'; import { importer } from 'ipfs-unixfs-importer'; @@ -10,10 +11,20 @@ import { Readable } from 'readable-stream'; * Leverages LevelDB under the hood. */ export class DataStoreLevel implements DataStore { + config: DataStoreLevelConfig; + blockstore: BlockstoreLevel; - constructor(blockstoreLocation: string = 'DATASTORE') { - this.blockstore = new BlockstoreLevel(blockstoreLocation); + constructor(config: DataStoreLevelConfig = {}) { + this.config = { + blockstoreLocation: 'DATASTORE', + createLevelDatabase, + ...config + }; + + this.blockstore = new BlockstoreLevel(this.config.blockstoreLocation, { + createLevelDatabase: this.config.createLevelDatabase, + }); } public async open(): Promise { @@ -82,3 +93,8 @@ export class DataStoreLevel implements DataStore { await this.blockstore.clear(); } } + +type DataStoreLevelConfig = { + blockstoreLocation?: string, + createLevelDatabase?: typeof createLevelDatabase, +}; \ No newline at end of file diff --git a/src/store/message-store-level.ts b/src/store/message-store-level.ts index 2e90c40c8..edacc398c 100644 --- a/src/store/message-store-level.ts +++ b/src/store/message-store-level.ts @@ -9,6 +9,7 @@ import searchIndex from 'search-index'; import { abortOr } from '../utils/abort.js'; import { BlockstoreLevel } from './blockstore-level.js'; import { CID } from 'multiformats/cid'; +import { createLevelDatabase } from './create-level.js'; import { RangeCriterion } from '../interfaces/records/types.js'; import { sha256 } from 'multiformats/hashes/sha2'; @@ -19,7 +20,7 @@ import { sha256 } from 'multiformats/hashes/sha2'; export class MessageStoreLevel implements MessageStore { config: MessageStoreLevelConfig; - public readonly blockstore: BlockstoreLevel; + blockstore: BlockstoreLevel; // levelDB doesn't natively provide the querying capabilities needed for DWN, // to accommodate, we're leveraging a level-backed inverted index. @@ -36,10 +37,13 @@ export class MessageStoreLevel implements MessageStore { this.config = { blockstoreLocation : 'BLOCKSTORE', indexLocation : 'INDEX', + createLevelDatabase, ...config }; - this.blockstore = new BlockstoreLevel(this.config.blockstoreLocation); + this.blockstore = new BlockstoreLevel(this.config.blockstoreLocation, { + createLevelDatabase: this.config.createLevelDatabase, + }); } async open(): Promise { @@ -242,4 +246,5 @@ type RangeSearchIndexTerm = { type MessageStoreLevelConfig = { blockstoreLocation?: string, indexLocation?: string, + createLevelDatabase?: typeof createLevelDatabase, }; \ No newline at end of file diff --git a/tests/dwn.spec.ts b/tests/dwn.spec.ts index 5ab056338..88aa3d13a 100644 --- a/tests/dwn.spec.ts +++ b/tests/dwn.spec.ts @@ -25,7 +25,9 @@ describe('DWN', () => { indexLocation : 'TEST-INDEX' }); - dataStore = new DataStoreLevel('TEST-DATASTORE'); + dataStore = new DataStoreLevel({ + blockstoreLocation: 'TEST-DATASTORE' + }); dwn = await Dwn.create({ messageStore, dataStore }); }); diff --git a/tests/interfaces/protocols/handlers/protocols-configure.spec.ts b/tests/interfaces/protocols/handlers/protocols-configure.spec.ts index 5857fd08b..35221104e 100644 --- a/tests/interfaces/protocols/handlers/protocols-configure.spec.ts +++ b/tests/interfaces/protocols/handlers/protocols-configure.spec.ts @@ -32,7 +32,9 @@ describe('ProtocolsConfigureHandler.handle()', () => { indexLocation : 'TEST-INDEX' }); - dataStore = new DataStoreLevel('TEST-DATASTORE'); + dataStore = new DataStoreLevel({ + blockstoreLocation: 'TEST-DATASTORE' + }); dwn = await Dwn.create({ didResolver, messageStore, dataStore }); }); diff --git a/tests/interfaces/protocols/handlers/protocols-query.spec.ts b/tests/interfaces/protocols/handlers/protocols-query.spec.ts index b0dfed5b8..4db3b40ec 100644 --- a/tests/interfaces/protocols/handlers/protocols-query.spec.ts +++ b/tests/interfaces/protocols/handlers/protocols-query.spec.ts @@ -30,7 +30,9 @@ describe('ProtocolsQueryHandler.handle()', () => { indexLocation : 'TEST-INDEX' }); - dataStore = new DataStoreLevel('TEST-DATASTORE'); + dataStore = new DataStoreLevel({ + blockstoreLocation: 'TEST-DATASTORE' + }); dwn = await Dwn.create({ didResolver, messageStore, dataStore }); }); diff --git a/tests/interfaces/records/handlers/records-delete.spec.ts b/tests/interfaces/records/handlers/records-delete.spec.ts index 52dee1fae..b0fea4943 100644 --- a/tests/interfaces/records/handlers/records-delete.spec.ts +++ b/tests/interfaces/records/handlers/records-delete.spec.ts @@ -30,7 +30,9 @@ describe('RecordsDeleteHandler.handle()', () => { indexLocation : 'TEST-INDEX' }); - dataStore = new DataStoreLevel('TEST-DATASTORE'); + dataStore = new DataStoreLevel({ + blockstoreLocation: 'TEST-DATASTORE' + }); dwn = await Dwn.create({ didResolver, messageStore, dataStore }); }); diff --git a/tests/interfaces/records/handlers/records-query.spec.ts b/tests/interfaces/records/handlers/records-query.spec.ts index d19e5c6c1..0cbed481b 100644 --- a/tests/interfaces/records/handlers/records-query.spec.ts +++ b/tests/interfaces/records/handlers/records-query.spec.ts @@ -36,7 +36,9 @@ describe('RecordsQueryHandler.handle()', () => { indexLocation : 'TEST-INDEX' }); - dataStore = new DataStoreLevel('TEST-DATASTORE'); + dataStore = new DataStoreLevel({ + blockstoreLocation: 'TEST-DATASTORE' + }); dwn = await Dwn.create({ didResolver, messageStore, dataStore }); }); diff --git a/tests/interfaces/records/handlers/records-read.spec.ts b/tests/interfaces/records/handlers/records-read.spec.ts index f238eb242..b70343dff 100644 --- a/tests/interfaces/records/handlers/records-read.spec.ts +++ b/tests/interfaces/records/handlers/records-read.spec.ts @@ -31,7 +31,9 @@ describe('RecordsReadHandler.handle()', () => { indexLocation : 'TEST-INDEX' }); - dataStore = new DataStoreLevel('TEST-DATASTORE'); + dataStore = new DataStoreLevel({ + blockstoreLocation: 'TEST-DATASTORE' + }); dwn = await Dwn.create({ didResolver, messageStore, dataStore }); }); diff --git a/tests/interfaces/records/handlers/records-write.spec.ts b/tests/interfaces/records/handlers/records-write.spec.ts index 716a75b59..d06855a62 100644 --- a/tests/interfaces/records/handlers/records-write.spec.ts +++ b/tests/interfaces/records/handlers/records-write.spec.ts @@ -43,7 +43,9 @@ describe('RecordsWriteHandler.handle()', () => { indexLocation : 'TEST-INDEX' }); - dataStore = new DataStoreLevel('TEST-DATASTORE'); + dataStore = new DataStoreLevel({ + blockstoreLocation: 'TEST-DATASTORE' + }); dwn = await Dwn.create({ didResolver, messageStore, dataStore }); }); diff --git a/tests/store/message-store.spec.ts b/tests/store/message-store.spec.ts index fecb06cb8..cea8ce163 100644 --- a/tests/store/message-store.spec.ts +++ b/tests/store/message-store.spec.ts @@ -5,6 +5,7 @@ import { Message } from '../../src/core/message.js'; import { MessageStoreLevel } from '../../src/store/message-store-level.js'; import { RecordsWriteMessage } from '../../src/interfaces/records/types.js'; import { TestDataGenerator } from '../utils/test-data-generator.js'; +import { createLevelDatabase, LevelDatabase, LevelDatabaseOptions } from '../../src/store/create-level.js'; let messageStore: MessageStoreLevel; @@ -161,4 +162,22 @@ describe('MessageStoreLevel Tests', () => { expect(results.length).to.equal(0); }); }); + + describe('createLevelDatabase', function () { + it('should be called if provided', async () => { + let called = 0; + + new MessageStoreLevel({ + blockstoreLocation : 'TEST-BLOCKSTORE', + indexLocation : 'TEST-INDEX', + createLevelDatabase(location, options?: LevelDatabaseOptions): LevelDatabase { + ++called; + expect(location).to.equal('TEST-BLOCKSTORE'); + return createLevelDatabase(location, options); + } + }); + + expect(called).to.equal(1); + }); + }); }); \ No newline at end of file