From e32a57d07dc05c8275180e59cff59cd76cbd4841 Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 19 Mar 2024 11:55:09 +0100 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20=F0=9F=8E=B8=20allow=20to=20customi?= =?UTF-8?q?ze=20CAS=20storage=20hash=20and=20location=20mappin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cas/types.ts | 10 ++++---- src/crud-to-cas/CrudCas.ts | 35 ++++++++++++++------------ src/crud-to-cas/__tests__/testCasfs.ts | 2 +- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/cas/types.ts b/src/cas/types.ts index 6b2671039..9ec8d7866 100644 --- a/src/cas/types.ts +++ b/src/cas/types.ts @@ -1,10 +1,10 @@ import type { CrudResourceInfo } from '../crud/types'; -export interface CasApi { - put(blob: Uint8Array): Promise; - get(hash: string, options?: CasGetOptions): Promise; - del(hash: string, silent?: boolean): Promise; - info(hash: string): Promise; +export interface CasApi { + put(blob: Uint8Array): Promise; + get(hash: Hash, options?: CasGetOptions): Promise; + del(hash: Hash, silent?: boolean): Promise; + info(hash: Hash): Promise; } export interface CasGetOptions { diff --git a/src/crud-to-cas/CrudCas.ts b/src/crud-to-cas/CrudCas.ts index a9fc8e01e..75ea6526b 100644 --- a/src/crud-to-cas/CrudCas.ts +++ b/src/crud-to-cas/CrudCas.ts @@ -1,9 +1,11 @@ -import { hashToLocation } from './util'; +import { hashToLocation as defaultHashToLocation } from './util'; import type { CasApi, CasGetOptions } from '../cas/types'; import type { CrudApi, CrudResourceInfo } from '../crud/types'; +import type { FsLocation } from '../fsa-to-node/types'; -export interface CrudCasOptions { - hash: (blob: Uint8Array) => Promise; +export interface CrudCasOptions { + hash: (blob: Uint8Array) => Promise; + hash2Loc?: (hash: Hash) => FsLocation; } const normalizeErrors = async (code: () => Promise): Promise => { @@ -21,21 +23,22 @@ const normalizeErrors = async (code: () => Promise): Promise => { } }; -export class CrudCas implements CasApi { - constructor( - protected readonly crud: CrudApi, - protected readonly options: CrudCasOptions, - ) {} +export class CrudCas implements CasApi { + protected readonly hash2Loc: (hash: Hash) => FsLocation; - public readonly put = async (blob: Uint8Array): Promise => { + constructor(protected readonly crud: CrudApi, protected readonly options: CrudCasOptions) { + this.hash2Loc = options.hash2Loc || <(hash: Hash) => FsLocation>defaultHashToLocation; + } + + public readonly put = async (blob: Uint8Array): Promise => { const digest = await this.options.hash(blob); - const [collection, resource] = hashToLocation(digest); + const [collection, resource] = this.hash2Loc(digest); await this.crud.put(collection, resource, blob); return digest; }; - public readonly get = async (hash: string, options?: CasGetOptions): Promise => { - const [collection, resource] = hashToLocation(hash); + public readonly get = async (hash: Hash, options?: CasGetOptions): Promise => { + const [collection, resource] = this.hash2Loc(hash); return await normalizeErrors(async () => { const blob = await this.crud.get(collection, resource); if (!options?.skipVerification) { @@ -46,15 +49,15 @@ export class CrudCas implements CasApi { }); }; - public readonly del = async (hash: string, silent?: boolean): Promise => { - const [collection, resource] = hashToLocation(hash); + public readonly del = async (hash: Hash, silent?: boolean): Promise => { + const [collection, resource] = this.hash2Loc(hash); await normalizeErrors(async () => { return await this.crud.del(collection, resource, silent); }); }; - public readonly info = async (hash: string): Promise => { - const [collection, resource] = hashToLocation(hash); + public readonly info = async (hash: Hash): Promise => { + const [collection, resource] = this.hash2Loc(hash); return await normalizeErrors(async () => { return await this.crud.info(collection, resource); }); diff --git a/src/crud-to-cas/__tests__/testCasfs.ts b/src/crud-to-cas/__tests__/testCasfs.ts index 16c7f5cda..e71f4b4fd 100644 --- a/src/crud-to-cas/__tests__/testCasfs.ts +++ b/src/crud-to-cas/__tests__/testCasfs.ts @@ -16,7 +16,7 @@ const b = (str: string) => { }; export type Setup = () => { - cas: CasApi; + cas: CasApi; crud: CrudApi; snapshot: () => Record; }; From 63ae860ec1b36d920bd6eea7eb1c1a28bb626823 Mon Sep 17 00:00:00 2001 From: streamich Date: Tue, 19 Mar 2024 12:31:00 +0100 Subject: [PATCH 2/3] =?UTF-8?q?refactor:=20=F0=9F=92=A1=20separate=20abstr?= =?UTF-8?q?act=20logic=20into=20CrudCasBase=20class?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/crud-to-cas/CrudCas.ts | 68 +++---------------- src/crud-to-cas/CrudCasBase.ts | 60 ++++++++++++++++ src/crud-to-cas/__tests__/CrudCasBase.test.ts | 59 ++++++++++++++++ 3 files changed, 128 insertions(+), 59 deletions(-) create mode 100644 src/crud-to-cas/CrudCasBase.ts create mode 100644 src/crud-to-cas/__tests__/CrudCasBase.test.ts diff --git a/src/crud-to-cas/CrudCas.ts b/src/crud-to-cas/CrudCas.ts index 75ea6526b..d9881697e 100644 --- a/src/crud-to-cas/CrudCas.ts +++ b/src/crud-to-cas/CrudCas.ts @@ -1,65 +1,15 @@ -import { hashToLocation as defaultHashToLocation } from './util'; -import type { CasApi, CasGetOptions } from '../cas/types'; -import type { CrudApi, CrudResourceInfo } from '../crud/types'; -import type { FsLocation } from '../fsa-to-node/types'; +import { hashToLocation } from './util'; +import { CrudCasBase } from './CrudCasBase'; +import type { CrudApi } from '../crud/types'; -export interface CrudCasOptions { - hash: (blob: Uint8Array) => Promise; - hash2Loc?: (hash: Hash) => FsLocation; +export interface CrudCasOptions { + hash: (blob: Uint8Array) => Promise; } -const normalizeErrors = async (code: () => Promise): Promise => { - try { - return await code(); - } catch (error) { - if (error && typeof error === 'object') { - switch (error.name) { - case 'ResourceNotFound': - case 'CollectionNotFound': - throw new DOMException(error.message, 'BlobNotFound'); - } - } - throw error; - } -}; - -export class CrudCas implements CasApi { - protected readonly hash2Loc: (hash: Hash) => FsLocation; +const hashEqual = (h1: string, h2: string) => h1 === h2; - constructor(protected readonly crud: CrudApi, protected readonly options: CrudCasOptions) { - this.hash2Loc = options.hash2Loc || <(hash: Hash) => FsLocation>defaultHashToLocation; +export class CrudCas extends CrudCasBase { + constructor(protected readonly crud: CrudApi, protected readonly options: CrudCasOptions) { + super(crud, options.hash, hashToLocation, hashEqual); } - - public readonly put = async (blob: Uint8Array): Promise => { - const digest = await this.options.hash(blob); - const [collection, resource] = this.hash2Loc(digest); - await this.crud.put(collection, resource, blob); - return digest; - }; - - public readonly get = async (hash: Hash, options?: CasGetOptions): Promise => { - const [collection, resource] = this.hash2Loc(hash); - return await normalizeErrors(async () => { - const blob = await this.crud.get(collection, resource); - if (!options?.skipVerification) { - const digest = await this.options.hash(blob); - if (hash !== digest) throw new DOMException('Blob contents does not match hash', 'Integrity'); - } - return blob; - }); - }; - - public readonly del = async (hash: Hash, silent?: boolean): Promise => { - const [collection, resource] = this.hash2Loc(hash); - await normalizeErrors(async () => { - return await this.crud.del(collection, resource, silent); - }); - }; - - public readonly info = async (hash: Hash): Promise => { - const [collection, resource] = this.hash2Loc(hash); - return await normalizeErrors(async () => { - return await this.crud.info(collection, resource); - }); - }; } diff --git a/src/crud-to-cas/CrudCasBase.ts b/src/crud-to-cas/CrudCasBase.ts new file mode 100644 index 000000000..8a12df7f2 --- /dev/null +++ b/src/crud-to-cas/CrudCasBase.ts @@ -0,0 +1,60 @@ +import type { CasApi, CasGetOptions } from '../cas/types'; +import type { CrudApi, CrudResourceInfo } from '../crud/types'; +import type { FsLocation } from '../fsa-to-node/types'; + +const normalizeErrors = async (code: () => Promise): Promise => { + try { + return await code(); + } catch (error) { + if (error && typeof error === 'object') { + switch (error.name) { + case 'ResourceNotFound': + case 'CollectionNotFound': + throw new DOMException(error.message, 'BlobNotFound'); + } + } + throw error; + } +}; + +export class CrudCasBase implements CasApi { + constructor( + protected readonly crud: CrudApi, + protected readonly hash: (blob: Uint8Array) => Promise, + protected readonly hash2Loc: (hash: Hash) => FsLocation, + protected readonly hashEqual: (h1: Hash, h2: Hash) => boolean, + ) {} + + public readonly put = async (blob: Uint8Array): Promise => { + const digest = await this.hash(blob); + const [collection, resource] = this.hash2Loc(digest); + await this.crud.put(collection, resource, blob); + return digest; + }; + + public readonly get = async (hash: Hash, options?: CasGetOptions): Promise => { + const [collection, resource] = this.hash2Loc(hash); + return await normalizeErrors(async () => { + const blob = await this.crud.get(collection, resource); + if (!options?.skipVerification) { + const digest = await this.hash(blob); + if (!this.hashEqual(digest, hash)) throw new DOMException('Blob contents does not match hash', 'Integrity'); + } + return blob; + }); + }; + + public readonly del = async (hash: Hash, silent?: boolean): Promise => { + const [collection, resource] = this.hash2Loc(hash); + await normalizeErrors(async () => { + return await this.crud.del(collection, resource, silent); + }); + }; + + public readonly info = async (hash: Hash): Promise => { + const [collection, resource] = this.hash2Loc(hash); + return await normalizeErrors(async () => { + return await this.crud.info(collection, resource); + }); + }; +} diff --git a/src/crud-to-cas/__tests__/CrudCasBase.test.ts b/src/crud-to-cas/__tests__/CrudCasBase.test.ts new file mode 100644 index 000000000..f3945c7bf --- /dev/null +++ b/src/crud-to-cas/__tests__/CrudCasBase.test.ts @@ -0,0 +1,59 @@ +import { memfs } from '../..'; +import { onlyOnNode20 } from '../../__tests__/util'; +import { NodeFileSystemDirectoryHandle } from '../../node-to-fsa'; +import { FsaCrud } from '../../fsa-to-crud/FsaCrud'; +import { createHash } from 'crypto'; +import { hashToLocation } from '../util'; +import { CrudCasBase } from '../CrudCasBase'; + +onlyOnNode20('CrudCas on FsaCrud', () => { + const setup = () => { + const { fs } = memfs(); + const fsa = new NodeFileSystemDirectoryHandle(fs, '/', { mode: 'readwrite' }); + const crud = new FsaCrud(fsa); + return { fs, fsa, crud, snapshot: () => (fs).__vol.toJSON() }; + }; + + test('can use a custom hashing digest type', async () => { + const { crud } = setup(); + class Hash { + constructor(public readonly digest: string) {} + } + const hash = async (blob: Uint8Array): Promise => { + const shasum = createHash('sha1'); + shasum.update(blob); + const digest = shasum.digest('hex'); + return new Hash(digest); + }; + const cas = new CrudCasBase(crud, hash, (id: Hash) => hashToLocation(id.digest), (h1: Hash, h2: Hash) => h1.digest === h2.digest); + const blob = Buffer.from('hello world'); + const id = await cas.put(blob); + expect(id).toBeInstanceOf(Hash); + const id2 = await hash(blob); + expect(id.digest).toEqual(id2.digest); + const blob2 = await cas.get(id); + expect(String.fromCharCode(...blob2)).toEqual('hello world'); + expect(await cas.info(id)).toMatchObject({ size: 11 }); + await cas.del(id2); + expect(() => cas.info(id)).rejects.toThrowError(); + }); + + test('can use custom folder sharding strategy', async () => { + const { crud } = setup(); + const hash = async (blob: Uint8Array): Promise => { + const shasum = createHash('sha1'); + shasum.update(blob); + return shasum.digest('hex'); + }; + const cas = new CrudCasBase(crud, hash, (h: string) => [[h[0], h[1], h[2]], h[3]], (h1: string, h2: string) => h1 === h2); + const blob = Buffer.from('hello world'); + const id = await cas.put(blob); + expect(typeof id).toBe('string'); + const id2 = await hash(blob); + expect(id).toBe(id2); + const blob2 = await cas.get(id); + expect(String.fromCharCode(...blob2)).toEqual('hello world'); + const blob3 = await crud.get([id2[0], id2[1], id2[2]], id2[3]); + expect(String.fromCharCode(...blob3)).toEqual('hello world'); + }); +}); From 956e7fdf7720d36cb93aa75a43223fcccfc54f4f Mon Sep 17 00:00:00 2001 From: Va Da Date: Tue, 19 Mar 2024 11:53:07 +0000 Subject: [PATCH 3/3] =?UTF-8?q?style:=20=F0=9F=92=84=20run=20Prettier?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/crud-to-cas/CrudCas.ts | 5 ++++- src/crud-to-cas/__tests__/CrudCasBase.test.ts | 18 +++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/crud-to-cas/CrudCas.ts b/src/crud-to-cas/CrudCas.ts index d9881697e..d08e82665 100644 --- a/src/crud-to-cas/CrudCas.ts +++ b/src/crud-to-cas/CrudCas.ts @@ -9,7 +9,10 @@ export interface CrudCasOptions { const hashEqual = (h1: string, h2: string) => h1 === h2; export class CrudCas extends CrudCasBase { - constructor(protected readonly crud: CrudApi, protected readonly options: CrudCasOptions) { + constructor( + protected readonly crud: CrudApi, + protected readonly options: CrudCasOptions, + ) { super(crud, options.hash, hashToLocation, hashEqual); } } diff --git a/src/crud-to-cas/__tests__/CrudCasBase.test.ts b/src/crud-to-cas/__tests__/CrudCasBase.test.ts index f3945c7bf..d38540f47 100644 --- a/src/crud-to-cas/__tests__/CrudCasBase.test.ts +++ b/src/crud-to-cas/__tests__/CrudCasBase.test.ts @@ -5,6 +5,7 @@ import { FsaCrud } from '../../fsa-to-crud/FsaCrud'; import { createHash } from 'crypto'; import { hashToLocation } from '../util'; import { CrudCasBase } from '../CrudCasBase'; +import { FsLocation } from '../../fsa-to-node/types'; onlyOnNode20('CrudCas on FsaCrud', () => { const setup = () => { @@ -13,7 +14,7 @@ onlyOnNode20('CrudCas on FsaCrud', () => { const crud = new FsaCrud(fsa); return { fs, fsa, crud, snapshot: () => (fs).__vol.toJSON() }; }; - + test('can use a custom hashing digest type', async () => { const { crud } = setup(); class Hash { @@ -25,7 +26,12 @@ onlyOnNode20('CrudCas on FsaCrud', () => { const digest = shasum.digest('hex'); return new Hash(digest); }; - const cas = new CrudCasBase(crud, hash, (id: Hash) => hashToLocation(id.digest), (h1: Hash, h2: Hash) => h1.digest === h2.digest); + const cas = new CrudCasBase( + crud, + hash, + (id: Hash) => hashToLocation(id.digest), + (h1: Hash, h2: Hash) => h1.digest === h2.digest, + ); const blob = Buffer.from('hello world'); const id = await cas.put(blob); expect(id).toBeInstanceOf(Hash); @@ -45,7 +51,13 @@ onlyOnNode20('CrudCas on FsaCrud', () => { shasum.update(blob); return shasum.digest('hex'); }; - const cas = new CrudCasBase(crud, hash, (h: string) => [[h[0], h[1], h[2]], h[3]], (h1: string, h2: string) => h1 === h2); + const theCustomFolderShardingStrategy = (h: string): FsLocation => [[h[0], h[1], h[2]], h[3]]; + const cas = new CrudCasBase( + crud, + hash, + theCustomFolderShardingStrategy, + (h1: string, h2: string) => h1 === h2, + ); const blob = Buffer.from('hello world'); const id = await cas.put(blob); expect(typeof id).toBe('string');