diff --git a/src/cas/README.md b/src/cas/README.md new file mode 100644 index 000000000..a2342fac8 --- /dev/null +++ b/src/cas/README.md @@ -0,0 +1,6 @@ +`casfs` is a Content Addressable Storage (CAS) abstraction over a file system. +It has no folders nor files. Instead, it has *blobs* which are identified by their content. + +Essentially, it provides two main operations: `put` and `get`. The `put` operation +takes a blob and stores it in the underlying file system and returns the blob's hash digest. +The `get` operation takes a hash and returns the blob, which matches the hash digest, if it exists. diff --git a/src/cas/types.ts b/src/cas/types.ts new file mode 100644 index 000000000..4d9a303e4 --- /dev/null +++ b/src/cas/types.ts @@ -0,0 +1,12 @@ +import type {CrudResourceInfo} from "../crud/types"; + +export interface CasApi { + put(blob: Uint8Array): Promise; + get(hash: string, options?: CasGetOptions): Promise; + del(hash: string, silent?: boolean): Promise; + info(hash: string): Promise; +} + +export interface CasGetOptions { + skipVerification?: boolean; +} diff --git a/src/crud-to-cas/CrudCas.ts b/src/crud-to-cas/CrudCas.ts new file mode 100644 index 000000000..cd4e1be05 --- /dev/null +++ b/src/crud-to-cas/CrudCas.ts @@ -0,0 +1,54 @@ +import {hashToLocation} from "./util"; +import type {CasApi} from "../cas/types"; +import type {CrudApi, CrudResourceInfo} from "../crud/types"; + +export interface CrudCasOptions { + hash: (blob: Uint8Array) => Promise; +} + +const normalizeErrors = async (code: () => Promise): Promise => { + try { + return await code(); + } catch (error) { + if (error && typeof error === 'object') { + switch (error.name) { + case 'ResourceNotFound': + case 'CollectionNotFound': + throw new DOMException(error.message, 'BlobNotFound'); + } + } + throw error; + } +}; + +export class CrudCas implements CasApi { + constructor(protected readonly crud: CrudApi, protected readonly options: CrudCasOptions) {} + + public readonly put = async (blob: Uint8Array): Promise => { + const digest = await this.options.hash(blob); + const [collection, resource] = hashToLocation(digest); + await this.crud.put(collection, resource, blob); + return digest; + }; + + public readonly get = async (hash: string): Promise => { + const [collection, resource] = hashToLocation(hash); + return await normalizeErrors(async () => { + return await this.crud.get(collection, resource); + }); + }; + + public readonly del = async (hash: string, silent?: boolean): Promise => { + const [collection, resource] = hashToLocation(hash); + await normalizeErrors(async () => { + return await this.crud.del(collection, resource, silent); + }); + }; + + public readonly info = async (hash: string): Promise => { + const [collection, resource] = hashToLocation(hash); + return await normalizeErrors(async () => { + return await this.crud.info(collection, resource); + }); + }; +} diff --git a/src/crud-to-cas/__tests__/CrudCas.test.ts b/src/crud-to-cas/__tests__/CrudCas.test.ts new file mode 100644 index 000000000..6b116ab6b --- /dev/null +++ b/src/crud-to-cas/__tests__/CrudCas.test.ts @@ -0,0 +1,105 @@ +import { of } from 'thingies'; +import { createHash } from 'crypto'; +import { memfs } from '../..'; +import { onlyOnNode20 } from '../../__tests__/util'; +import { NodeFileSystemDirectoryHandle } from '../../node-to-fsa'; +import { FsaCrud } from '../../fsa-to-crud/FsaCrud'; +import { CrudCas } from '../CrudCas'; + +const hash = async (blob: Uint8Array): Promise => { + const shasum = createHash('sha1') + shasum.update(blob) + return shasum.digest('hex') +}; + +const setup = () => { + const fs = memfs(); + const fsa = new NodeFileSystemDirectoryHandle(fs, '/', { mode: 'readwrite' }); + const crud = new FsaCrud(fsa); + const cas = new CrudCas(crud, {hash}); + return { fs, fsa, crud, cas, snapshot: () => (fs).__vol.toJSON() }; +}; + +const b = (str: string) => { + const buf = Buffer.from(str); + return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength); +}; + +onlyOnNode20('CrudCas', () => { + describe('.put()', () => { + test('can store a blob', async () => { + const blob = b('hello world'); + const { cas, snapshot } = setup(); + const hash = await cas.put(blob); + expect(hash).toBe('2aae6c35c94fcfb415dbe95f408b9ce91ee846ed'); + expect(snapshot()).toMatchSnapshot(); + }); + }); + + describe('.get()', () => { + test('can retrieve existing blob', async () => { + const blob = b('hello world'); + const { cas } = setup(); + const hash = await cas.put(blob); + const blob2 = await cas.get(hash); + expect(blob2).toStrictEqual(blob); + }); + + test('throws if blob does not exist', async () => { + const blob = b('hello world 2'); + const { cas } = setup(); + const hash = await cas.put(blob); + const [, err] = await of(cas.get('2aae6c35c94fcfb415dbe95f408b9ce91ee846ed')); + expect(err).toBeInstanceOf(DOMException); + expect((err).name).toBe('BlobNotFound'); + }); + }); + + describe('.info()', () => { + test('can retrieve existing blob info', async () => { + const blob = b('hello world'); + const { cas } = setup(); + const hash = await cas.put(blob); + const info = await cas.info(hash); + expect(info.size).toBe(11); + }); + + test('throws if blob does not exist', async () => { + const blob = b('hello world 2'); + const { cas } = setup(); + const hash = await cas.put(blob); + const [, err] = await of(cas.info('2aae6c35c94fcfb415dbe95f408b9ce91ee846ed')); + expect(err).toBeInstanceOf(DOMException); + expect((err).name).toBe('BlobNotFound'); + }); + }); + + describe('.del()', () => { + test('can delete an existing blob', async () => { + const blob = b('hello world'); + const { cas } = setup(); + const hash = await cas.put(blob); + const info = await cas.info(hash); + await cas.del(hash); + const [, err] = await of(cas.info(hash)); + expect(err).toBeInstanceOf(DOMException); + expect((err).name).toBe('BlobNotFound'); + }); + + test('throws if blob does not exist', async () => { + const blob = b('hello world 2'); + const { cas } = setup(); + const hash = await cas.put(blob); + const [, err] = await of(cas.del('2aae6c35c94fcfb415dbe95f408b9ce91ee846ed')); + expect(err).toBeInstanceOf(DOMException); + expect((err).name).toBe('BlobNotFound'); + }); + + test('does not throw if "silent" flag is provided', async () => { + const blob = b('hello world 2'); + const { cas } = setup(); + const hash = await cas.put(blob); + await cas.del('2aae6c35c94fcfb415dbe95f408b9ce91ee846ed', true); + }); + }); +}); diff --git a/src/crud-to-cas/__tests__/__snapshots__/CrudCas.test.ts.snap b/src/crud-to-cas/__tests__/__snapshots__/CrudCas.test.ts.snap new file mode 100644 index 000000000..48f1ceea7 --- /dev/null +++ b/src/crud-to-cas/__tests__/__snapshots__/CrudCas.test.ts.snap @@ -0,0 +1,7 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`CrudCas .put() can store a blob 1`] = ` +Object { + "/ed/46/2aae6c35c94fcfb415dbe95f408b9ce91ee846ed": "hello world", +} +`; diff --git a/src/crud-to-cas/util.ts b/src/crud-to-cas/util.ts new file mode 100644 index 000000000..e12ab1cab --- /dev/null +++ b/src/crud-to-cas/util.ts @@ -0,0 +1,9 @@ +import type {FsLocation} from "../fsa-to-node/types"; + +export const hashToLocation = (hash: string): FsLocation => { + if (hash.length < 20) throw new TypeError('Hash is too short'); + const lastTwo = hash.slice(-2); + const twoBeforeLastTwo = hash.slice(-4, -2); + const folder = [lastTwo, twoBeforeLastTwo]; + return [folder, hash]; +}; diff --git a/src/crud/types.ts b/src/crud/types.ts index 05b51e398..e75cb8c03 100644 --- a/src/crud/types.ts +++ b/src/crud/types.ts @@ -7,7 +7,7 @@ export interface CrudApi { * @param data Blob content of the resource. * @param options Write behavior options. */ - put: (collection: CrudCollection, id: string, data: Uint8Array, options: CrudPutOptions) => Promise; + put: (collection: CrudCollection, id: string, data: Uint8Array, options?: CrudPutOptions) => Promise; /** * Retrieves the content of a resource.