feat: 🎸 add code

src/__tests__/util.ts

@@ -0,0 +1,31 @@
+import { createFsFromVolume, Volume } from 'memfs';
+import { Link, Node } from 'memfs/lib/node';
+
+export const create = (json: { [s: string]: string } = { '/foo': 'bar' }) => {
+  const vol = Volume.fromJSON(json);
+  return vol;
+};
+
+export const createFs = (json?) => {
+  return createFsFromVolume(create(json));
+};
+
+export const tryGetChild = (link: Link, name: string): Link => {
+  const child = link.getChild(name);
+
+  if (!child) {
+    throw new Error(`expected link to have a child named "${name}"`);
+  }
+
+  return child;
+};
+
+export const tryGetChildNode = (link: Link, name: string): Node => tryGetChild(link, name).getNode();
+
+const nodeMajorVersion = +process.version.split('.')[0].slice(1);
+
+/**
+ * The `File` global is available only starting in Node v20. Hence we run the
+ * tests only in those versions.
+ */
+export const onlyOnNode20 = nodeMajorVersion >= 20 ? describe : describe.skip;

src/cas/README.md

@@ -0,0 +1,6 @@
+`casfs` is a Content Addressable Storage (CAS) abstraction over a file system.
+It has no folders nor files. Instead, it has _blobs_ which are identified by their content.
+
+Essentially, it provides two main operations: `put` and `get`. The `put` operation
+takes a blob and stores it in the underlying file system and returns the blob's hash digest.
+The `get` operation takes a hash and returns the blob, which matches the hash digest, if it exists.

src/cas/types.ts

@@ -0,0 +1,12 @@
+import type { CrudResourceInfo } from '../crud/types';
+
+export interface CasApi<Hash> {
+  put(blob: Uint8Array): Promise<Hash>;
+  get(hash: Hash, options?: CasGetOptions): Promise<Uint8Array>;
+  del(hash: Hash, silent?: boolean): Promise<void>;
+  info(hash: Hash): Promise<CrudResourceInfo>;
+}
+
+export interface CasGetOptions {
+  skipVerification?: boolean;
+}

src/crud-to-cas/CrudCas.ts

@@ -0,0 +1,18 @@
+import { hashToLocation } from './util';
+import { CrudCasBase } from './CrudCasBase';
+import type { CrudApi } from '../crud/types';
+
+export interface CrudCasOptions {
+  hash: (blob: Uint8Array) => Promise<string>;
+}
+
+const hashEqual = (h1: string, h2: string) => h1 === h2;
+
+export class CrudCas extends CrudCasBase<string> {
+  constructor(
+    protected readonly crud: CrudApi,
+    protected readonly options: CrudCasOptions,
+  ) {
+    super(crud, options.hash, hashToLocation, hashEqual);
+  }
+}

src/crud-to-cas/CrudCasBase.ts

@@ -0,0 +1,60 @@
+import type { CasApi, CasGetOptions } from '../cas/types';
+import type { CrudApi, CrudResourceInfo } from '../crud/types';
+import type { FsLocation } from '../fsa-to-node/types';
+
+const normalizeErrors = async <T>(code: () => Promise<T>): Promise<T> => {
+  try {
+    return await code();
+  } catch (error) {
+    if (error && typeof error === 'object') {
+      switch (error.name) {
+        case 'ResourceNotFound':
+        case 'CollectionNotFound':
+          throw new DOMException(error.message, 'BlobNotFound');
+      }
+    }
+    throw error;
+  }
+};
+
+export class CrudCasBase<Hash> implements CasApi<Hash> {
+  constructor(
+    protected readonly crud: CrudApi,
+    protected readonly hash: (blob: Uint8Array) => Promise<Hash>,
+    protected readonly hash2Loc: (hash: Hash) => FsLocation,
+    protected readonly hashEqual: (h1: Hash, h2: Hash) => boolean,
+  ) {}
+
+  public readonly put = async (blob: Uint8Array): Promise<Hash> => {
+    const digest = await this.hash(blob);
+    const [collection, resource] = this.hash2Loc(digest);
+    await this.crud.put(collection, resource, blob);
+    return digest;
+  };
+
+  public readonly get = async (hash: Hash, options?: CasGetOptions): Promise<Uint8Array> => {
+    const [collection, resource] = this.hash2Loc(hash);
+    return await normalizeErrors(async () => {
+      const blob = await this.crud.get(collection, resource);
+      if (!options?.skipVerification) {
+        const digest = await this.hash(blob);
+        if (!this.hashEqual(digest, hash)) throw new DOMException('Blob contents does not match hash', 'Integrity');
+      }
+      return blob;
+    });
+  };
+
+  public readonly del = async (hash: Hash, silent?: boolean): Promise<void> => {
+    const [collection, resource] = this.hash2Loc(hash);
+    await normalizeErrors(async () => {
+      return await this.crud.del(collection, resource, silent);
+    });
+  };
+
+  public readonly info = async (hash: Hash): Promise<CrudResourceInfo> => {
+    const [collection, resource] = this.hash2Loc(hash);
+    return await normalizeErrors(async () => {
+      return await this.crud.info(collection, resource);
+    });
+  };
+}

src/crud-to-cas/__tests__/CrudCas.test.ts

@@ -0,0 +1,38 @@
+import { memfs } from 'memfs';
+import { onlyOnNode20 } from '../../__tests__/util';
+import { NodeFileSystemDirectoryHandle } from '../../node-to-fsa';
+import { FsaCrud } from '../../fsa-to-crud/FsaCrud';
+import { CrudCas } from '../CrudCas';
+import { testCasfs, hash } from './testCasfs';
+import { NodeCrud } from '../../node-to-crud/NodeCrud';
+
+onlyOnNode20('CrudCas on FsaCrud', () => {
+  const setup = () => {
+    const { fs } = memfs();
+    const fsa = new NodeFileSystemDirectoryHandle(fs, '/', { mode: 'readwrite' });
+    const crud = new FsaCrud(fsa);
+    const cas = new CrudCas(crud, { hash });
+    return { fs, fsa, crud, cas, snapshot: () => (<any>fs).__vol.toJSON() };
+  };
+  testCasfs(setup);
+});
+
+onlyOnNode20('CrudCas on NodeCrud at root', () => {
+  const setup = () => {
+    const { fs } = memfs();
+    const crud = new NodeCrud({ fs: fs.promises, dir: '/' });
+    const cas = new CrudCas(crud, { hash });
+    return { fs, crud, cas, snapshot: () => (<any>fs).__vol.toJSON() };
+  };
+  testCasfs(setup);
+});
+
+onlyOnNode20('CrudCas on NodeCrud at in sub-folder', () => {
+  const setup = () => {
+    const { fs } = memfs({ '/a/b/c': null });
+    const crud = new NodeCrud({ fs: fs.promises, dir: '/a/b/c' });
+    const cas = new CrudCas(crud, { hash });
+    return { fs, crud, cas, snapshot: () => (<any>fs).__vol.toJSON() };
+  };
+  testCasfs(setup);
+});

src/crud-to-cas/__tests__/CrudCasBase.test.ts

@@ -0,0 +1,71 @@
+import { memfs } from 'memfs';
+import { onlyOnNode20 } from '../../__tests__/util';
+import { NodeFileSystemDirectoryHandle } from '../../node-to-fsa';
+import { FsaCrud } from '../../fsa-to-crud/FsaCrud';
+import { createHash } from 'crypto';
+import { hashToLocation } from '../util';
+import { CrudCasBase } from '../CrudCasBase';
+import { FsLocation } from '../../fsa-to-node/types';
+
+onlyOnNode20('CrudCas on FsaCrud', () => {
+  const setup = () => {
+    const { fs } = memfs();
+    const fsa = new NodeFileSystemDirectoryHandle(fs, '/', { mode: 'readwrite' });
+    const crud = new FsaCrud(fsa);
+    return { fs, fsa, crud, snapshot: () => (<any>fs).__vol.toJSON() };
+  };
+
+  test('can use a custom hashing digest type', async () => {
+    const { crud } = setup();
+    class Hash {
+      constructor(public readonly digest: string) {}
+    }
+    const hash = async (blob: Uint8Array): Promise<Hash> => {
+      const shasum = createHash('sha1');
+      shasum.update(blob);
+      const digest = shasum.digest('hex');
+      return new Hash(digest);
+    };
+    const cas = new CrudCasBase<Hash>(
+      crud,
+      hash,
+      (id: Hash) => hashToLocation(id.digest),
+      (h1: Hash, h2: Hash) => h1.digest === h2.digest,
+    );
+    const blob = Buffer.from('hello world');
+    const id = await cas.put(blob);
+    expect(id).toBeInstanceOf(Hash);
+    const id2 = await hash(blob);
+    expect(id.digest).toEqual(id2.digest);
+    const blob2 = await cas.get(id);
+    expect(String.fromCharCode(...blob2)).toEqual('hello world');
+    expect(await cas.info(id)).toMatchObject({ size: 11 });
+    await cas.del(id2);
+    expect(() => cas.info(id)).rejects.toThrowError();
+  });
+
+  test('can use custom folder sharding strategy', async () => {
+    const { crud } = setup();
+    const hash = async (blob: Uint8Array): Promise<string> => {
+      const shasum = createHash('sha1');
+      shasum.update(blob);
+      return shasum.digest('hex');
+    };
+    const theCustomFolderShardingStrategy = (h: string): FsLocation => [[h[0], h[1], h[2]], h[3]];
+    const cas = new CrudCasBase<string>(
+      crud,
+      hash,
+      theCustomFolderShardingStrategy,
+      (h1: string, h2: string) => h1 === h2,
+    );
+    const blob = Buffer.from('hello world');
+    const id = await cas.put(blob);
+    expect(typeof id).toBe('string');
+    const id2 = await hash(blob);
+    expect(id).toBe(id2);
+    const blob2 = await cas.get(id);
+    expect(String.fromCharCode(...blob2)).toEqual('hello world');
+    const blob3 = await crud.get([id2[0], id2[1], id2[2]], id2[3]);
+    expect(String.fromCharCode(...blob3)).toEqual('hello world');
+  });
+});

src/crud-to-cas/__tests__/__snapshots__/CrudCas.test.ts.snap

@@ -0,0 +1,19 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`CrudCas on FsaCrud .put() can store a blob 1`] = `
+{
+  "/ed/46/2aae6c35c94fcfb415dbe95f408b9ce91ee846ed": "hello world",
+}
+`;
+
+exports[`CrudCas on NodeCrud at in sub-folder .put() can store a blob 1`] = `
+{
+  "/a/b/c/ed/46/2aae6c35c94fcfb415dbe95f408b9ce91ee846ed": "hello world",
+}
+`;
+
+exports[`CrudCas on NodeCrud at root .put() can store a blob 1`] = `
+{
+  "/ed/46/2aae6c35c94fcfb415dbe95f408b9ce91ee846ed": "hello world",
+}
+`;

src/crud-to-cas/__tests__/testCasfs.ts

@@ -0,0 +1,122 @@
+import { of } from 'thingies';
+import { createHash } from 'crypto';
+import { hashToLocation } from '../util';
+import type { CasApi } from '../../cas/types';
+import type { CrudApi } from '../../crud/types';
+
+export const hash = async (blob: Uint8Array): Promise<string> => {
+  const shasum = createHash('sha1');
+  shasum.update(blob);
+  return shasum.digest('hex');
+};
+
+const b = (str: string) => {
+  const buf = Buffer.from(str);
+  return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
+};
+
+export type Setup = () => {
+  cas: CasApi<string>;
+  crud: CrudApi;
+  snapshot: () => Record<string, string | null>;
+};
+
+export const testCasfs = (setup: Setup) => {
+  describe('.put()', () => {
+    test('can store a blob', async () => {
+      const blob = b('hello world');
+      const { cas, snapshot } = setup();
+      const hash = await cas.put(blob);
+      expect(hash).toBe('2aae6c35c94fcfb415dbe95f408b9ce91ee846ed');
+      expect(snapshot()).toMatchSnapshot();
+    });
+  });
+
+  describe('.get()', () => {
+    test('can retrieve existing blob', async () => {
+      const blob = b('hello world');
+      const { cas } = setup();
+      const hash = await cas.put(blob);
+      const blob2 = await cas.get(hash);
+      expect(blob2).toStrictEqual(blob);
+    });
+
+    test('throws if blob does not exist', async () => {
+      const blob = b('hello world 2');
+      const { cas } = setup();
+      const hash = await cas.put(blob);
+      const [, err] = await of(cas.get('2aae6c35c94fcfb415dbe95f408b9ce91ee846ed'));
+      expect(err).toBeInstanceOf(DOMException);
+      expect((<any>err).name).toBe('BlobNotFound');
+    });
+
+    test('throws if blob contents does not match the hash', async () => {
+      const blob = b('hello world');
+      const { cas, crud } = setup();
+      const hash = await cas.put(blob);
+      const location = hashToLocation(hash);
+      await crud.put(location[0], location[1], b('hello world!'));
+      const [, err] = await of(cas.get(hash));
+      expect(err).toBeInstanceOf(DOMException);
+      expect((<any>err).name).toBe('Integrity');
+    });
+
+    test('does not throw if integrity check is skipped', async () => {
+      const blob = b('hello world');
+      const { cas, crud } = setup();
+      const hash = await cas.put(blob);
+      const location = hashToLocation(hash);
+      await crud.put(location[0], location[1], b('hello world!'));
+      const blob2 = await cas.get(hash, { skipVerification: true });
+      expect(blob2).toStrictEqual(b('hello world!'));
+    });
+  });
+
+  describe('.info()', () => {
+    test('can retrieve existing blob info', async () => {
+      const blob = b('hello world');
+      const { cas } = setup();
+      const hash = await cas.put(blob);
+      const info = await cas.info(hash);
+      expect(info.size).toBe(11);
+    });
+
+    test('throws if blob does not exist', async () => {
+      const blob = b('hello world 2');
+      const { cas } = setup();
+      const hash = await cas.put(blob);
+      const [, err] = await of(cas.info('2aae6c35c94fcfb415dbe95f408b9ce91ee846ed'));
+      expect(err).toBeInstanceOf(DOMException);
+      expect((<any>err).name).toBe('BlobNotFound');
+    });
+  });
+
+  describe('.del()', () => {
+    test('can delete an existing blob', async () => {
+      const blob = b('hello world');
+      const { cas } = setup();
+      const hash = await cas.put(blob);
+      const info = await cas.info(hash);
+      await cas.del(hash);
+      const [, err] = await of(cas.info(hash));
+      expect(err).toBeInstanceOf(DOMException);
+      expect((<any>err).name).toBe('BlobNotFound');
+    });
+
+    test('throws if blob does not exist', async () => {
+      const blob = b('hello world 2');
+      const { cas } = setup();
+      const hash = await cas.put(blob);
+      const [, err] = await of(cas.del('2aae6c35c94fcfb415dbe95f408b9ce91ee846ed'));
+      expect(err).toBeInstanceOf(DOMException);
+      expect((<any>err).name).toBe('BlobNotFound');
+    });
+
+    test('does not throw if "silent" flag is provided', async () => {
+      const blob = b('hello world 2');
+      const { cas } = setup();
+      const hash = await cas.put(blob);
+      await cas.del('2aae6c35c94fcfb415dbe95f408b9ce91ee846ed', true);
+    });
+  });
+};

src/crud-to-cas/index.ts

@@ -0,0 +1 @@
+export { CrudCas, CrudCasOptions } from './CrudCas';

src/crud-to-cas/util.ts

@@ -0,0 +1,9 @@
+import type { FsLocation } from '../fsa-to-node/types';
+
+export const hashToLocation = (hash: string): FsLocation => {
+  if (hash.length < 20) throw new TypeError('Hash is too short');
+  const lastTwo = hash.slice(-2);
+  const twoBeforeLastTwo = hash.slice(-4, -2);
+  const folder = [lastTwo, twoBeforeLastTwo];
+  return [folder, hash];
+};