-
-
Notifications
You must be signed in to change notification settings - Fork 130
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
7 changed files
with
194 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
`casfs` is a Content Addressable Storage (CAS) abstraction over a file system. | ||
It has no folders nor files. Instead, it has *blobs* which are identified by their content. | ||
|
||
Essentially, it provides two main operations: `put` and `get`. The `put` operation | ||
takes a blob and stores it in the underlying file system and returns the blob's hash digest. | ||
The `get` operation takes a hash and returns the blob, which matches the hash digest, if it exists. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
import type {CrudResourceInfo} from "../crud/types"; | ||
|
||
export interface CasApi { | ||
put(blob: Uint8Array): Promise<string>; | ||
get(hash: string, options?: CasGetOptions): Promise<Uint8Array>; | ||
del(hash: string, silent?: boolean): Promise<void>; | ||
info(hash: string): Promise<CrudResourceInfo>; | ||
} | ||
|
||
export interface CasGetOptions { | ||
skipVerification?: boolean; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,54 @@ | ||
import {hashToLocation} from "./util"; | ||
import type {CasApi} from "../cas/types"; | ||
import type {CrudApi, CrudResourceInfo} from "../crud/types"; | ||
|
||
export interface CrudCasOptions { | ||
hash: (blob: Uint8Array) => Promise<string>; | ||
} | ||
|
||
const normalizeErrors = async <T>(code: () => Promise<T>): Promise<T> => { | ||
try { | ||
return await code(); | ||
} catch (error) { | ||
if (error && typeof error === 'object') { | ||
switch (error.name) { | ||
case 'ResourceNotFound': | ||
case 'CollectionNotFound': | ||
throw new DOMException(error.message, 'BlobNotFound'); | ||
} | ||
} | ||
throw error; | ||
} | ||
}; | ||
|
||
export class CrudCas implements CasApi { | ||
constructor(protected readonly crud: CrudApi, protected readonly options: CrudCasOptions) {} | ||
|
||
public readonly put = async (blob: Uint8Array): Promise<string> => { | ||
const digest = await this.options.hash(blob); | ||
const [collection, resource] = hashToLocation(digest); | ||
await this.crud.put(collection, resource, blob); | ||
return digest; | ||
}; | ||
|
||
public readonly get = async (hash: string): Promise<Uint8Array> => { | ||
const [collection, resource] = hashToLocation(hash); | ||
return await normalizeErrors(async () => { | ||
return await this.crud.get(collection, resource); | ||
}); | ||
}; | ||
|
||
public readonly del = async (hash: string, silent?: boolean): Promise<void> => { | ||
const [collection, resource] = hashToLocation(hash); | ||
await normalizeErrors(async () => { | ||
return await this.crud.del(collection, resource, silent); | ||
}); | ||
}; | ||
|
||
public readonly info = async (hash: string): Promise<CrudResourceInfo> => { | ||
const [collection, resource] = hashToLocation(hash); | ||
return await normalizeErrors(async () => { | ||
return await this.crud.info(collection, resource); | ||
}); | ||
}; | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,105 @@ | ||
import { of } from 'thingies'; | ||
import { createHash } from 'crypto'; | ||
import { memfs } from '../..'; | ||
import { onlyOnNode20 } from '../../__tests__/util'; | ||
import { NodeFileSystemDirectoryHandle } from '../../node-to-fsa'; | ||
import { FsaCrud } from '../../fsa-to-crud/FsaCrud'; | ||
import { CrudCas } from '../CrudCas'; | ||
|
||
const hash = async (blob: Uint8Array): Promise<string> => { | ||
const shasum = createHash('sha1') | ||
shasum.update(blob) | ||
return shasum.digest('hex') | ||
}; | ||
|
||
const setup = () => { | ||
const fs = memfs(); | ||
const fsa = new NodeFileSystemDirectoryHandle(fs, '/', { mode: 'readwrite' }); | ||
const crud = new FsaCrud(fsa); | ||
const cas = new CrudCas(crud, {hash}); | ||
return { fs, fsa, crud, cas, snapshot: () => (<any>fs).__vol.toJSON() }; | ||
}; | ||
|
||
const b = (str: string) => { | ||
const buf = Buffer.from(str); | ||
return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength); | ||
}; | ||
|
||
onlyOnNode20('CrudCas', () => { | ||
describe('.put()', () => { | ||
test('can store a blob', async () => { | ||
const blob = b('hello world'); | ||
const { cas, snapshot } = setup(); | ||
const hash = await cas.put(blob); | ||
expect(hash).toBe('2aae6c35c94fcfb415dbe95f408b9ce91ee846ed'); | ||
expect(snapshot()).toMatchSnapshot(); | ||
}); | ||
}); | ||
|
||
describe('.get()', () => { | ||
test('can retrieve existing blob', async () => { | ||
const blob = b('hello world'); | ||
const { cas } = setup(); | ||
const hash = await cas.put(blob); | ||
const blob2 = await cas.get(hash); | ||
expect(blob2).toStrictEqual(blob); | ||
}); | ||
|
||
test('throws if blob does not exist', async () => { | ||
const blob = b('hello world 2'); | ||
const { cas } = setup(); | ||
const hash = await cas.put(blob); | ||
const [, err] = await of(cas.get('2aae6c35c94fcfb415dbe95f408b9ce91ee846ed')); | ||
expect(err).toBeInstanceOf(DOMException); | ||
expect((<any>err).name).toBe('BlobNotFound'); | ||
}); | ||
}); | ||
|
||
describe('.info()', () => { | ||
test('can retrieve existing blob info', async () => { | ||
const blob = b('hello world'); | ||
const { cas } = setup(); | ||
const hash = await cas.put(blob); | ||
const info = await cas.info(hash); | ||
expect(info.size).toBe(11); | ||
}); | ||
|
||
test('throws if blob does not exist', async () => { | ||
const blob = b('hello world 2'); | ||
const { cas } = setup(); | ||
const hash = await cas.put(blob); | ||
const [, err] = await of(cas.info('2aae6c35c94fcfb415dbe95f408b9ce91ee846ed')); | ||
expect(err).toBeInstanceOf(DOMException); | ||
expect((<any>err).name).toBe('BlobNotFound'); | ||
}); | ||
}); | ||
|
||
describe('.del()', () => { | ||
test('can delete an existing blob', async () => { | ||
const blob = b('hello world'); | ||
const { cas } = setup(); | ||
const hash = await cas.put(blob); | ||
const info = await cas.info(hash); | ||
await cas.del(hash); | ||
const [, err] = await of(cas.info(hash)); | ||
expect(err).toBeInstanceOf(DOMException); | ||
expect((<any>err).name).toBe('BlobNotFound'); | ||
}); | ||
|
||
test('throws if blob does not exist', async () => { | ||
const blob = b('hello world 2'); | ||
const { cas } = setup(); | ||
const hash = await cas.put(blob); | ||
const [, err] = await of(cas.del('2aae6c35c94fcfb415dbe95f408b9ce91ee846ed')); | ||
expect(err).toBeInstanceOf(DOMException); | ||
expect((<any>err).name).toBe('BlobNotFound'); | ||
}); | ||
|
||
test('does not throw if "silent" flag is provided', async () => { | ||
const blob = b('hello world 2'); | ||
const { cas } = setup(); | ||
const hash = await cas.put(blob); | ||
await cas.del('2aae6c35c94fcfb415dbe95f408b9ce91ee846ed', true); | ||
}); | ||
}); | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
// Jest Snapshot v1, https://goo.gl/fbAQLP | ||
|
||
exports[`CrudCas .put() can store a blob 1`] = ` | ||
Object { | ||
"/ed/46/2aae6c35c94fcfb415dbe95f408b9ce91ee846ed": "hello world", | ||
} | ||
`; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
import type {FsLocation} from "../fsa-to-node/types"; | ||
|
||
export const hashToLocation = (hash: string): FsLocation => { | ||
if (hash.length < 20) throw new TypeError('Hash is too short'); | ||
const lastTwo = hash.slice(-2); | ||
const twoBeforeLastTwo = hash.slice(-4, -2); | ||
const folder = [lastTwo, twoBeforeLastTwo]; | ||
return [folder, hash]; | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters