CRUD FS read streaming

package.json

@@ -87,8 +87,8 @@
   },
   "dependencies": {
     "@jsonjoy.com/json-pack": "^1.0.4",
-    "@jsonjoy.com/util": "^1.2.0",
-    "memfs": "^4.9.4",
+    "@jsonjoy.com/util": "^1.3.0",
+    "memfs": "^4.10.0",
     "sonic-forest": "^1.0.3",
     "thingies": "^2.1.1",
     "tslib": "^2.6.3"

src/crud/__tests__/testCrudfs.ts

@@ -1,4 +1,6 @@
 import { of } from 'thingies';
+import { fromStream } from '@jsonjoy.com/util/lib/streams/fromStream';
+import { bufferToUint8Array } from '@jsonjoy.com/util/lib/buffers/bufferToUint8Array';
 import type { CrudApi, CrudCollectionEntry } from '../types';
 
 export type Setup = () => {
@@ -172,6 +174,16 @@ export const testCrudfs = (setup: Setup) => {
     });
   });
 
+  describe('.getStream()', () => {
+    test('can fetch an existing resource as stream', async () => {
+      const { crud } = setup();
+      await crud.put(['foo'], 'bar', b('abc'));
+      const stream = await crud.getStream(['foo'], 'bar');
+      const blob = bufferToUint8Array((await fromStream(stream)) as any);
+      expect(blob).toStrictEqual(b('abc'));
+    });
+  });
+
   describe('.del()', () => {
     test('throws if the type is not valid', async () => {
       const { crud } = setup();

src/crud/types.ts

@@ -18,6 +18,15 @@ export interface CrudApi {
    */
   get: (collection: CrudCollection, id: string) => Promise<Uint8Array>;
 
+  /**
+   * Retrieves the content of a resource as a file.
+   *
+   * @param collection Type of the resource, collection name.
+   * @param id Id of the resource, document name.
+   * @returns Blob content of the resource.
+   */
+  getStream: (collection: CrudCollection, id: string) => Promise<ReadableStream>;
+
   /**
    * Deletes a resource.
    *

src/fsa-to-crud/FsaCrud.ts

@@ -28,7 +28,7 @@ export class FsaCrud implements crud.CrudApi {
     }
   }
 
-  protected async getFile(
+  protected async __resolve(
     collection: crud.CrudCollection,
     id: string,
   ): Promise<[dir: fsa.IFileSystemDirectoryHandle, file: fsa.IFileSystemFileHandle]> {
@@ -98,20 +98,29 @@ export class FsaCrud implements crud.CrudApi {
     await writable.close();
   };
 
-  public readonly get = async (collection: crud.CrudCollection, id: string): Promise<Uint8Array> => {
+  public async _get(collection: crud.CrudCollection, id: string): Promise<File> {
     assertType(collection, 'get', 'crudfs');
     assertName(id, 'get', 'crudfs');
-    const [, file] = await this.getFile(collection, id);
-    const blob = await file.getFile();
-    const buffer = await blob.arrayBuffer();
+    const [, file] = await this.__resolve(collection, id);
+    return await file.getFile();
+  }
+
+  public readonly getStream = async (collection: crud.CrudCollection, id: string): Promise<ReadableStream> => {
+    const file = await this._get(collection, id);
+    return file.stream();
+  };
+
+  public readonly get = async (collection: crud.CrudCollection, id: string): Promise<Uint8Array> => {
+    const file = await this._get(collection, id);
+    const buffer = await file.arrayBuffer();
     return new Uint8Array(buffer);
   };
 
   public readonly del = async (collection: crud.CrudCollection, id: string, silent?: boolean): Promise<void> => {
     assertType(collection, 'del', 'crudfs');
     assertName(id, 'del', 'crudfs');
     try {
-      const [dir] = await this.getFile(collection, id);
+      const [dir] = await this.__resolve(collection, id);
       await dir.removeEntry(id, { recursive: false });
     } catch (error) {
       if (!silent) throw error;
@@ -122,7 +131,7 @@ export class FsaCrud implements crud.CrudApi {
     assertType(collection, 'info', 'crudfs');
     if (id) {
       assertName(id, 'info', 'crudfs');
-      const [, file] = await this.getFile(collection, id);
+      const [, file] = await this.__resolve(collection, id);
       const blob = await file.getFile();
       return {
         type: 'resource',

src/node-to-crud/NodeCrud.ts

@@ -4,7 +4,7 @@ import { FLAG } from 'memfs/lib/consts/FLAG';
 import { newExistsError, newFile404Error, newFolder404Error, newMissingError } from '../fsa-to-crud/util';
 import type { FsPromisesApi } from 'memfs/lib/node/types';
 import type * as crud from '../crud/types';
-import type { IDirent, IFileHandle } from 'memfs/lib/node/types/misc';
+import type { IDirent, IFileHandle, TFlags } from 'memfs/lib/node/types/misc';
 
 export interface NodeCrudOptions {
   readonly fs: FsPromisesApi;
@@ -98,14 +98,34 @@ export class NodeCrud implements crud.CrudApi {
     }
   };
 
-  public readonly get = async (collection: crud.CrudCollection, id: string): Promise<Uint8Array> => {
+  public async _file(collection: crud.CrudCollection, id: string, flags: TFlags): Promise<IFileHandle> {
     assertType(collection, 'get', 'crudfs');
     assertName(id, 'get', 'crudfs');
     const dir = await this.checkDir(collection);
     const filename = dir + id;
     const fs = this.fs;
+    return await fs.open(filename, flags);
+  }
+
+  public readonly getStream = async (collection: crud.CrudCollection, id: string): Promise<ReadableStream> => {
+    try {
+      const handle = await this._file(collection, id, FLAG.O_RDONLY);
+      return handle.readableWebStream();
+    } catch (error) {
+      if (error && typeof error === 'object') {
+        switch (error.code) {
+          case 'ENOENT':
+            throw newFile404Error(collection, id);
+        }
+      }
+      throw error;
+    }
+  };
+
+  public readonly get = async (collection: crud.CrudCollection, id: string): Promise<Uint8Array> => {
     try {
-      const buf = (await fs.readFile(filename)) as Buffer;
+      const handle = await this._file(collection, id, FLAG.O_RDONLY);
+      const buf = (await handle.readFile()) as Buffer;
       return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
     } catch (error) {
       if (error && typeof error === 'object') {

yarn.lock

@@ -596,10 +596,10 @@
   resolved "https://registry.yarnpkg.com/@jsonjoy.com/util/-/util-1.1.2.tgz#5072c27ecdb16d1ed7a2d125a1d0ed8aba01d652"
   integrity sha512-HOGa9wtE6LEz2I5mMQ2pMSjth85PmD71kPbsecs02nEUq3/Kw0wRK3gmZn5BCEB8mFLXByqPxjHgApoMwIPMKQ==
 
-"@jsonjoy.com/util@^1.2.0":
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/@jsonjoy.com/util/-/util-1.2.0.tgz#0fe9a92de72308c566ebcebe8b5a3f01d3149df2"
-  integrity sha512-4B8B+3vFsY4eo33DMKyJPlQ3sBMpPFUZK2dr3O3rXrOGKKbYG44J0XSFkDo1VOQiri5HFEhIeVvItjR2xcazmg==
+"@jsonjoy.com/util@^1.3.0":
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/@jsonjoy.com/util/-/util-1.3.0.tgz#e5623885bb5e0c48c1151e4dae422fb03a5887a1"
+  integrity sha512-Cebt4Vk7k1xHy87kHY7KSPLT77A7Ev7IfOblyLZhtYEhrdQ6fX4EoLq3xOQ3O/DRMEh2ok5nyC180E+ABS8Wmw==
 
 "@leichtgewicht/ip-codec@^2.0.1":
   version "2.0.5"
@@ -3156,13 +3156,13 @@ memfs@^3.4.3:
   dependencies:
     fs-monkey "^1.0.4"
 
-memfs@^4.9.4:
-  version "4.9.4"
-  resolved "https://registry.yarnpkg.com/memfs/-/memfs-4.9.4.tgz#803eb7f2091d1c6198ec9ba9b582505ad8699c9e"
-  integrity sha512-Xlj8b2rU11nM6+KU6wC7cuWcHQhVINWCUgdPS4Ar9nPxLaOya3RghqK7ALyDW2QtGebYAYs6uEdEVnwPVT942A==
+memfs@^4.10.0:
+  version "4.10.0"
+  resolved "https://registry.yarnpkg.com/memfs/-/memfs-4.10.0.tgz#f691ac909b679e0be6e29a6a15d8db4f9160a3f2"
+  integrity sha512-CkZUf+y0Ph/hu+mh0LPFrzDTPvGxUOjEqs4/DkIjFACbrCx8gT3VjqPY6EEtl5leC6JOkrm4wOlhQo8028Tj1g==
   dependencies:
     "@jsonjoy.com/json-pack" "^1.0.3"
-    "@jsonjoy.com/util" "^1.1.2"
+    "@jsonjoy.com/util" "^1.3.0"
     tree-dump "^1.0.1"
     tslib "^2.0.0"