Merge pull request #180 from AliMD/feat/storage-api-nanoservice

Storage Nanoservice API

demo/storage/big-data-test.ts

@@ -10,7 +10,7 @@ interface User extends DocumentObject {
   token: string;
 }
 
-const db = new AlwatrStorage<User>('junk-data', 'temp');
+const db = new AlwatrStorage<User>({name: 'junk-data', path: 'db'});
 
 db.readyPromise.then(() => {
   for (let i = 0; i < 10000; i++) {

demo/storage/index.ts

@@ -9,7 +9,7 @@ interface User extends DocumentObject {
   token?: string;
 }
 
-const db = new AlwatrStorage<User>('user-list', 'temp');
+const db = new AlwatrStorage<User>({name: 'user-list', path: 'db'});
 
 // await db.readyPromise
 // or

packages/core/nano-server/src/nano-server.ts

@@ -21,10 +21,10 @@ export class AlwatrNanoServer {
   protected _logger: AlwatrLogger;
   protected _server = createServer(this._requestListener.bind(this));
 
-  constructor(config: Partial<Config>) {
+  constructor(config?: Partial<Config>) {
     this._config = config = {...this._config, ...config};
     this._logger = createLogger(`alwatr-nano-server:${config.port}`);
-    this._logger.logMethodArgs('new', config);
+    this._logger.logMethodArgs('constructor', config);
     this._server.on('error', this._errorListener.bind(this));
     this._server.on('clientError', this._clientErrorListener.bind(this));
     this.route('GET', '/health', this._onHealthCheckRequest.bind(this));
@@ -249,19 +249,20 @@ export class AlwatrConnection {
     this.serverResponse.end();
   }
 
-  protected _getToken(): string | void {
+  protected _getToken(): string | null {
     const auth = this.incomingMessage.headers.authorization?.split(' ');
-    if (auth != null && auth[0] === 'Bearer') {
-      return auth[1];
-    } else {
-      return;
+
+    if (auth == null || auth[0] !== 'Bearer') {
+      return null;
     }
+
+    return auth[1];
   }
 
-  protected async _getRequestBody(): Promise<string | void> {
+  protected async _getRequestBody(): Promise<string | null> {
     // method must be POST or PUT
     if (!(this.method === 'POST' || this.method === 'PUT')) {
-      return;
+      return null;
     }
 
     let body = '';
@@ -275,26 +276,28 @@ export class AlwatrConnection {
     return body;
   }
 
-  async requireJsonBody<Type extends Record<string, unknown>>(): Promise<Type | void> {
+  async requireJsonBody<Type extends Record<string, unknown>>(): Promise<Type | null> {
     // if request content type is json, parse the body
     const body = await this.bodyPromise;
 
-    if (body == null || body.length === 0) {
-      return this.reply({
+    if (body === null || body.length === 0) {
+      this.reply({
         ok: false,
         statusCode: 400,
         errorCode: 'require_body',
       });
+      return null;
     }
 
     try {
       return JSON.parse(body) as Type;
     } catch (err) {
-      return this.reply({
+      this.reply({
         ok: false,
         statusCode: 400,
         errorCode: 'invalid_json',
       });
+      return null;
     }
   }
 }

packages/core/storage/src/provider.ts

@@ -0,0 +1,61 @@
+import {createLogger} from '@alwatr/logger';
+
+import {AlwatrStorage} from './storage.js';
+
+import type {AlwatrStorageConfig, AlwatrStorageProviderConfig, DocumentObject} from './type';
+
+// TODO: auto unload base of last usage time and memory limit.
+
+/**
+ * Easy access to many storages with auto garbage collector
+ *
+ * Example:
+ *
+ * ```ts
+ * import {AlwatrStorageProvider} from '@alwatr/storage';
+ * const storageList = new AlwatrStorageProvider();
+ * // ...
+ * const user = (await storageList.get('user-list')).get('userId1');
+ * ```
+ */
+export class AlwatrStorageProvider {
+  protected _logger = createLogger('alwatr-storage-provider');
+  protected _list: Record<string, AlwatrStorage<DocumentObject>> = {};
+  protected _config: AlwatrStorageProviderConfig;
+
+  constructor(config: AlwatrStorageProviderConfig) {
+    this._logger.logMethodArgs('constructor', config);
+    this._config = config;
+  }
+
+  // TODO: update all jsdoc and readme.
+  async get<DocumentType extends DocumentObject = DocumentObject>(
+      config: AlwatrStorageConfig,
+  ): Promise<AlwatrStorage<DocumentType>> {
+    if (!this._list[config.name]) {
+      this._list[config.name] = new AlwatrStorage<DocumentType>({
+        ...this._config,
+        ...config,
+      });
+    }
+    if (this._list[config.name].readyState !== true) {
+      await this._list[config.name].readyPromise;
+      console.log('Memory usage: %sMB', Math.round(process.memoryUsage.rss() / 100000) / 10);
+    }
+    return this._list[config.name] as AlwatrStorage<DocumentType>;
+  }
+
+  unload(name: string): void {
+    if (this._list[name] == null) return;
+    this._list[name].unload();
+    delete this._list[name];
+  }
+
+  unloadAll(): void {
+    // eslint-disable-next-line guard-for-in
+    for (const name in this._list) {
+      this._list[name].unload();
+    }
+    this._list = {};
+  }
+}

packages/core/storage/src/storage.ts

@@ -1,13 +1,13 @@
-import {existsSync} from 'fs';
+import {resolve} from 'node:path';
 
 import {alwatrRegisteredList, createLogger} from '@alwatr/logger';
 
 import {readJsonFile, writeJsonFile} from './util.js';
 
-import type {DocumentObject, DocumentListStorage} from './type.js';
+import type {DocumentObject, DocumentListStorage, AlwatrStorageConfig} from './type.js';
 import type {AlwatrLogger} from '@alwatr/logger';
 
-export * from './type.js';
+export {DocumentObject, DocumentListStorage, AlwatrStorageConfig as Config};
 
 alwatrRegisteredList.push({
   name: '@alwatr/storage',
@@ -36,48 +36,89 @@ export class AlwatrStorage<DocumentType extends DocumentObject> {
    */
   readonly name: string;
 
+  /**
+   * Storage file full path.
+   */
+  readonly storagePath: string;
+
   /**
    * Ready promise resolved when the storage is ready.
    * you can use this promise to wait for the storage to be loaded successfully and ready to use.
    *
    * Example:
    *
    * ```ts
-   * const db = new AlwatrStorage<User>('user-list');
+   * const db = new AlwatrStorage<User>({name: 'user-list', path: 'db'});
    * await db.readyPromise
    * const user = db.get('user-1');
    * ```
    */
-  readonly readyPromise: Promise<void>;
+  readyPromise: Promise<void>;
 
   /**
    * Ready state set to true when the storage is ready and readyPromise resolved.
    */
-  readyState = false;
-
+  get readyState(): boolean {
+    return this._readyState;
+  }
 
+  protected _readyState = false;
   protected _logger: AlwatrLogger;
   protected _storage: DocumentListStorage<DocumentType> = {};
-  protected _storagePath: string;
+  protected _keys: Array<string> | null = null;
 
-  constructor(name: string, pathPrefix = 'data') {
-    this._logger = createLogger(`alwatr-storage:${name}`);
-    this.name = name;
-    this._storagePath = `${pathPrefix}/${name}.json`;
-    this.readyPromise = this._init();
-  }
+  /**
+   * All document ids in array.
+   */
+  get keys(): Array<string> {
+    if (this._readyState !== true) throw new Error('storage_not_ready');
 
-  private async _init(): Promise<void> {
-    this._logger.logMethod('_init');
-    if (existsSync(this._storagePath)) {
-      this._storage = await readJsonFile<DocumentListStorage<DocumentType>>(this._storagePath);
-    } else {
-      this._storage = {};
+    if (this._keys === null) {
+      this._keys = Object.keys(this._storage);
     }
-    this.readyState = true;
+    return this._keys;
+  }
+
+  /**
+   * Size of the storage.
+   */
+  get length(): number {
+    return this.keys.length;
+  }
+
+  constructor(config: AlwatrStorageConfig) {
+    this._logger = createLogger(`alwatr-storage:${config.name}`);
+    this._logger.logMethodArgs('constructor', config);
+    this.name = config.name;
+    this.storagePath = resolve(`${config.path ?? './db'}/${config.name}.json`);
+    this.readyPromise = this._load();
+  }
+
+  /**
+   * Initial process like open/parse storage file.
+   * readyState will be set to true and readPromise will be resolved when this process finished.
+   */
+  private async _load(): Promise<void> {
+    this._logger.logMethod('_load');
+    this._storage = await readJsonFile<DocumentListStorage<DocumentType>>(this.storagePath) ?? {};
+    this._readyState = true;
     this._logger.logProperty('readyState', this.readyState);
   }
 
+  /**
+   * Check documentId exist in the storage or not.
+   *
+   * Example:
+   *
+   * ```ts
+   * if(!userDB.has('user-1')) throw new Error('user not found');
+   * ```
+   */
+  has(documentId: string): boolean {
+    if (this._readyState !== true) throw new Error('storage_not_ready');
+    return this._storage[documentId] != null;
+  }
+
   /**
    * Get a document object by id.
    *
@@ -94,6 +135,8 @@ export class AlwatrStorage<DocumentType extends DocumentObject> {
    */
   get(documentId: string, fastInstance?: boolean): DocumentType | null {
     this._logger.logMethodArgs('get', documentId);
+    if (this._readyState !== true) throw new Error('storage_not_ready');
+
     const documentObject = this._storage[documentId];
     if (documentObject == null) {
       return null;
@@ -123,6 +166,7 @@ export class AlwatrStorage<DocumentType extends DocumentObject> {
    */
   set(documentObject: DocumentType, fastInstance?: boolean): void {
     this._logger.logMethodArgs('set', documentObject._id);
+    if (this._readyState !== true) throw new Error('storage_not_ready');
 
     // update meta
     const oldData = this._storage[documentObject._id];
@@ -151,23 +195,41 @@ export class AlwatrStorage<DocumentType extends DocumentObject> {
    */
   remove(documentId: string): void {
     this._logger.logMethodArgs('remove', documentId);
+    if (this._readyState !== true) throw new Error('storage_not_ready');
+
     delete this._storage[documentId];
   }
 
-  private _saveTimer?: NodeJS.Timeout | number;
+  private _saveTimer: NodeJS.Timeout | null = null;
+
   /**
    * Save the storage to disk.
    */
   save(): void {
     this._logger.logMethod('save.request');
-    if (this._saveTimer != null) {
-      return;
-    }
+    if (this._readyState !== true) throw new Error('storage_not_ready');
+
+    if (this._saveTimer != null) return; // save already requested
+
     this._saveTimer = setTimeout(() => {
       this._logger.logMethod('save.action');
-      clearTimeout(this._saveTimer);
-      delete this._saveTimer;
-      writeJsonFile(this._storagePath, this._storage);
+      this._saveTimer = null;
+      // TODO: catch errors
+      writeJsonFile(this.storagePath, this._storage);
     }, 100);
   }
+
+  // TODO: update all jsdoc and readme.
+  unload(): void {
+    this._logger.logMethod('unload');
+    this._readyState = false;
+    this._storage = {};
+    this.readyPromise = Promise.reject(new Error('storage_unloaded'));
+  }
+
+  reload(): void {
+    this._logger.logMethod('reload');
+    this._readyState = false;
+    this.readyPromise = this._load();
+  }
 }

packages/core/storage/src/type.ts

@@ -1,6 +1,7 @@
 export type JSON = Record<string, unknown>;
 
 export interface DocumentObject {
+  [key: string]: unknown;
   _id: string;
   _rev?: number;
   _createdAt?: number;
@@ -11,3 +12,26 @@ export interface DocumentObject {
 
 export type DocumentListStorage<DocType extends DocumentObject> =
   Record<string, DocType | undefined> & {_last?: string};
+
+export type AlwatrStorageConfig = {
+  /**
+   * Storage name.
+   */
+  name: string,
+
+  /**
+   * Storage path.
+   *
+   * @default './db'
+   */
+  path?: string,
+}
+
+export type AlwatrStorageProviderConfig = {
+  /**
+   * Default storage path. you can override it in get config params.
+   *
+   * @default './db'
+   */
+  path?: string,
+}