Add IPFS storage

.gitignore

@@ -102,4 +102,7 @@ dist
 
 # TernJS port file
 .tern-port
-.DS_Store
+.DS_Store
+build/
+
+package-lock.json

zkdb/.npmignore

@@ -9,3 +9,4 @@ src
 
 # Never reveal your keys!
 keys
+.env

zkdb/package.json

@@ -41,18 +41,27 @@
     "@babel/preset-env": "^7.16.4",
     "@babel/preset-typescript": "^7.16.0",
     "@types/jest": "^27.0.3",
+    "@types/safer-buffer": "^2.1.0",
     "@typescript-eslint/eslint-plugin": "^5.5.0",
     "@typescript-eslint/parser": "^5.5.0",
     "eslint": "^8.7.0",
     "eslint-plugin-snarkyjs": "^0.1.0",
     "husky": "^7.0.1",
+    "ipfs-core-types": "^0.14.0",
     "jest": "^27.3.1",
     "lint-staged": "^11.0.1",
     "prettier": "^2.3.2",
     "ts-jest": "^27.0.7",
+    "tsconfig-paths": "^4.1.2",
     "typescript": "^4.7.2"
   },
   "peerDependencies": {
     "snarkyjs": "^0.8.0"
+  },
+  "dependencies": {
+    "ipfs-core": "^0.18.0",
+    "multiformats": "^11.0.1",
+    "safer-buffer": "^2.1.2",
+    "snarkyjs": "^0.8.0"
   }
 }

zkdb/src/storage/index.ts

@@ -0,0 +1,3 @@
+import { IPFSStorage, IPFSStorageConfiguration } from "./ipfs-storage";
+
+export { IPFSStorage, IPFSStorageConfiguration }

zkdb/src/storage/ipfs-storage.ts

@@ -0,0 +1,192 @@
+import { CID } from 'multiformats';
+import * as IPFS from 'ipfs-core';
+import { PutOptions } from 'ipfs-core-types/src/dag';
+import { Poseidon, Encoding } from 'snarkyjs';
+
+export function convertHexToUint8Array(hexString: string): Uint8Array {
+  const hex = hexString
+    .replace(/^0x/i, '')
+    .padStart(hexString.length + (hexString.length % 2), '0');
+  const result = new Uint8Array(hex.length / 2);
+
+  let j = 0;
+  for (let i = 0; i < result.length; i += 1) {
+    j = i * 2;
+    result[i] = parseInt(hex.substring(j, j + 2), 16);
+  }
+
+  return result;
+}
+
+export interface IPFSStorageConfiguration {
+  database: string;
+}
+
+export class IPFSStorage {
+  private config: IPFSStorageConfiguration;
+
+  private collections: any = {};
+
+  private nodeInstance: IPFS.IPFS;
+
+  constructor(
+    IPFSNodeInstance: IPFS.IPFS,
+    config?: Partial<IPFSStorageConfiguration>
+  ) {
+    this.config = { ...this.config, ...config };
+    this.nodeInstance = IPFSNodeInstance;
+  }
+
+  private get databasePath(): string {
+    return `/${this.config.database}`;
+  }
+
+  private get metadataPath(): string {
+    return `${this.databasePath}/metadata.zkdb`;
+  }
+
+  private getCollectionPath(collection: string): string {
+    return `${this.databasePath}/${collection}.json`;
+  }
+
+  private async poseidonHash(document: any): Promise<string> {
+    const encoder = new TextEncoder();
+
+    const doc = encoder.encode(JSON.stringify(document))
+
+    // Calculate poseidon hash of document
+    const hexDigest = convertHexToUint8Array(
+      Poseidon.hash(Encoding.Bijective.Fp.fromBytes(doc)).toString()
+    );
+
+    return (await this.nodeInstance.bases.getBase('base32')).encoder
+      .encode(hexDigest)
+      .toString();
+  }
+
+  private async isExist(path: string, filename: string) {
+    let status = false;
+    const fullPath = path + '/' + filename
+    try {
+      const fileStatus = await this.nodeInstance.files.stat(fullPath);
+      if (fileStatus) {
+        status = true;
+      }
+    } catch (e) {
+      let message = 'Unknown Error'
+      if (e instanceof Error) message = e.message
+      console.log(message)
+      status = false;
+    }
+    return status
+  }
+
+  private async readFile(filename: string): Promise<Uint8Array> {
+    let chunks = [];
+    let length = 0;
+    for await (const chunk of this.nodeInstance.files.read(filename)) {
+      chunks.push(chunk);
+      length += chunk.length;
+    }
+    let data = new Uint8Array(length);
+    length = 0;
+    for (let i = 0; i < chunks.length; i += 1) {
+      data.set(chunks[i], length);
+      length += chunks[i].length;
+    }
+    return data;
+  }
+
+  private async readJSON(filename: string): Promise<any> {
+    let data = '';
+    let decoder = new TextDecoder();
+
+    for await (const chunk of this.nodeInstance.files.read(filename)) {
+      data += decoder.decode(chunk);
+    }
+    return JSON.parse(data);
+  }
+
+  private async loadDatabase() {
+    if (!(await this.isExist('/', this.config.database))) {
+      await this.nodeInstance.files.mkdir(this.databasePath);
+    }
+  }
+
+  private async loadCollection(collection: string) {
+    if (!(await this.isExist(this.databasePath, `${collection}.json`))) {
+      // Create metadata file for zkDatabase
+      await this.nodeInstance.files.touch(this.getCollectionPath(collection));
+      // Write {} to the file
+      await this.nodeInstance.files.write(
+        this.getCollectionPath(collection),
+        new Uint8Array([123, 125])
+      );
+    }
+
+    this.collections[collection] = await this.readJSON(
+      this.getCollectionPath(collection)
+    );
+  }
+
+  public static async init(
+    config?: IPFSStorageConfiguration
+  ): Promise<IPFSStorage> {
+    const instance = new IPFSStorage(
+      await IPFS.create({ peerStoreCacheSize: 10 }),
+      config
+    );
+    await instance.loadDatabase();
+    return instance;
+  }
+
+  public async put<T>(collection: string, document: T, option?: PutOptions) {
+    await this.loadCollection(collection);
+    let documentDigest = await this.poseidonHash(document);
+
+    const result = await this.nodeInstance.dag.put(document, {
+      pin: true,
+      ...option,
+    });
+
+    const cid = result.toString();
+    this.collections[collection][documentDigest] = cid;
+
+    await this.nodeInstance.files.write(
+      this.getCollectionPath(collection),
+      JSON.stringify(this.collections[collection])
+    );
+
+    return {
+      CID: cid,
+      documentID: documentDigest,
+      timestamp: Date.now(),
+      database: this.config.database,
+      collection,
+      document,
+    };
+  }
+
+  public async get(collection: string, documentID: string) {
+    if (
+      typeof this.collections[collection] === 'undefined' ||
+      typeof this.collections[collection][documentID] === 'undefined'
+    ) {
+      await this.loadCollection(collection);
+    }
+
+    if (
+      typeof this.collections[collection] !== 'undefined' &&
+      typeof this.collections[collection][documentID] !== 'undefined'
+    ) {
+      const cid = CID.parse(this.collections[collection][documentID]);
+      const DAGResult = await this.nodeInstance.dag.get(cid);
+      return {
+        CID: cid.toString(),
+        documentID: documentID,
+        ...DAGResult,
+      };
+    }
+    return undefined;
+  }
+}

zkdb/src/storage/storage-test.ts

@@ -0,0 +1,24 @@
+import { IPFSStorage, IPFSStorageConfiguration } from "./ipfs-storage.js";
+
+async function run() {
+    const config: IPFSStorageConfiguration = { database: "TestDatabase" }
+
+    const storage = await IPFSStorage.init(config);
+
+    const obj = {
+        employees: [
+            { firstName: "John", lastName: "Doe" },
+            { firstName: "Anna", lastName: "Smith" },
+            { firstName: "Peter", lastName: "Jones" },
+        ],
+    };
+
+    const result = await storage.put("TestFile", obj, { pin: true });
+    console.log(result)
+
+    const getTestResult = await storage.get("TestFile", result.documentID)
+    console.log(getTestResult)
+    console.log(getTestResult?.value)
+}
+
+run()

zkdb/tsconfig.json

@@ -17,7 +17,12 @@
     "declaration": true,
     "sourceMap": true,
     "noFallthroughCasesInSwitch": true,
-    "allowSyntheticDefaultImports": true
+    "allowSyntheticDefaultImports": true,
+    "baseUrl": "./src",
+    "paths": {
+      "@utilities/*": ["utilities/*"]
+    }
   },
-  "include": ["./src"]
+  "include": ["./src"],
+  "exclude": ["node_modules", "build"]
 }