Use CIDs for content addressing rather than plain hashes

.gitignore

@@ -1,4 +1,3 @@
 /node_modules
 *.DS_Store
 dist/
-test/assets/

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "deno.enable": true
+}

build/main.js

@@ -11,15 +11,17 @@ var __export = (target, all) => {
 };
 
 // src/deps.ts
+import { assert, assertEquals } from "https://deno.land/std@0.141.0/testing/asserts.ts";
+import * as base64 from "https://deno.land/std@0.141.0/encoding/base64.ts";
 import * as flags from "https://deno.land/std@0.141.0/flags/mod.ts";
-import * as path from "https://deno.land/std@0.141.0/path/mod.ts";
 import * as colors from "https://deno.land/std@0.141.0/fmt/colors.ts";
-import * as streams from "https://deno.land/std@0.141.0/streams/mod.ts";
 import * as fs from "https://deno.land/std@0.141.0/fs/mod.ts";
-import * as base64 from "https://deno.land/std@0.141.0/encoding/base64.ts";
+import * as path from "https://deno.land/std@0.141.0/path/mod.ts";
+import * as streams from "https://deno.land/std@0.141.0/streams/mod.ts";
 import { base58btc } from "https://esm.sh/multiformats@11.0.2/bases/base58?pin=v120";
 import {} from "https://esm.sh/multiformats@11.0.2?pin=v120";
 import { default as default2 } from "https://esm.sh/@multiformats/blake2@1.0.13?pin=v120";
+import { CID } from "https://esm.sh/multiformats@11.0.2/cid?pin=v120";
 import { miniexec } from "https://deno.land/x/miniexec@1.0.0/mod.ts";
 import * as esbuild from "https://deno.land/x/esbuild@v0.14.47/mod.js";
 import * as sqlite from "https://deno.land/x/sqlite@v3.7.1/mod.ts";
@@ -158,16 +160,25 @@ var init_database_sqlite = __esm({
 });
 
 // src/utils.ts
-function blake32Hash(data) {
-  const uint8array = typeof data === "string" ? new TextEncoder().encode(data) : data;
-  const digest = default2.blake2b.blake2b256.digest(uint8array);
-  return base58btc.encode(digest.bytes);
-}
 function checkKey(key) {
   if (!isValidKey(key)) {
     throw new Error(`bad key: ${JSON.stringify(key)}`);
   }
 }
+async function createEntryFromFile(filepath) {
+  const buffer = await Deno.readFile(filepath);
+  const multicode = getPathExtension(filepath) === ".json" ? multicodes.JSON : multicodes.RAW;
+  const key = createCID(buffer, multicode);
+  return [key, buffer];
+}
+function createCID(data, multicode = multicodes.RAW) {
+  const uint8array = typeof data === "string" ? new TextEncoder().encode(data) : data;
+  const digest = multihasher.digest(uint8array);
+  const cid = CID.create(1, multicode, digest);
+  const key = cid.toString(multibase.encoder);
+  checkKey(key);
+  return key;
+}
 function exit(message) {
   console.error("[chel]", colors.red("Error:"), message);
   Deno.exit(1);
@@ -191,6 +202,12 @@ async function getBackend(src) {
   }
   return backend2;
 }
+function getPathExtension(path2) {
+  const index = path2.lastIndexOf(".");
+  if (index === -1 || index === 0)
+    return "";
+  return path2.slice(index).toLowerCase();
+}
 function isArrayLength(arg) {
   return Number.isInteger(arg) && arg >= 0 && arg <= 2 ** 32 - 1;
 }
@@ -226,14 +243,17 @@ async function readRemoteData(src, key) {
 async function revokeNet() {
   await Deno.permissions.revoke({ name: "net" });
 }
-var backends;
+var backends, multibase, multicodes, multihasher;
 var init_utils = __esm({
   "src/utils.ts"() {
     "use strict";
     init_deps();
     init_database_fs();
     init_database_sqlite();
     backends = { fs: database_fs_exports, sqlite: database_sqlite_exports };
+    multibase = base58btc;
+    multicodes = { JSON: 512, RAW: 85 };
+    multihasher = default2.blake2b.blake2b256;
   }
 });
 
@@ -262,9 +282,10 @@ async function upload(args, internal = false) {
   if (files.length === 0)
     throw new Error(`missing files!`);
   const uploaded = [];
-  const uploaderFn = isDir(urlOrDirOrSqliteFile) ? uploadToDir : urlOrDirOrSqliteFile.endsWith(".db") ? uploadToSQLite : uploadToURL;
+  const uploaderFn = isDir(urlOrDirOrSqliteFile) ? uploadEntryToDir : urlOrDirOrSqliteFile.endsWith(".db") ? uploadEntryToSQLite : uploadEntryToURL;
   for (const filepath of files) {
-    const destination = await uploaderFn(filepath, urlOrDirOrSqliteFile);
+    const entry = await createEntryFromFile(filepath);
+    const destination = await uploaderFn(entry, urlOrDirOrSqliteFile);
     if (!internal) {
       console.log(colors.green("uploaded:"), destination);
     } else {
@@ -274,35 +295,29 @@ async function upload(args, internal = false) {
   }
   return uploaded;
 }
-function uploadToURL(filepath, url) {
-  const buffer = Deno.readFileSync(filepath);
-  const hash2 = blake32Hash(buffer);
+function uploadEntryToURL([cid, buffer], url) {
   const form = new FormData();
-  form.append("hash", hash2);
-  form.append("data", new Blob([buffer]), path.basename(filepath));
+  form.append("hash", cid);
+  form.append("data", new Blob([buffer]));
   return fetch(`${url}/file`, { method: "POST", body: form }).then(handleFetchResult("text")).then((r) => {
-    if (r !== `/file/${hash2}`) {
+    if (r !== `/file/${cid}`) {
       throw new Error(`server returned bad URL: ${r}`);
     }
     return `${url}${r}`;
   });
 }
-async function uploadToDir(filepath, dir) {
+async function uploadEntryToDir([cid, buffer], dir) {
   await revokeNet();
-  const buffer = Deno.readFileSync(filepath);
-  const hash2 = blake32Hash(buffer);
-  const destination = path.join(dir, hash2);
+  const destination = path.join(dir, cid);
   await Deno.writeFile(destination, buffer);
   return destination;
 }
-async function uploadToSQLite(filepath, sqlitedb) {
+async function uploadEntryToSQLite([cid, buffer], sqlitedb) {
   await revokeNet();
   const { initStorage: initStorage3, writeData: writeData3 } = await Promise.resolve().then(() => (init_database_sqlite(), database_sqlite_exports));
   initStorage3({ dirname: path.dirname(sqlitedb), filename: path.basename(sqlitedb) });
-  const buffer = await Deno.readFile(filepath);
-  const hash2 = blake32Hash(buffer);
-  writeData3(hash2, buffer);
-  return hash2;
+  writeData3(cid, buffer);
+  return cid;
 }
 function handleFetchResult(type) {
   return function(r) {
@@ -453,22 +468,18 @@ async function get(args) {
 }
 
 // src/hash.ts
-init_deps();
 init_utils();
 async function hash(args, internal = false) {
   const [filename] = args;
   if (!filename) {
     console.error("please pass in a file");
     Deno.exit(1);
   }
-  const file = await Deno.open(filename, { read: true });
-  const myFileContent = await streams.readAll(file);
-  Deno.close(file.rid);
-  const hash2 = blake32Hash(myFileContent);
+  const [cid] = await createEntryFromFile(filename);
   if (!internal) {
-    console.log(`blake32Hash(${filename}):`, hash2);
+    console.log(`CID for file (${filename}):`, cid);
   }
-  return hash2;
+  return cid;
 }
 
 // src/help.ts
@@ -504,6 +515,9 @@ var helpDict = {
   `,
   hash: `
     chel hash <file>
+
+    Computes and logs the content identifier (CID) for the given file.
+    File contents will be interpreted as raw binary data, unless the file extension is '.json'.
   `,
   manifest: `
     chel manifest [-k|--key <pubkey1> [-k|--key <pubkey2> ...]]

deno.json

@@ -1,11 +1,12 @@
 {
   "tasks": {
-    "chel": "deno task build && deno run --import-map=./vendor/import_map.json --allow-net --allow-read=. --allow-write=. build/main.js",
+    "chel": "deno task build && deno run --import-map=./vendor/import_map.json --allow-net --allow-read=. --allow-write=. --no-remote build/main.js",
     "lint": "standard ./src",
     "vendor": "deno vendor -f src/main.ts scripts/compile.ts",
     "compile": "deno run --import-map=./vendor/import_map.json --no-remote --allow-run --allow-read=. --allow-write=./dist scripts/compile.ts",
     "build": "deno run --import-map=./vendor/import_map.json --no-remote --allow-run --allow-read --allow-env --allow-write=./build --allow-net scripts/build.ts",
-    "dist": "deno lint && deno task build && deno task compile"
+    "dist": "deno lint && deno task build && deno task compile",
+    "test": "deno test --import-map=./vendor/import_map.json --allow-read=. --no-remote"
   },
   "imports": {
     "https://deno.land/std/io/util.ts": "https://deno.land/std@0.156.0/io/util.ts"

src/deps.ts

@@ -1,12 +1,14 @@
+export { assert, assertEquals } from 'https://deno.land/std@0.141.0/testing/asserts.ts'
+export * as base64 from 'https://deno.land/std@0.141.0/encoding/base64.ts'
 export * as flags from 'https://deno.land/std@0.141.0/flags/mod.ts'
-export * as path from 'https://deno.land/std@0.141.0/path/mod.ts'
 export * as colors from "https://deno.land/std@0.141.0/fmt/colors.ts"
-export * as streams from "https://deno.land/std@0.141.0/streams/mod.ts"
 export * as fs from 'https://deno.land/std@0.141.0/fs/mod.ts'
-export * as base64 from 'https://deno.land/std@0.141.0/encoding/base64.ts'
+export * as path from 'https://deno.land/std@0.141.0/path/mod.ts'
+export * as streams from "https://deno.land/std@0.141.0/streams/mod.ts"
 export { base58btc } from 'https://esm.sh/multiformats@11.0.2/bases/base58?pin=v120'
 export { type Multibase } from 'https://esm.sh/multiformats@11.0.2?pin=v120'
-export { default as blake } from "https://esm.sh/@multiformats/blake2@1.0.13?pin=v120"
+export { default as blake } from 'https://esm.sh/@multiformats/blake2@1.0.13?pin=v120'
+export { CID } from 'https://esm.sh/multiformats@11.0.2/cid?pin=v120'
 export { miniexec as sh } from "https://deno.land/x/miniexec@1.0.0/mod.ts"
 export * as esbuild from "https://deno.land/x/esbuild@v0.14.47/mod.js"
 export * as sqlite from "https://deno.land/x/sqlite@v3.7.1/mod.ts"

src/hash.ts

@@ -1,7 +1,6 @@
 'use strict'
 
-import { streams } from './deps.ts'
-import { blake32Hash } from './utils.ts'
+import { createEntryFromFile } from './utils.ts'
 
 // TODO: use https://doc.deno.land/https://deno.land/std@0.140.0/streams/mod.ts/~/iterateReader instead to read in large files, and then use blake2b[Init,Update,Final] to iteratively calculate the hash
 //       Verify that it returns the same hash as when we use readAll https://doc.deno.land/https://deno.land/std@0.140.0/streams/mod.ts/~/readAll
@@ -12,13 +11,9 @@ export async function hash (args: string[], internal = false) {
     console.error('please pass in a file')
     Deno.exit(1)
   }
-  // TODO: implement this in a way that chunks the input, so that we don't read the entire file into memory at once
-  const file = await Deno.open(filename, {read: true})
-  const myFileContent = await streams.readAll(file)
-  Deno.close(file.rid)
-  const hash = blake32Hash(myFileContent)
+  const [cid] = await createEntryFromFile(filename)
   if (!internal) {
-    console.log(`blake32Hash(${filename}):`, hash)
+    console.log(`CID for file (${filename}):`, cid)
   }
-  return hash
+  return cid
 }

src/help.ts

@@ -33,6 +33,9 @@ const helpDict: {[key:string]: string} = {
   `,
   hash: `
     chel hash <file>
+
+    Computes and logs the content identifier (CID) for the given file.
+    File contents will be interpreted as raw binary data, unless the file extension is '.json'.
   `,
   manifest: `
     chel manifest [-k|--key <pubkey1> [-k|--key <pubkey2> ...]]

src/upload.ts

@@ -1,7 +1,7 @@
 'use strict'
 
 import { path, colors } from './deps.ts'
-import { blake32Hash, isDir, revokeNet } from './utils.ts'
+import { type Entry, createEntryFromFile, isDir, revokeNet } from './utils.ts'
 
 // chel upload <url-or-dir-or-sqlitedb> <file1> [<file2> [<file3> ...]]
 
@@ -10,12 +10,13 @@ export async function upload (args: string[], internal = false) {
   if (files.length === 0) throw new Error(`missing files!`)
   const uploaded = []
   const uploaderFn = isDir(urlOrDirOrSqliteFile)
-    ? uploadToDir
+    ? uploadEntryToDir
     : urlOrDirOrSqliteFile.endsWith('.db')
-      ? uploadToSQLite
-      : uploadToURL
+      ? uploadEntryToSQLite
+      : uploadEntryToURL
   for (const filepath of files) {
-    const destination = await uploaderFn(filepath, urlOrDirOrSqliteFile)
+    const entry = await createEntryFromFile(filepath)
+    const destination = await uploaderFn(entry, urlOrDirOrSqliteFile)
     if (!internal) {
       console.log(colors.green('uploaded:'), destination)
     } else {
@@ -26,46 +27,40 @@ export async function upload (args: string[], internal = false) {
   return uploaded
 }
 
-function uploadToURL (filepath: string, url: string): Promise<string> {
-  const buffer = Deno.readFileSync(filepath)
-  const hash = blake32Hash(buffer)
+function uploadEntryToURL ([cid, buffer]: Entry, url: string): Promise<string> {
   const form = new FormData()
-  form.append('hash', hash)
-  form.append('data', new Blob([buffer]), path.basename(filepath))
+  form.append('hash', cid)
+  form.append('data', new Blob([buffer]))
   return fetch(`${url}/file`, { method: 'POST', body: form })
     .then(handleFetchResult('text'))
     .then(r => {
-      if (r !== `/file/${hash}`) {
+      if (r !== `/file/${cid}`) {
         throw new Error(`server returned bad URL: ${r}`)
       }
       return `${url}${r}`
     })
 }
 
-async function uploadToDir (filepath: string, dir: string) {
+async function uploadEntryToDir ([cid, buffer]: Entry, dir: string): Promise<string> {
   await revokeNet()
-  const buffer = Deno.readFileSync(filepath)
-  const hash = blake32Hash(buffer)
-  const destination = path.join(dir, hash)
+  const destination = path.join(dir, cid)
   await Deno.writeFile(destination, buffer)
   return destination
 }
 
-async function uploadToSQLite (filepath: string, sqlitedb: string) {
+async function uploadEntryToSQLite ([cid, buffer]: Entry, sqlitedb: string): Promise<string> {
   await revokeNet()
   const { initStorage, writeData } = await import('./database-sqlite.ts')
   initStorage({dirname: path.dirname(sqlitedb), filename: path.basename(sqlitedb)})
-  const buffer = await Deno.readFile(filepath)
-  const hash = blake32Hash(buffer)
-  writeData(hash, buffer)
-  return hash
+  writeData(cid, buffer)
+  return cid
 }
 
 type ResponseTypeFn = 'arrayBuffer' | 'blob' | 'clone' | 'formData' | 'json' | 'text'
+
 export function handleFetchResult (type: ResponseTypeFn): ((r: Response) => unknown) {
   return function (r: Response) {
     if (!r.ok) throw new Error(`${r.status}: ${r.statusText}`)
     return r[type]()
   }
 }
-