fix(core): reduce probability of id collisions (#594)

use fnv1a 64bit for id segments
kukhariev · Jun 30, 2022 · f417258 · f417258
1 parent f52503b
commit f417258
Show file tree

Hide file tree

Showing 6 changed files with 33 additions and 15 deletions.
diff --git a/packages/core/src/utils/primitives.ts b/packages/core/src/utils/primitives.ts
@@ -28,6 +28,20 @@ export function fnv(str: string): string {
   return (hash >>> 0).toString(16);
 }
 
+/**
+ * FNV1A64 hash
+ */
+export function fnv64(str: string): string {
+  let hash = BigInt('14695981039346656037');
+  const offset = BigInt(1099511628211);
+  const len = str.length;
+  for (let i = 0; i < len; i++) {
+    hash ^= BigInt(str.charCodeAt(i));
+    hash *= offset;
+  }
+  return BigInt.asUintN(64, hash).toString(16);
+}
+
 export function mapValues<T>(
   object: Record<string, any>,
   func: (value: any) => T
@@ -92,4 +106,4 @@ export const memoize = <T, K>(fn: (val: T) => K): ((val: T) => K) => {
   return cached;
 };
 
-export const hash = memoize(fnv);
+export const hash = memoize(fnv64);
diff --git a/test/__snapshots__/disk-storage.spec.ts.snap b/test/__snapshots__/disk-storage.spec.ts.snap
@@ -15,7 +15,7 @@ DiskFile {
   "contentType": "video/mp4",
   "createdAt": "2022-02-02T00:00:00.000Z",
   "expiredAt": "2022-02-02T01:00:00.000Z",
-  "id": "11f967df-da1013ca-19e8e887-b9682398",
+  "id": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78",
   "metadata": Object {
     "custom": "",
     "lastModified": 1635398061454,
@@ -36,7 +36,7 @@ exports[`DiskStorage .list() should return all user files 1`] = `
 Array [
   Object {
     "createdAt": 2022-02-02T00:00:00.000Z,
-    "id": "11f967df-da1013ca-19e8e887-b9682398",
+    "id": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78",
     "modifiedAt": 2022-02-02T00:00:00.000Z,
   },
 ]

diff --git a/test/__snapshots__/gcs-storage.spec.ts.snap b/test/__snapshots__/gcs-storage.spec.ts.snap
@@ -5,7 +5,7 @@ Object {
   "bytesWritten": 6,
   "contentType": "video/mp4",
   "createdAt": "2022-02-02T00:00:00.000Z",
-  "id": "11f967df-da1013ca-19e8e887-b9682398",
+  "id": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78",
   "metadata": Object {
     "custom": "",
     "lastModified": 1635398061454,
@@ -28,7 +28,7 @@ GCSFile {
   "contentType": "video/mp4",
   "createdAt": "2022-02-02T00:00:00.000Z",
   "expiredAt": "2022-02-02T01:00:00.000Z",
-  "id": "11f967df-da1013ca-19e8e887-b9682398",
+  "id": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78",
   "metadata": Object {
     "custom": "",
     "lastModified": 1635398061454,
@@ -59,7 +59,7 @@ exports[`GCStorage .create() should request api and set status and uri 2`] = `
         "params": Object {
           "alt": "media",
         },
-        "url": "https://storage.googleapis.com/storage/v1/b/test-bucket/o/11f967df-da1013ca-19e8e887-b9682398.META",
+        "url": "https://storage.googleapis.com/storage/v1/b/test-bucket/o/f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78.META",
       },
     ],
     Array [
@@ -82,13 +82,13 @@ exports[`GCStorage .create() should request api and set status and uri 2`] = `
     ],
     Array [
       Object {
-        "body": "{\\"bytesWritten\\":null,\\"name\\":\\"userId/testfile.mp4\\",\\"metadata\\":{\\"name\\":\\"testfile.mp4\\",\\"size\\":64,\\"mimeType\\":\\"video/mp4\\",\\"lastModified\\":1635398061454,\\"custom\\":\\"\\",\\"sha1\\":\\"ZAPAntzKARqtb+j3B529GAOf3kI=\\"},\\"originalName\\":\\"testfile.mp4\\",\\"contentType\\":\\"video/mp4\\",\\"size\\":64,\\"userId\\":\\"userId\\",\\"id\\":\\"11f967df-da1013ca-19e8e887-b9682398\\",\\"uri\\":\\"http://api.com?upload_id=123456789\\",\\"createdAt\\":\\"2022-02-02T00:00:00.000Z\\",\\"expiredAt\\":\\"2022-02-02T01:00:00.000Z\\"}",
+        "body": "{\\"bytesWritten\\":null,\\"name\\":\\"userId/testfile.mp4\\",\\"metadata\\":{\\"name\\":\\"testfile.mp4\\",\\"size\\":64,\\"mimeType\\":\\"video/mp4\\",\\"lastModified\\":1635398061454,\\"custom\\":\\"\\",\\"sha1\\":\\"ZAPAntzKARqtb+j3B529GAOf3kI=\\"},\\"originalName\\":\\"testfile.mp4\\",\\"contentType\\":\\"video/mp4\\",\\"size\\":64,\\"userId\\":\\"userId\\",\\"id\\":\\"f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78\\",\\"uri\\":\\"http://api.com?upload_id=123456789\\",\\"createdAt\\":\\"2022-02-02T00:00:00.000Z\\",\\"expiredAt\\":\\"2022-02-02T01:00:00.000Z\\"}",
         "headers": Object {
           "Content-Type": "application/json; charset=utf-8",
         },
         "method": "POST",
         "params": Object {
-          "name": "11f967df-da1013ca-19e8e887-b9682398.META",
+          "name": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78.META",
           "uploadType": "media",
         },
         "url": "https://storage.googleapis.com/upload/storage/v1/b/test-bucket/o",
@@ -120,7 +120,7 @@ exports[`GCStorage .list() should return all user files 1`] = `
 Array [
   Object {
     "createdAt": Date { NaN },
-    "id": "11f967df-da1013ca-19e8e887-b9682398",
+    "id": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78",
   },
 ]
 `;
@@ -130,7 +130,7 @@ Object {
   "bytesWritten": 64,
   "contentType": "video/mp4",
   "createdAt": "2022-02-02T00:00:00.000Z",
-  "id": "11f967df-da1013ca-19e8e887-b9682398",
+  "id": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78",
   "metadata": Object {
     "custom": "",
     "lastModified": 1635398061454,

diff --git a/test/__snapshots__/s3-storage.spec.ts.snap b/test/__snapshots__/s3-storage.spec.ts.snap
@@ -6,7 +6,7 @@ Object {
   "bytesWritten": 0,
   "contentType": "video/mp4",
   "createdAt": "2022-02-02T00:00:00.000Z",
-  "id": "11f967df-da1013ca-19e8e887-b9682398",
+  "id": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78",
   "metadata": Object {
     "custom": "",
     "lastModified": 1635398061454,
@@ -30,7 +30,7 @@ S3File {
   "contentType": "video/mp4",
   "createdAt": "2022-02-02T00:00:00.000Z",
   "expiredAt": "2022-02-02T01:00:00.000Z",
-  "id": "11f967df-da1013ca-19e8e887-b9682398",
+  "id": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78",
   "metadata": Object {
     "custom": "",
     "lastModified": 1635398061454,
@@ -62,7 +62,7 @@ Object {
   "bytesWritten": 64,
   "contentType": "video/mp4",
   "createdAt": "2022-02-02T00:00:00.000Z",
-  "id": "11f967df-da1013ca-19e8e887-b9682398",
+  "id": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78",
   "metadata": Object {
     "custom": "",
     "lastModified": 1635398061454,
@@ -87,7 +87,7 @@ Object {
   "bytesWritten": 0,
   "contentType": "video/mp4",
   "createdAt": "2022-02-02T00:00:00.000Z",
-  "id": "11f967df-da1013ca-19e8e887-b9682398",
+  "id": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78",
   "metadata": Object {
     "custom": "",
     "lastModified": 1635398061454,

diff --git a/test/shared/config.ts b/test/shared/config.ts
@@ -21,7 +21,7 @@ async function* generateChunks(): AsyncIterableIterator<string> {
   yield 'xz'.repeat(16);
   yield 'xz'.repeat(16);
 }
-const id = '11f967df-da1013ca-19e8e887-b9682398';
+const id = 'f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78';
 const contentType = 'video/mp4';
 const fileAsBuffer = Buffer.from('xz'.repeat(32));
 const size = Buffer.byteLength(fileAsBuffer);

diff --git a/test/util.spec.ts b/test/util.spec.ts
@@ -164,6 +164,10 @@ describe('utils', () => {
       expect(utils.fnv('123456')).toBe('9995b6aa');
     });
 
+    it('fnv64', () => {
+      expect(utils.fnv64('123456')).toBe('f6e3ed7e0e67290a');
+    });
+
     it('pick', () => {
       expect(utils.pick({ test: 'test', rest: 'rest' }, ['test'])).toMatchObject({
         test: 'test'