From f41725865e068d9cab068ecd63b9c4ba0087f02c Mon Sep 17 00:00:00 2001 From: q_h Date: Thu, 30 Jun 2022 19:54:33 +0300 Subject: [PATCH] fix(core): reduce probability of id collisions (#594) use fnv1a 64bit for id segments --- packages/core/src/utils/primitives.ts | 16 +++++++++++++++- test/__snapshots__/disk-storage.spec.ts.snap | 4 ++-- test/__snapshots__/gcs-storage.spec.ts.snap | 14 +++++++------- test/__snapshots__/s3-storage.spec.ts.snap | 8 ++++---- test/shared/config.ts | 2 +- test/util.spec.ts | 4 ++++ 6 files changed, 33 insertions(+), 15 deletions(-) diff --git a/packages/core/src/utils/primitives.ts b/packages/core/src/utils/primitives.ts index c7e6c364..d3876988 100644 --- a/packages/core/src/utils/primitives.ts +++ b/packages/core/src/utils/primitives.ts @@ -28,6 +28,20 @@ export function fnv(str: string): string { return (hash >>> 0).toString(16); } +/** + * FNV1A64 hash + */ +export function fnv64(str: string): string { + let hash = BigInt('14695981039346656037'); + const offset = BigInt(1099511628211); + const len = str.length; + for (let i = 0; i < len; i++) { + hash ^= BigInt(str.charCodeAt(i)); + hash *= offset; + } + return BigInt.asUintN(64, hash).toString(16); +} + export function mapValues( object: Record, func: (value: any) => T @@ -92,4 +106,4 @@ export const memoize = (fn: (val: T) => K): ((val: T) => K) => { return cached; }; -export const hash = memoize(fnv); +export const hash = memoize(fnv64); diff --git a/test/__snapshots__/disk-storage.spec.ts.snap b/test/__snapshots__/disk-storage.spec.ts.snap index 1be083b3..d91a29ca 100644 --- a/test/__snapshots__/disk-storage.spec.ts.snap +++ b/test/__snapshots__/disk-storage.spec.ts.snap @@ -15,7 +15,7 @@ DiskFile { "contentType": "video/mp4", "createdAt": "2022-02-02T00:00:00.000Z", "expiredAt": "2022-02-02T01:00:00.000Z", - "id": "11f967df-da1013ca-19e8e887-b9682398", + "id": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78", "metadata": Object { "custom": "", "lastModified": 1635398061454, @@ -36,7 +36,7 @@ exports[`DiskStorage .list() should return all user files 1`] = ` Array [ Object { "createdAt": 2022-02-02T00:00:00.000Z, - "id": "11f967df-da1013ca-19e8e887-b9682398", + "id": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78", "modifiedAt": 2022-02-02T00:00:00.000Z, }, ] diff --git a/test/__snapshots__/gcs-storage.spec.ts.snap b/test/__snapshots__/gcs-storage.spec.ts.snap index 07b41c45..0689c45d 100644 --- a/test/__snapshots__/gcs-storage.spec.ts.snap +++ b/test/__snapshots__/gcs-storage.spec.ts.snap @@ -5,7 +5,7 @@ Object { "bytesWritten": 6, "contentType": "video/mp4", "createdAt": "2022-02-02T00:00:00.000Z", - "id": "11f967df-da1013ca-19e8e887-b9682398", + "id": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78", "metadata": Object { "custom": "", "lastModified": 1635398061454, @@ -28,7 +28,7 @@ GCSFile { "contentType": "video/mp4", "createdAt": "2022-02-02T00:00:00.000Z", "expiredAt": "2022-02-02T01:00:00.000Z", - "id": "11f967df-da1013ca-19e8e887-b9682398", + "id": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78", "metadata": Object { "custom": "", "lastModified": 1635398061454, @@ -59,7 +59,7 @@ exports[`GCStorage .create() should request api and set status and uri 2`] = ` "params": Object { "alt": "media", }, - "url": "https://storage.googleapis.com/storage/v1/b/test-bucket/o/11f967df-da1013ca-19e8e887-b9682398.META", + "url": "https://storage.googleapis.com/storage/v1/b/test-bucket/o/f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78.META", }, ], Array [ @@ -82,13 +82,13 @@ exports[`GCStorage .create() should request api and set status and uri 2`] = ` ], Array [ Object { - "body": "{\\"bytesWritten\\":null,\\"name\\":\\"userId/testfile.mp4\\",\\"metadata\\":{\\"name\\":\\"testfile.mp4\\",\\"size\\":64,\\"mimeType\\":\\"video/mp4\\",\\"lastModified\\":1635398061454,\\"custom\\":\\"\\",\\"sha1\\":\\"ZAPAntzKARqtb+j3B529GAOf3kI=\\"},\\"originalName\\":\\"testfile.mp4\\",\\"contentType\\":\\"video/mp4\\",\\"size\\":64,\\"userId\\":\\"userId\\",\\"id\\":\\"11f967df-da1013ca-19e8e887-b9682398\\",\\"uri\\":\\"http://api.com?upload_id=123456789\\",\\"createdAt\\":\\"2022-02-02T00:00:00.000Z\\",\\"expiredAt\\":\\"2022-02-02T01:00:00.000Z\\"}", + "body": "{\\"bytesWritten\\":null,\\"name\\":\\"userId/testfile.mp4\\",\\"metadata\\":{\\"name\\":\\"testfile.mp4\\",\\"size\\":64,\\"mimeType\\":\\"video/mp4\\",\\"lastModified\\":1635398061454,\\"custom\\":\\"\\",\\"sha1\\":\\"ZAPAntzKARqtb+j3B529GAOf3kI=\\"},\\"originalName\\":\\"testfile.mp4\\",\\"contentType\\":\\"video/mp4\\",\\"size\\":64,\\"userId\\":\\"userId\\",\\"id\\":\\"f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78\\",\\"uri\\":\\"http://api.com?upload_id=123456789\\",\\"createdAt\\":\\"2022-02-02T00:00:00.000Z\\",\\"expiredAt\\":\\"2022-02-02T01:00:00.000Z\\"}", "headers": Object { "Content-Type": "application/json; charset=utf-8", }, "method": "POST", "params": Object { - "name": "11f967df-da1013ca-19e8e887-b9682398.META", + "name": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78.META", "uploadType": "media", }, "url": "https://storage.googleapis.com/upload/storage/v1/b/test-bucket/o", @@ -120,7 +120,7 @@ exports[`GCStorage .list() should return all user files 1`] = ` Array [ Object { "createdAt": Date { NaN }, - "id": "11f967df-da1013ca-19e8e887-b9682398", + "id": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78", }, ] `; @@ -130,7 +130,7 @@ Object { "bytesWritten": 64, "contentType": "video/mp4", "createdAt": "2022-02-02T00:00:00.000Z", - "id": "11f967df-da1013ca-19e8e887-b9682398", + "id": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78", "metadata": Object { "custom": "", "lastModified": 1635398061454, diff --git a/test/__snapshots__/s3-storage.spec.ts.snap b/test/__snapshots__/s3-storage.spec.ts.snap index 4c3dfe06..71ed4d1b 100644 --- a/test/__snapshots__/s3-storage.spec.ts.snap +++ b/test/__snapshots__/s3-storage.spec.ts.snap @@ -6,7 +6,7 @@ Object { "bytesWritten": 0, "contentType": "video/mp4", "createdAt": "2022-02-02T00:00:00.000Z", - "id": "11f967df-da1013ca-19e8e887-b9682398", + "id": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78", "metadata": Object { "custom": "", "lastModified": 1635398061454, @@ -30,7 +30,7 @@ S3File { "contentType": "video/mp4", "createdAt": "2022-02-02T00:00:00.000Z", "expiredAt": "2022-02-02T01:00:00.000Z", - "id": "11f967df-da1013ca-19e8e887-b9682398", + "id": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78", "metadata": Object { "custom": "", "lastModified": 1635398061454, @@ -62,7 +62,7 @@ Object { "bytesWritten": 64, "contentType": "video/mp4", "createdAt": "2022-02-02T00:00:00.000Z", - "id": "11f967df-da1013ca-19e8e887-b9682398", + "id": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78", "metadata": Object { "custom": "", "lastModified": 1635398061454, @@ -87,7 +87,7 @@ Object { "bytesWritten": 0, "contentType": "video/mp4", "createdAt": "2022-02-02T00:00:00.000Z", - "id": "11f967df-da1013ca-19e8e887-b9682398", + "id": "f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78", "metadata": Object { "custom": "", "lastModified": 1635398061454, diff --git a/test/shared/config.ts b/test/shared/config.ts index e79506f5..41d42f4b 100644 --- a/test/shared/config.ts +++ b/test/shared/config.ts @@ -21,7 +21,7 @@ async function* generateChunks(): AsyncIterableIterator { yield 'xz'.repeat(16); yield 'xz'.repeat(16); } -const id = '11f967df-da1013ca-19e8e887-b9682398'; +const id = 'f7d13faa74e2475f-e8fed598250d10ea-7f59007b4b7cf67-120941ca7dc37b78'; const contentType = 'video/mp4'; const fileAsBuffer = Buffer.from('xz'.repeat(32)); const size = Buffer.byteLength(fileAsBuffer); diff --git a/test/util.spec.ts b/test/util.spec.ts index d028a80a..9e3d6655 100644 --- a/test/util.spec.ts +++ b/test/util.spec.ts @@ -164,6 +164,10 @@ describe('utils', () => { expect(utils.fnv('123456')).toBe('9995b6aa'); }); + it('fnv64', () => { + expect(utils.fnv64('123456')).toBe('f6e3ed7e0e67290a'); + }); + it('pick', () => { expect(utils.pick({ test: 'test', rest: 'rest' }, ['test'])).toMatchObject({ test: 'test'