diff --git a/package-lock.json b/package-lock.json index 2e7ec2da7640..63793fee320e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,6 +40,7 @@ "basic-auth-parser": "^0.0.2", "body-parser": "^1.20.0", "commitlint": "^17.1.2", + "cross-env": "^7.0.3", "deep-equal": "^2.0.5", "eslint": "^8.24.0", "express": "^4.18.1", @@ -6266,6 +6267,24 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "dev": true }, + "node_modules/cross-env": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", + "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.1" + }, + "bin": { + "cross-env": "src/bin/cross-env.js", + "cross-env-shell": "src/bin/cross-env-shell.js" + }, + "engines": { + "node": ">=10.14", + "npm": ">=6", + "yarn": ">=1" + } + }, "node_modules/cross-fetch": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz", diff --git a/package.json b/package.json index 05b3732ccd97..d06f3d448243 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "postci:build": "node ./scripts/typescript_fixes.mjs", "test": "jest --silent", "test:e2e": "node test/e2e/run.mjs", + "test:full": "cross-env CRAWLEE_DIFFICULT_TESTS=1 jest --silent", "coverage": "jest --coverage", "publish:next": "lerna publish from-package --contents dist --dist-tag next --force-publish", "release:next": "npm run build && npm run publish:next", @@ -80,6 +81,7 @@ "basic-auth-parser": "^0.0.2", "body-parser": "^1.20.0", "commitlint": "^17.1.2", + "cross-env": "^7.0.3", "deep-equal": "^2.0.5", "eslint": "^8.24.0", "express": "^4.18.1", diff --git a/packages/memory-storage/src/resource-clients/key-value-store.ts b/packages/memory-storage/src/resource-clients/key-value-store.ts index 1da291e89e29..5b3ae1699734 100644 --- a/packages/memory-storage/src/resource-clients/key-value-store.ts +++ b/packages/memory-storage/src/resource-clients/key-value-store.ts @@ -211,7 +211,7 @@ export class KeyValueStoreClient extends BaseClient { async setRecord(record: storage.KeyValueStoreRecord): Promise { s.object({ key: s.string.lengthGreaterThan(0), - value: s.union(s.null, s.string, s.number, s.object({}).passthrough), + value: s.union(s.null, s.string, s.number, s.instance(Buffer), s.object({}).passthrough), contentType: s.string.lengthGreaterThan(0).optional, }).parse(record); diff --git a/packages/memory-storage/test/no-crash-on-big-buffers.test.ts b/packages/memory-storage/test/no-crash-on-big-buffers.test.ts new file mode 100644 index 000000000000..84d0e859576d --- /dev/null +++ b/packages/memory-storage/test/no-crash-on-big-buffers.test.ts @@ -0,0 +1,45 @@ +// https://github.com/apify/crawlee/issues/1732 +// https://github.com/apify/crawlee/issues/1710 + +import { rm } from 'node:fs/promises'; +import { resolve } from 'node:path'; +import { MemoryStorage } from '@crawlee/memory-storage'; +import type { KeyValueStoreClient, KeyValueStoreInfo } from '@crawlee/types'; + +describe('MemoryStorage should not crash when saving a big buffer', () => { + const tmpLocation = resolve(__dirname, './tmp/no-buffer-crash'); + const storage = new MemoryStorage({ + localDataDirectory: tmpLocation, + persistStorage: false, + }); + + let kvs: KeyValueStoreInfo; + let store: KeyValueStoreClient; + + beforeAll(async () => { + kvs = await storage.keyValueStores().getOrCreate(); + store = storage.keyValueStore(kvs.id); + }); + + afterAll(async () => { + await rm(tmpLocation, { force: true, recursive: true }); + }); + + test('should not crash when saving a big buffer', async () => { + let zip: Buffer; + + if (process.env.CRAWLEE_DIFFICULT_TESTS) { + const numbers = Array.from(([...Array(18_100_000).keys()]).map((i) => i * 3_000_000)); + + zip = Buffer.from([...numbers]); + } else { + zip = Buffer.from([...Array(100_000)].map((i) => i * 8)); + } + + try { + await store.setRecord({ key: 'owo.zip', value: zip }); + } catch (err) { + expect(err).not.toBeDefined(); + } + }); +});