From 609932ed0827cd2f55ce46c6cf1005527694e290 Mon Sep 17 00:00:00 2001 From: Nicolas DUBIEN Date: Sun, 17 Mar 2019 10:20:15 +0100 Subject: [PATCH] Add fuzzing based tests in Jest (#8012) * Automatic bug detection with property based testing * Switch to built-in fc.anything() * Disable Map into fc.anything() * Rewrite matcher.property.test in TS * Update properties following review * Ignore key ordering for Set and Map * Remove potentially unstable property * One spec by matcher for property based tests * Add missing copyright * Ignore __arbitraries__ * Remove unecessary comments --- jest.config.js | 1 + package.json | 1 + .../__arbitraries__/sharedSettings.ts | 22 +++++++ .../matchers-toContain.property.test.ts | 48 +++++++++++++++ .../matchers-toContainEqual.property.test.ts | 46 +++++++++++++++ .../matchers-toEqual.property.test.ts | 58 +++++++++++++++++++ .../matchers-toStrictEqual.property.test.ts | 49 ++++++++++++++++ yarn.lock | 22 ++++++- 8 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 packages/expect/src/__tests__/__arbitraries__/sharedSettings.ts create mode 100644 packages/expect/src/__tests__/matchers-toContain.property.test.ts create mode 100644 packages/expect/src/__tests__/matchers-toContainEqual.property.test.ts create mode 100644 packages/expect/src/__tests__/matchers-toEqual.property.test.ts create mode 100644 packages/expect/src/__tests__/matchers-toStrictEqual.property.test.ts diff --git a/jest.config.js b/jest.config.js index 1d4488d6904d..c6661b005dea 100644 --- a/jest.config.js +++ b/jest.config.js @@ -31,6 +31,7 @@ module.exports = { ], testEnvironment: './packages/jest-environment-node', testPathIgnorePatterns: [ + '/__arbitraries__/', '/node_modules/', '/examples/', '/e2e/.*/__tests__', diff --git a/package.json b/package.json index 504eeb41d7e3..ae5ee0e2378b 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "eslint-plugin-react": "^7.1.0", "eslint-plugin-relay": "~0.0.19", "execa": "^1.0.0", + "fast-check": "^1.12.0", "glob": "^7.1.1", "graceful-fs": "^4.1.15", "isbinaryfile": "^4.0.0", diff --git a/packages/expect/src/__tests__/__arbitraries__/sharedSettings.ts b/packages/expect/src/__tests__/__arbitraries__/sharedSettings.ts new file mode 100644 index 000000000000..98a48910811e --- /dev/null +++ b/packages/expect/src/__tests__/__arbitraries__/sharedSettings.ts @@ -0,0 +1,22 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + */ + +import fc from 'fast-check'; + +// settings for anything arbitrary +export const anythingSettings = { + key: fc.oneof(fc.string(), fc.constantFrom('k1', 'k2', 'k3')), + withBoxedValues: true, + // Issue #7975 have to be fixed before enabling the generation of Map + withMap: false, + // Issue #7975 have to be fixed before enabling the generation of Set + withSet: false, +}; + +// assertion settings +export const assertSettings = {}; // eg.: {numRuns: 10000} diff --git a/packages/expect/src/__tests__/matchers-toContain.property.test.ts b/packages/expect/src/__tests__/matchers-toContain.property.test.ts new file mode 100644 index 000000000000..29e9b4c1d4e5 --- /dev/null +++ b/packages/expect/src/__tests__/matchers-toContain.property.test.ts @@ -0,0 +1,48 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + */ + +import fc from 'fast-check'; +import { + anythingSettings, + assertSettings, +} from './__arbitraries__/sharedSettings'; + +describe('toContain', () => { + it('should always find the value when inside the array', () => { + fc.assert( + fc.property( + fc.array(fc.anything(anythingSettings)), + fc.array(fc.anything(anythingSettings)), + fc.anything(anythingSettings).filter(v => !Number.isNaN(v)), + (startValues, endValues, v) => { + // Given: startValues, endValues arrays and v value (not NaN) + expect([...startValues, v, ...endValues]).toContain(v); + }, + ), + assertSettings, + ); + }); + + it('should not find the value if it has been cloned into the array', () => { + fc.assert( + fc.property( + fc.array(fc.anything(anythingSettings)), + fc.array(fc.anything(anythingSettings)), + fc.dedup(fc.anything(anythingSettings), 2), + (startValues, endValues, [a, b]) => { + // Given: startValues, endValues arrays + // and [a, b] equal, but not the same values + // with `typeof a === 'object && a !== null` + fc.pre(typeof a === 'object' && a !== null); + expect([...startValues, a, ...endValues]).not.toContain(b); + }, + ), + assertSettings, + ); + }); +}); diff --git a/packages/expect/src/__tests__/matchers-toContainEqual.property.test.ts b/packages/expect/src/__tests__/matchers-toContainEqual.property.test.ts new file mode 100644 index 000000000000..ab6145a645f7 --- /dev/null +++ b/packages/expect/src/__tests__/matchers-toContainEqual.property.test.ts @@ -0,0 +1,46 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + */ + +import fc from 'fast-check'; +import { + anythingSettings, + assertSettings, +} from './__arbitraries__/sharedSettings'; + +describe('toContainEqual', () => { + it('should always find the value when inside the array', () => { + fc.assert( + fc.property( + fc.array(fc.anything(anythingSettings)), + fc.array(fc.anything(anythingSettings)), + fc.anything(anythingSettings), + (startValues, endValues, v) => { + // Given: startValues, endValues arrays and v any value + expect([...startValues, v, ...endValues]).toContainEqual(v); + }, + ), + assertSettings, + ); + }); + + it('should always find the value when cloned inside the array', () => { + fc.assert( + fc.property( + fc.array(fc.anything(anythingSettings)), + fc.array(fc.anything(anythingSettings)), + fc.dedup(fc.anything(anythingSettings), 2), + (startValues, endValues, [a, b]) => { + // Given: startValues, endValues arrays + // and [a, b] identical values + expect([...startValues, a, ...endValues]).toContainEqual(b); + }, + ), + assertSettings, + ); + }); +}); diff --git a/packages/expect/src/__tests__/matchers-toEqual.property.test.ts b/packages/expect/src/__tests__/matchers-toEqual.property.test.ts new file mode 100644 index 000000000000..ca7d88ef7390 --- /dev/null +++ b/packages/expect/src/__tests__/matchers-toEqual.property.test.ts @@ -0,0 +1,58 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + */ + +import fc from 'fast-check'; +import { + anythingSettings, + assertSettings, +} from './__arbitraries__/sharedSettings'; + +describe('toEqual', () => { + it('should be reflexive', () => { + fc.assert( + fc.property(fc.dedup(fc.anything(anythingSettings), 2), ([a, b]) => { + // Given: a and b identical values + expect(a).toEqual(b); + }), + assertSettings, + ); + }); + + it('should be symmetric', () => { + const safeExpectEqual = (a, b) => { + try { + expect(a).toEqual(b); + return true; + } catch (err) { + return false; + } + }; + fc.assert( + fc.property( + fc.anything(anythingSettings), + fc.anything(anythingSettings), + (a, b) => { + // Given: a and b values + // Assert: We expect `expect(a).toEqual(b)` + // to be equivalent to `expect(b).toEqual(a)` + expect(safeExpectEqual(a, b)).toBe(safeExpectEqual(b, a)); + }, + ), + { + ...assertSettings, + examples: [ + [0, 5e-324], // Issue #7941 + // [ + // new Set([false, true]), + // new Set([new Boolean(true), new Boolean(true)]), + // ], // Issue #7975 + ], + }, + ); + }); +}); diff --git a/packages/expect/src/__tests__/matchers-toStrictEqual.property.test.ts b/packages/expect/src/__tests__/matchers-toStrictEqual.property.test.ts new file mode 100644 index 000000000000..70dcd4e5cd1b --- /dev/null +++ b/packages/expect/src/__tests__/matchers-toStrictEqual.property.test.ts @@ -0,0 +1,49 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + */ + +import fc from 'fast-check'; +import { + anythingSettings, + assertSettings, +} from './__arbitraries__/sharedSettings'; + +describe('toStrictEqual', () => { + it('should be reflexive', () => { + fc.assert( + fc.property(fc.dedup(fc.anything(anythingSettings), 2), ([a, b]) => { + // Given: a and b identical values + expect(a).toStrictEqual(b); + }), + assertSettings, + ); + }); + + it('should be symmetric', () => { + const safeExpectStrictEqual = (a, b) => { + try { + expect(a).toStrictEqual(b); + return true; + } catch (err) { + return false; + } + }; + fc.assert( + fc.property( + fc.anything(anythingSettings), + fc.anything(anythingSettings), + (a, b) => { + // Given: a and b values + // Assert: We expect `expect(a).toStrictEqual(b)` + // to be equivalent to `expect(b).toStrictEqual(a)` + expect(safeExpectStrictEqual(a, b)).toBe(safeExpectStrictEqual(b, a)); + }, + ), + assertSettings, + ); + }); +}); diff --git a/yarn.lock b/yarn.lock index 22e5b83b0035..7d32f60984e1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5641,6 +5641,14 @@ fancy-log@^1.3.2: parse-node-version "^1.0.0" time-stamp "^1.0.0" +fast-check@^1.12.0: + version "1.12.0" + resolved "https://registry.yarnpkg.com/fast-check/-/fast-check-1.12.0.tgz#bff7908aa357703f207ee1877ffcf4486a698aa0" + integrity sha512-r/zmvxG/8IsxyOzf9T6wnjJQjSLmymAETeRVfs4/0VzxBmyK48o871BRKmEIgEBLmrg9klh3e4A4BZc3yrH2QQ== + dependencies: + lorem-ipsum "~1.0.6" + pure-rand "^1.6.2" + fast-deep-equal@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz#7b05218ddf9667bf7f370bf7fdb2cb15fdd0aa49" @@ -8348,6 +8356,13 @@ loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.3.1, loose-envify@^1.4 dependencies: js-tokens "^3.0.0 || ^4.0.0" +lorem-ipsum@~1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/lorem-ipsum/-/lorem-ipsum-1.0.6.tgz#69e9ab02bbb0991915d71b5559fe016d526f013f" + integrity sha512-Rx4XH8X4KSDCKAVvWGYlhAfNqdUP5ZdT4rRyf0jjrvWgtViZimDIlopWNfn/y3lGM5K4uuiAoY28TaD+7YKFrQ== + dependencies: + minimist "~1.2.0" + loud-rejection@^1.0.0: version "1.6.0" resolved "https://registry.yarnpkg.com/loud-rejection/-/loud-rejection-1.6.0.tgz#5b46f80147edee578870f086d04821cf998e551f" @@ -8912,7 +8927,7 @@ minimist@0.0.8: resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" integrity sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0= -minimist@1.2.0, minimist@^1.1.1, minimist@^1.1.3, minimist@^1.2.0: +minimist@1.2.0, minimist@^1.1.1, minimist@^1.1.3, minimist@^1.2.0, minimist@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" integrity sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ= @@ -10690,6 +10705,11 @@ punycode@^2.1.0, punycode@^2.1.1: resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== +pure-rand@^1.6.2: + version "1.6.2" + resolved "https://registry.yarnpkg.com/pure-rand/-/pure-rand-1.6.2.tgz#90b3ae78efe36f7e6e27bfffedf934f77382e6e6" + integrity sha512-HNwHOH63m7kCxe0kWEe5jSLwJiL2N83RUUN8POniFuZS+OsbFcMWlvXgxIU2nwKy2zYG2bQan40WBNK4biYPRg== + q@0.9.7: version "0.9.7" resolved "https://registry.yarnpkg.com/q/-/q-0.9.7.tgz#4de2e6cb3b29088c9e4cbc03bf9d42fb96ce2f75"