♻️ Implement sliced based generator for arrays (#3047)

Sliced based generators will make it possible to implement features like the long awaited one: "Helper to deal with dangerous strings" #484. Indeed they will make arrays able to generate pre-defined values so that some barely impossible to generate values like `['_','_','p','r','o','t','o','_','_']` will be more likely for strings.

The current PR mostly adds the internals to make it possible within arrays. But there is for the moment nothing to connect it, neither externally for fast-check's users nor internally for our strings. Here we mostly pave the way for it without really making it real. So from an external point of view, this PR mostly deals with refactoring internals.

.yarn/versions/2aeef278.yml

@@ -0,0 +1,24 @@
+releases:
+  fast-check: patch
+
+declined:
+  - "@fast-check/monorepo"
+  - "@fast-check/examples"
+  - "@fast-check/ava"
+  - "@fast-check/jest"
+  - "@fast-check/test-ava-bundle-cjs"
+  - "@fast-check/test-ava-bundle-esm"
+  - "@fast-check/test-bundle-esbuild-with-import"
+  - "@fast-check/test-bundle-esbuild-with-require"
+  - "@fast-check/test-bundle-node-extension-cjs"
+  - "@fast-check/test-bundle-node-extension-mjs"
+  - "@fast-check/test-bundle-node-with-import"
+  - "@fast-check/test-bundle-node-with-require"
+  - "@fast-check/test-bundle-rollup-with-import"
+  - "@fast-check/test-bundle-rollup-with-require"
+  - "@fast-check/test-bundle-webpack-with-import"
+  - "@fast-check/test-bundle-webpack-with-require"
+  - "@fast-check/test-jest-bundle-cjs"
+  - "@fast-check/test-jest-bundle-esm"
+  - "@fast-check/test-minimal-support"
+  - "@fast-check/test-types"

packages/fast-check/src/arbitrary/_internals/ArrayArbitrary.ts

@@ -7,6 +7,7 @@ import { Arbitrary } from '../../check/arbitrary/definition/Arbitrary';
 import { Value } from '../../check/arbitrary/definition/Value';
 import { CustomSetBuilder } from './interfaces/CustomSet';
 import { DepthContext, DepthIdentifier, getDepthContextFor } from './helpers/DepthContext';
+import { buildSlicedGenerator } from './helpers/BuildSlicedGenerator';
 
 /** @internal */
 type ArrayArbitraryContext = {
@@ -75,13 +76,14 @@ export class ArrayArbitrary<T> extends Arbitrary<T[]> {
   ): Value<T>[] {
     let numSkippedInRow = 0;
     const s = setBuilder();
+    const slicedGenerator = buildSlicedGenerator(this.arb, mrng, [], biasFactorItems);
     // Try to append into items up to the target size
     // We may reject some items as they are already part of the set
     // so we need to retry and generate other ones. In order to prevent infinite loop,
     // we accept a max of maxGeneratedLength consecutive failures. This circuit breaker may cause
     // generated to be smaller than the minimal accepted one.
     while (s.size() < N && numSkippedInRow < this.maxGeneratedLength) {
-      const current = this.arb.generate(mrng, biasFactorItems);
+      const current = slicedGenerator.next();
       if (s.tryAdd(current)) {
         numSkippedInRow = 0;
       } else {
@@ -108,8 +110,10 @@ export class ArrayArbitrary<T> extends Arbitrary<T[]> {
 
   private generateNItems(N: number, mrng: Random, biasFactorItems: number | undefined): Value<T>[] {
     const items: Value<T>[] = [];
+    const slicedGenerator = buildSlicedGenerator(this.arb, mrng, [], biasFactorItems);
+    slicedGenerator.attemptExact(N);
     for (let index = 0; index !== N; ++index) {
-      const current = this.arb.generate(mrng, biasFactorItems);
+      const current = slicedGenerator.next();
       items.push(current);
     }
     return items;

packages/fast-check/src/arbitrary/_internals/helpers/BuildSlicedGenerator.ts

@@ -0,0 +1,31 @@
+import { Arbitrary } from '../../../check/arbitrary/definition/Arbitrary';
+import { Random } from '../../../random/generator/Random';
+import { NoopSlicedGenerator } from '../implementations/NoopSlicedGenerator';
+import { SlicedBasedGenerator } from '../implementations/SlicedBasedGenerator';
+import { SlicedGenerator } from '../interfaces/SlicedGenerator';
+
+/**
+ * Build a {@link SlicedGenerator}
+ *
+ * @param arb - Arbitrary able to generate values
+ * @param mrng - Random number generator
+ * @param slices - Slices to be used (WARNING: while we accept no slices, slices themselves must never empty)
+ * @param biasFactor - The current bias factor
+ *
+ * @internal
+ */
+export function buildSlicedGenerator<T>(
+  arb: Arbitrary<T>,
+  mrng: Random,
+  slices: T[][],
+  biasFactor: number | undefined
+): SlicedGenerator<T> {
+  // We by-pass any slice-based logic if one of:
+  // - no bias
+  // - no slices
+  // - not our turn: we only apply the slices fallbacks on 1 run over biasFactor
+  if (biasFactor === undefined || slices.length === 0 || mrng.nextInt(1, biasFactor) !== 1) {
+    return new NoopSlicedGenerator(arb, mrng, biasFactor);
+  }
+  return new SlicedBasedGenerator(arb, mrng, slices, biasFactor);
+}

packages/fast-check/src/arbitrary/_internals/implementations/NoopSlicedGenerator.ts

@@ -0,0 +1,19 @@
+import { Arbitrary } from '../../../check/arbitrary/definition/Arbitrary';
+import { Value } from '../../../check/arbitrary/definition/Value';
+import { Random } from '../../../random/generator/Random';
+import { SlicedGenerator } from '../interfaces/SlicedGenerator';
+
+/** @internal */
+export class NoopSlicedGenerator<T> implements SlicedGenerator<T> {
+  constructor(
+    private readonly arb: Arbitrary<T>,
+    private readonly mrng: Random,
+    private readonly biasFactor: number | undefined
+  ) {}
+  attemptExact(): void {
+    return;
+  }
+  next(): Value<T> {
+    return this.arb.generate(this.mrng, this.biasFactor);
+  }
+}

packages/fast-check/src/arbitrary/_internals/implementations/SlicedBasedGenerator.ts

@@ -0,0 +1,60 @@
+import { Arbitrary } from '../../../check/arbitrary/definition/Arbitrary';
+import { Value } from '../../../check/arbitrary/definition/Value';
+import { Random } from '../../../random/generator/Random';
+import { SlicedGenerator } from '../interfaces/SlicedGenerator';
+
+/** @internal */
+export class SlicedBasedGenerator<T> implements SlicedGenerator<T> {
+  private activeSliceIndex = 0;
+  private nextIndexInSlice = 0; // the next index to take from the slice
+  private lastIndexInSlice = -1; // the last index accepted for the current slice
+  constructor(
+    private readonly arb: Arbitrary<T>,
+    private readonly mrng: Random,
+    private readonly slices: T[][],
+    private readonly biasFactor: number
+  ) {}
+  attemptExact(targetLength: number): void {
+    if (targetLength !== 0 && this.mrng.nextInt(1, this.biasFactor) === 1) {
+      // Let's setup the generator for exact matching if any possible
+      const eligibleIndices: number[] = [];
+      for (let index = 0; index !== this.slices.length; ++index) {
+        const slice = this.slices[index];
+        if (slice.length === targetLength) {
+          eligibleIndices.push(index);
+        }
+      }
+      if (eligibleIndices.length === 0) {
+        return;
+      }
+      this.activeSliceIndex = eligibleIndices[this.mrng.nextInt(0, eligibleIndices.length - 1)];
+      this.nextIndexInSlice = 0;
+      this.lastIndexInSlice = targetLength - 1;
+    }
+  }
+  next(): Value<T> {
+    if (this.nextIndexInSlice <= this.lastIndexInSlice) {
+      // We continue on the previously selected slice
+      return new Value(this.slices[this.activeSliceIndex][this.nextIndexInSlice++], undefined);
+    }
+    if (this.mrng.nextInt(1, this.biasFactor) !== 1) {
+      // We don't use the slices
+      return this.arb.generate(this.mrng, this.biasFactor);
+    }
+    // We update the active slice
+    this.activeSliceIndex = this.mrng.nextInt(0, this.slices.length - 1);
+    if (this.mrng.nextInt(1, this.biasFactor) !== 1) {
+      // We will consider the whole slice and not a sub-set of it
+      this.nextIndexInSlice = 1;
+      this.lastIndexInSlice = this.slices.length - 1;
+      return new Value(this.slices[this.activeSliceIndex][0], undefined);
+    }
+    const slice = this.slices[this.activeSliceIndex];
+    const rangeBoundaryA = this.mrng.nextInt(0, slice.length - 1);
+    const rangeBoundaryB = this.mrng.nextInt(0, slice.length - 1);
+    this.nextIndexInSlice = Math.min(rangeBoundaryA, rangeBoundaryB);
+    this.lastIndexInSlice = Math.max(rangeBoundaryA, rangeBoundaryB);
+    this.lastIndexInSlice = this.slices.length - 1;
+    return new Value(this.slices[this.activeSliceIndex][this.nextIndexInSlice++], undefined);
+  }
+}

packages/fast-check/src/arbitrary/_internals/interfaces/SlicedGenerator.ts

@@ -0,0 +1,20 @@
+import { Value } from '../../../check/arbitrary/definition/Value';
+
+/**
+ * Internal helper responsible to fallback from time to time to already pre-computed entries
+ * provided by the caller and referred as slices
+ * @internal
+ */
+export type SlicedGenerator<T> = {
+  /**
+   * Warm-up the generator with an idea of the exact size.
+   * It may be used by the generator to favor some values (the ones with the right length) instead of others.
+   * @internal
+   */
+  attemptExact: (targetLength: number) => void;
+  /**
+   * Compute the next generated value
+   * @internal
+   */
+  next: () => Value<T>;
+};

packages/fast-check/test/unit/arbitrary/_internals/implementations/SlicedBasedGenerator.spec.ts

@@ -0,0 +1,119 @@
+import fc from 'fast-check';
+import { SlicedBasedGenerator } from '../../../../../src/arbitrary/_internals/implementations/SlicedBasedGenerator';
+import { Value } from '../../../../../src/check/arbitrary/definition/Value';
+import { fakeArbitrary } from '../../__test-helpers__/ArbitraryHelpers';
+import { fakeRandom } from '../../__test-helpers__/RandomHelpers';
+
+describe('SlicedBasedGenerator', () => {
+  describe('attemptExact', () => {
+    it('should take one of the provided slices and return it item by item', () => {
+      fc.assert(
+        fc.property(
+          fc.array(fc.array(fc.anything(), { minLength: 1 }), { minLength: 1 }),
+          fc.nat(),
+          fc.nat(),
+          fc.integer({ min: 2 }),
+          (slices, targetLengthMod, selectOneMod, biasFactor) => {
+            // Arrange
+            const { instance: arb } = fakeArbitrary();
+            const { instance: mrng, nextInt } = fakeRandom();
+            nextInt
+              .mockReturnValueOnce(1) // 1 to go for "value from slices"
+              .mockImplementationOnce((min, max) => (selectOneMod % (max - min + 1)) + min);
+            const targetLength = slices[targetLengthMod % slices.length].length;
+
+            // Act
+            const generator = new SlicedBasedGenerator(arb, mrng, slices, biasFactor);
+            const readFromGenerator: unknown[] = [];
+            generator.attemptExact(targetLength);
+            for (let index = 0; index !== targetLength; ++index) {
+              readFromGenerator.push(generator.next().value);
+            }
+
+            // Assert
+            expect(nextInt).toHaveBeenCalledTimes(2); // only called twice: 1/ to bias to one of the slices 2/ to select which one
+            expect(nextInt).toHaveBeenCalledWith(1, biasFactor);
+            expect(slices).toContainEqual(readFromGenerator);
+          }
+        )
+      );
+    });
+  });
+
+  describe('next', () => {
+    it('should only go for values coming from the source arbitrary when tossing for unbias', () => {
+      fc.assert(
+        fc.property(
+          fc.array(fc.array(fc.anything(), { minLength: 1 }), { minLength: 1 }),
+          fc.infiniteStream(fc.anything()),
+          fc.nat({ max: 10 }),
+          fc.integer({ min: 2 }),
+          (slices, streamValues, targetLength, biasFactor) => {
+            // Arrange
+            const producedValues: unknown[] = [];
+            const { instance: arb, generate } = fakeArbitrary();
+            generate.mockImplementation(() => {
+              const value = streamValues.next().value;
+              const context = streamValues.next().value;
+              producedValues.push(value);
+              return new Value(value, context);
+            });
+            const { instance: mrng, nextInt } = fakeRandom();
+            nextInt.mockImplementation((_min, max) => max); // >min ie in [min+1,max] corresponds to unbiased
+
+            // Act
+            const generator = new SlicedBasedGenerator(arb, mrng, slices, biasFactor);
+            const readFromGenerator: unknown[] = [];
+            generator.attemptExact(targetLength);
+            for (let index = 0; index !== targetLength; ++index) {
+              readFromGenerator.push(generator.next().value);
+            }
+
+            // Assert
+            expect(generate).toHaveBeenCalledTimes(targetLength);
+            expect(readFromGenerator).toEqual(producedValues);
+          }
+        )
+      );
+    });
+
+    it('should only go for values coming from the slices when tossing for bias', () => {
+      fc.assert(
+        fc.property(
+          fc.array(fc.array(fc.anything(), { minLength: 1 }), { minLength: 1 }),
+          fc.infiniteStream(fc.nat()),
+          fc.nat({ max: 10 }),
+          fc.integer({ min: 2 }),
+          fc.boolean(),
+          (slices, streamModValues, targetLength, biasFactor, withAttemptExact) => {
+            // Arrange
+            const { instance: arb, generate } = fakeArbitrary();
+            const { instance: mrng, nextInt } = fakeRandom();
+
+            // Act
+            const generator = new SlicedBasedGenerator(arb, mrng, slices, biasFactor);
+            const readFromGenerator: unknown[] = [];
+            if (withAttemptExact) {
+              nextInt.mockImplementation((_min, max) => max); // no bias for attemptExact
+              generator.attemptExact(targetLength);
+            }
+            for (let index = 0; index !== targetLength; ++index) {
+              let returnedBias = false;
+              nextInt.mockImplementation((min, max) => {
+                if (!returnedBias) {
+                  returnedBias = true;
+                  return min; // ask for bias, to make sure we use slices
+                }
+                return (streamModValues.next().value % (max - min + 1)) + min; // pure random for next calls
+              });
+              readFromGenerator.push(generator.next().value);
+            }
+
+            // Assert
+            expect(generate).not.toHaveBeenCalled();
+          }
+        )
+      );
+    });
+  });
+});