Assert: Introduce QUnit.config.countStepsAsOne

Closes #1779.

src/assert.js

@@ -55,6 +55,7 @@ class Assert {
     // Since the steps array is just string values, we can clone with slice
     const actualStepsClone = this.test.steps.slice();
     this.deepEqual(actualStepsClone, steps, message);
+    this.test.stepsCount += this.test.steps.length;
     this.test.steps.length = 0;
   }
 

src/core/config.js

@@ -17,6 +17,8 @@ const config = {
   // If false, all failing tests will be expanded
   collapse: true,
 
+  countStepsAsOne: false,
+
   // TODO: Make explicit in QUnit 3.
   // current: undefined,
 
@@ -128,6 +130,7 @@ const config = {
 
   // Internal state
   _deprecated_timeout_shown: false,
+  _deprecated_countEachStep_shown: false,
   blocking: true,
   callbacks: {},
   modules: [],

src/test.js

@@ -24,6 +24,9 @@ export default function Test (settings) {
   this.assertions = [];
   this.module = config.currentModule;
   this.steps = [];
+  // This powers the QUnit.config.countStepsAsOne feature.
+  // https://github.com/qunitjs/qunit/pull/1775
+  this.stepsCount = 0;
   this.timeout = undefined;
   this.data = undefined;
   this.withData = false;
@@ -357,13 +360,41 @@ Test.prototype = {
         `after using assert.step(). Unverified steps: ${stepsList}`, this.stack);
     }
 
+    if (
+      !config._deprecated_countEachStep_shown &&
+      !config.countStepsAsOne &&
+      this.expected !== null &&
+      this.stepsCount
+    ) {
+      config._deprecated_countEachStep_shown = true;
+      if (config.requireExpects) {
+        Logger.warn('Counting each assert.step() for assert.expect() is changing in QUnit 3.0. You can enable QUnit.config.countStepsAsOne to prepare for the upgrade. https://qunitjs.com/api/assert/expect/');
+      } else {
+        Logger.warn('Counting each assert.step() for assert.expect() is changing in QUnit 3.0. Omit assert.expect() from tests that use assert.step(), or enable QUnit.config.countStepsAsOne to prepare for the upgrade. https://qunitjs.com/api/assert/expect/');
+      }
+    }
+
+    const actualCountForExpect = config.countStepsAsOne
+      ? (this.assertions.length - this.stepsCount)
+      : this.assertions.length;
+
     if (config.requireExpects && this.expected === null) {
       this.pushFailure('Expected number of assertions to be defined, but expect() was ' +
         'not called.', this.stack);
-    } else if (this.expected !== null && this.expected !== this.assertions.length) {
+    } else if (this.expected !== null && this.expected !== actualCountForExpect &&
+      (this.stepsCount && this.expected === (this.assertions.length - this.stepsCount) && !config.countStepsAsOne)
+    ) {
+      this.pushFailure('Expected ' + this.expected + ' assertions, but ' +
+        actualCountForExpect + ' were run\nIt looks like you might prefer to enable QUnit.config.countStepsAsOne, which will become the default in QUnit 3.0. https://qunitjs.com/api/assert/expect/', this.stack);
+    } else if (this.expected !== null && this.expected !== actualCountForExpect &&
+      (this.stepsCount && this.expected === this.assertions.length && config.countStepsAsOne)
+    ) {
+      this.pushFailure('Expected ' + this.expected + ' assertions, but ' +
+        actualCountForExpect + ' were run\nRemember that with QUnit.config.countStepsAsOne and in QUnit 3.0, steps no longer count as separate assertions. https://qunitjs.com/api/assert/expect/', this.stack);
+    } else if (this.expected !== null && this.expected !== actualCountForExpect) {
       this.pushFailure('Expected ' + this.expected + ' assertions, but ' +
-        this.assertions.length + ' were run', this.stack);
-    } else if (this.expected === null && !this.assertions.length) {
+        actualCountForExpect + ' were run', this.stack);
+    } else if (this.expected === null && !actualCountForExpect) {
       this.pushFailure('Expected at least one assertion, but none were run - call ' +
         'expect(0) to accept zero assertions.', this.stack);
     }

test/cli/fixtures/assert-expect-failure-step.js

@@ -0,0 +1,65 @@
+// Cherry-picked from 3.0.0, https://github.com/qunitjs/qunit/pull/1775
+QUnit.config.countStepsAsOne = true;
+
+QUnit.test('passing [once]', function (assert) {
+  assert.expect(1);
+
+  assert.step('a');
+  assert.step('b');
+  assert.verifySteps(['a', 'b']);
+});
+
+QUnit.test('passing [twice]', function (assert) {
+  assert.expect(2);
+
+  assert.step('a');
+  assert.step('b');
+  assert.verifySteps(['a', 'b']);
+
+  assert.step('c');
+  assert.step('d');
+  assert.step('e');
+  assert.verifySteps(['c', 'd', 'e']);
+});
+
+QUnit.test('wrong [a little off]', function (assert) {
+  assert.expect(2);
+
+  assert.step('a');
+  assert.step('b');
+  assert.verifySteps(['a', 'b']);
+});
+
+QUnit.test('wrong [way off]', function (assert) {
+  assert.expect(5);
+
+  assert.step('a');
+  assert.step('b');
+  assert.verifySteps(['a', 'b']);
+});
+
+// These were the correct counts in QUnit 2.x
+// https://github.com/qunitjs/qunit/issues/1226
+QUnit.test('previously passing [once]', function (assert) {
+  assert.expect(4);
+
+  assert.step('a');
+  assert.step('b');
+  assert.verifySteps(['a', 'b']);
+  assert.true(true);
+});
+
+QUnit.test('previously passing [twice]', function (assert) {
+  assert.expect(9);
+
+  assert.step('a');
+  assert.true(true);
+  assert.step('b');
+  assert.verifySteps(['a', 'b']);
+
+  assert.false(false);
+  assert.step('c');
+  assert.step('d');
+  assert.step('e');
+  assert.verifySteps(['c', 'd', 'e']);
+});

test/cli/fixtures/assert-expect-failure-step.tap.txt

@@ -0,0 +1,55 @@
+# command: ["qunit", "assert-expect-failure-step.js"]
+TAP version 13
+ok 1 passing [once]
+ok 2 passing [twice]
+not ok 3 wrong [a little off]
+  ---
+  message: Expected 2 assertions, but 1 were run
+  severity: failed
+  actual  : null
+  expected: undefined
+  stack: |
+        at /qunit/test/cli/fixtures/assert-expect-failure-step.js:25:7
+        at internal
+  ...
+not ok 4 wrong [way off]
+  ---
+  message: Expected 5 assertions, but 1 were run
+  severity: failed
+  actual  : null
+  expected: undefined
+  stack: |
+        at /qunit/test/cli/fixtures/assert-expect-failure-step.js:33:7
+        at internal
+  ...
+not ok 5 previously passing [once]
+  ---
+  message: |+
+    Expected 4 assertions, but 2 were run
+    Remember that with QUnit.config.countStepsAsOne and in QUnit 3.0, steps no longer count as separate assertions. https://qunitjs.com/api/assert/expect/
+  severity: failed
+  actual  : null
+  expected: undefined
+  stack: |
+        at /qunit/test/cli/fixtures/assert-expect-failure-step.js:43:7
+        at internal
+  ...
+not ok 6 previously passing [twice]
+  ---
+  message: |+
+    Expected 9 assertions, but 4 were run
+    Remember that with QUnit.config.countStepsAsOne and in QUnit 3.0, steps no longer count as separate assertions. https://qunitjs.com/api/assert/expect/
+  severity: failed
+  actual  : null
+  expected: undefined
+  stack: |
+        at /qunit/test/cli/fixtures/assert-expect-failure-step.js:52:7
+        at internal
+  ...
+1..6
+# pass 2
+# skip 0
+# todo 0
+# fail 4
+
+# exit code: 1

test/cli/fixtures/assert-expect-step-warning.js

@@ -0,0 +1,22 @@
+QUnit.config.requireExpects = true;
+
+QUnit.test('passing [once]', function (assert) {
+  assert.expect(3);
+
+  assert.step('a');
+  assert.step('b');
+  assert.verifySteps(['a', 'b']);
+});
+
+QUnit.test('passing [twice]', function (assert) {
+  assert.expect(7);
+
+  assert.step('a');
+  assert.step('b');
+  assert.verifySteps(['a', 'b']);
+
+  assert.step('c');
+  assert.step('d');
+  assert.step('e');
+  assert.verifySteps(['c', 'd', 'e']);
+});

test/cli/fixtures/assert-expect-step-warning.tap.txt

@@ -0,0 +1,13 @@
+# command: ["qunit", "assert-expect-step-warning.js"]
+
+TAP version 13
+ok 1 passing [once]
+ok 2 passing [twice]
+1..2
+# pass 2
+# skip 0
+# todo 0
+# fail 0
+
+# stderr
+Counting each assert.step() for assert.expect() is changing in QUnit 3.0. You can enable QUnit.config.countStepsAsOne to prepare for the upgrade. https://qunitjs.com/api/assert/expect/

test/main/assert-step.js

@@ -154,4 +154,49 @@ QUnit.module('assert.step', function () {
       assert.deepEqual(result, ['step one', 'step two']);
     });
   });
+
+  // Transition for https://github.com/qunitjs/qunit/issues/1226
+  QUnit.module('assert.verifySteps with QUnit.config.countStepsAsOne', function (hooks) {
+    var original;
+    hooks.before(function () {
+      original = QUnit.config.countStepsAsOne;
+      QUnit.config.countStepsAsOne = true;
+    });
+    hooks.after(function () {
+      QUnit.config.countStepsAsOne = original;
+    });
+
+    QUnit.test('empty', function (assert) {
+      assert.expect(1);
+
+      assert.verifySteps([]);
+    });
+
+    QUnit.test('passing [once]', function (assert) {
+      assert.expect(1);
+
+      assert.step('a');
+      assert.step('b');
+      assert.verifySteps(['a', 'b']);
+    });
+
+    QUnit.test('passing [twice]', function (assert) {
+      assert.expect(2);
+
+      assert.step('a');
+      assert.step('b');
+      assert.verifySteps(['a', 'b']);
+
+      assert.step('c');
+      assert.step('d');
+      assert.step('e');
+      assert.verifySteps(['c', 'd', 'e']);
+    });
+
+    QUnit.test('example', function (assert) {
+      // buffer between last test and hooks.after(),
+      // to restore QUnit.config.countStepsAsOne.
+      assert.true(true);
+    });
+  });
 });