Merge pull request #10293 from CesiumGS/jasmine-browser

Add updated browser spec runner

.eslintignore

@@ -7,6 +7,7 @@ Source/Shaders/**
 Source/ThirdParty/**
 Source/Workers/**
 !Source/Workers/transferTypedArrayTest.js
+Specs/jasmine/**
 ThirdParty/**
 Tools/**
 Apps/Sandcastle/jsHintOptions.js

.gitignore

@@ -17,6 +17,7 @@ Thumbs.db
 /Source/Cesium.d.ts
 
 /Specs/SpecList.js
+/Specs/jasmine/**
 /Source/Shaders/**/*.js
 /Source/ThirdParty/Shaders/**/*.js
 /Source/Workers/**

.prettierignore

@@ -26,6 +26,7 @@ Source/Scene/GltfPipeline/**
 Source/Shaders/**/*.js
 Source/ThirdParty/**
 Source/Workers/**/*
+Specs/jasmine/**
 Apps/Sandcastle/ThirdParty
 
 !Source/Workers/cesiumWorkerBootstrapper.js

.travis.yml

@@ -19,11 +19,11 @@ script:
   - npm pack &> /dev/null
 
   - npm --silent run buildApps
+  - npm --silent run build-specs
 
   - npm --silent run deploy-s3 -- -b cesium-dev -d cesium/$TRAVIS_BRANCH --confirm -c 'no-cache'
   - npm --silent run deploy-status -- --status success --message Deployed
 
-  - npm --silent run build-specs
   - npm --silent run test -- --browsers ChromeCI --failTaskOnError --webgl-stub --release --suppressPassed
 
   # Various Node.js smoke-screen tests

Documentation/Contributors/TestingGuide/README.md

@@ -4,7 +4,7 @@ Our development culture is committed to testing. CesiumJS is used in diverse use
 
 As of CesiumJS 1.35, CesiumJS has over 8,800 tests with 93% code coverage. CesiumJS has as much test code (126K lines) as engine code (126K). We are unaware of any other project of this size and lifetime and with this many contributors that has similar stats.
 
-All new code should have 100% code coverage and should pass all tests. Always run the tests before opening a pull request.
+All new code should have 100% code coverage and should pass all tests. Always run the tests before opening a pull request. It's also important that the tests run quickly so we run them often.
 
 - [Testing Guide](#testing-guide)
   - [Running the Tests](#running-the-tests)
@@ -17,6 +17,13 @@ All new code should have 100% code coverage and should pass all tests. Always ru
       - [Run All Tests Against the Minified Release Version of CesiumJS](#run-all-tests-against-the-minified-release-version-of-cesiumjs)
       - [Run a Single Test or Suite](#run-a-single-test-or-suite)
       - [Using Browser Debugging Tools](#using-browser-debugging-tools)
+    - [Running the Tests in the Browser](#running-the-tests-in-the-browser)
+      - [Run All Tests](#run-all-tests)
+      - [Run with WebGL validation](#run-with-webgl-validation)
+      - [Run with WebGL stub](#run-with-webgl-stub)
+      - [Select a Test to Run](#select-a-test-to-run)
+      - [Run Only WebGL Category Tests](#run-only-webgl-category-tests)
+      - [Run Only Non-WebGL Category Tests](#run-only-non-webgl-category-tests)
     - [Run Coverage](#run-coverage)
   - [`testfailure` Label for Issues](#testfailure-label-for-issues)
   - [Writing Tests](#writing-tests)
@@ -112,8 +119,6 @@ Alternatively, test suites can be run from the command line with the `includeNam
 
 `npm run test -- --includeName Cartesian2`
 
-Similarly, test suites can be excluded with the `--excludeName` flag.
-
 #### Using Browser Debugging Tools
 
 If it is helpful to step through a unit test in a browser debugger, run the tests with the `debug` flag:
@@ -124,6 +129,72 @@ The `--debug` flag will prevent the Karma browser from closing after running the
 
 ![](8.jpg)
 
+### Running the Tests in the Browser
+
+When running CesiumJS locally, [start the local server](https://github.com/CesiumGS/cesium/tree/main/Documentation/Contributors/BuildGuide#build-the-code) and browse to [http://localhost:8080/](http://localhost:8080/). There are several test options:
+
+#### Run All Tests
+
+When all the tests pass, the page looks like this:
+
+![Browser tests when all pass](browser-all.png)
+
+When one or more tests fail, the page looks like this:
+
+![Browser tests when a spec fails](browser-failed.png)
+
+In this case, the number of failing tests is listed at the top, and details on each failure are listed below, including the expected and actual value of the failed expectation and the call stack. The top several functions of the call stack are inside Jasmine and can be ignored. Above, the file and line of interest for the first failing test starts with an `@`:
+
+```
+  @at UserContext.<anonymous> (http://localhost:8080/Specs/Core/Cartesian3Spec.js:12:25)
+```
+
+Click on the failed test to rerun just that test. This is useful for saving time when fixing an issue as it avoids rerunning all the tests. Always rerun _all_ the tests before opening a pull request.
+
+#### Run with WebGL validation
+
+The link to **Run with WebGL validation** passes a query parameter to the tests to enable extra low-level WebGL validation such as calling `gl.getError()` after each WebGL call.
+
+#### Run with WebGL stub
+
+The **Run with WebGL stub** link passes a query parameter to the tests to use CesiumJS's WebGL stub. This makes all WebGL calls a noop and ignores test expectations that rely on reading back from WebGL. This allows running the tests on CI where a reasonable WebGL implementation is not available and still getting full code coverage albeit not all verification.
+
+#### Select a Test to Run
+
+This option loads the test page without running any tests.
+
+![Browser tests without running any specs](browser-none.png)
+
+We can then use the browser's built-in search to find a test or suite and run only that. For example, below just the tests for `Cartesian3` were run.
+
+![Browser tests with only Cartesian3 specs run](browser-cartesian3.png)
+
+This uses a query parameter to select the test/suite to run so refreshing the page will run just that test/suite again.
+
+Often when developing, it is useful to run only one suite to save time, instead of all the tests, and then run all the tests before opening a pull request.
+
+#### Run Only WebGL Category Tests
+
+Suites can have a category associated with them. This option runs all tests in the `WebGL` category, which includes all tests that use WebGL (basically anything that requires creating a `Viewer`, `CesiumWidget`, `Scene`, or `Context`).
+
+#### Run Only Non-WebGL Category Tests
+
+Likewise, this option runs all tests not in the WebGL category.
+
+Perhaps surprisingly, this is the bulk of CesiumJS tests, which include math and geometry tests, imagery provider tests, data source tests, etc.
+
+These tests run quickly (for example, 15 seconds compared to 60) and are very reliable across systems since they do not rely on the underlying WebGL implementation, which can vary based on the browser, OS, driver, and GPU.
+
+#### Run All Tests against Combined File (Run All Tests against Combined File with Debug Code Removed)
+
+Most test options load CesiumJS using the individual source files in the `Source` directory, which is great for debugging.
+
+However, many users build apps using the built Cesium.js in `Build/Cesium` (which is created, for example, by running `npm run combine`). This option runs the tests using this instead of individual CesiumJS source files.
+
+The **Run All Tests against Combined File with Debug Code Removed** is the same except it is for use with the release version of the built Cesium.js (which is created, for example, by running `npm run combineRelease`). The release version has `DeveloperError` exceptions optimized out so this test option makes `toThrowDeveloperError` always pass.
+
+See the [Build Guide](https://github.com/CesiumGS/cesium/blob/main/Documentation/Contributors/BuildGuide/README.md#build-scripts) for all the CesiumJS build options.
+
 ### Run Coverage
 
 We use [istanbul](https://istanbul.js.org/) via [karma-coverage](https://github.com/karma-runner/karma-coverage) to generate code coverage reports. It is especially important to have outstanding code coverage since JavaScript doesn't have a compiler and linker to catch early errors.

Specs/Scene/CameraFlightPathSpec.js

@@ -1,8 +1,8 @@
 import { Cartesian3 } from "../../Source/Cesium.js";
 import { Cartographic } from "../../Source/Cesium.js";
 import { Ellipsoid } from "../../Source/Cesium.js";
-import { GeographicProjection } from "../../../Source/Cesium.js";
-import { Globe } from "../../../Source/Cesium.js";
+import { GeographicProjection } from "../../Source/Cesium.js";
+import { Globe } from "../../Source/Cesium.js";
 import { Math as CesiumMath } from "../../Source/Cesium.js";
 import { OrthographicOffCenterFrustum } from "../../Source/Cesium.js";
 import { CameraFlightPath } from "../../Source/Cesium.js";

Specs/Scene/IonImageryProviderSpec.js

@@ -62,7 +62,7 @@ describe("Scene/IonImageryProvider", function () {
     }).toThrowDeveloperError(ImageryProvider);
   });
 
-  it("readyPromise rejects with non-imagery asset", function (done) {
+  it("readyPromise rejects with non-imagery asset", function () {
     const provider = createTestProvider({
       type: "3DTILES",
       url: "http://test.invalid/layer",
@@ -80,7 +80,7 @@ describe("Scene/IonImageryProvider", function () {
       });
   });
 
-  it("readyPromise rejects with unknown external asset type", function (done) {
+  it("readyPromise rejects with unknown external asset type", function () {
     const provider = createTestProvider({
       type: "IMAGERY",
       externalType: "TUBELCANE",

Specs/Scene/PntsParserSpec.js

@@ -1,5 +1,4 @@
-import { PntsParser } from "../../Source/Cesium.js";
-import { RuntimeError } from "../Source/Cesium.js";
+import { PntsParser, RuntimeError } from "../../Source/Cesium.js";
 import Cesium3DTilesTester from "../Cesium3DTilesTester.js";
 
 describe("Scene/PntsParser", function () {

Specs/SpecRunner.html

@@ -0,0 +1,32 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title>Jasmine Spec Runner v4.0.1</title>
+
+    <link rel="stylesheet" href="./jasmine/jasmine.css" />
+
+    <script src="./jasmine/jasmine.js"></script>
+    <script src="./jasmine/jasmine-html.js"></script>
+    <script src="./jasmine/boot0.js"></script>
+    <script type="module" src="./spec-main.js"></script>
+    <script src="./jasmine/boot1.js"></script>
+  </head>
+  <body>
+    <script>
+      if (window.location.search.indexOf("built") !== -1) {
+        if (window.location.search.indexOf("release") !== -1) {
+          document.write('<script src="../Build/Cesium/Cesium.js"><\/script>');
+        } else {
+          document.write(
+            '<script src="../Build/CesiumUnminified/Cesium.js"><\/script>'
+          );
+        }
+        document.write('<script src="../Build/Specs/Specs.js"><\/script>');
+      } else {
+        window.CESIUM_BASE_URL = "../Source/";
+        document.write('<script type="module" src="./SpecList.js"><\/script>');
+      }
+    </script>
+  </body>
+</html>

Specs/addDefaultMatchers.js

@@ -43,7 +43,7 @@ function makeThrowFunction(debug, Type, name) {
           }
 
           if (exception) {
-            result = exception instanceof Type;
+            result = exception instanceof Type || exception.name === name;
           }
 
           let message;

Specs/customizeJasmine.js

@@ -5,8 +5,6 @@ function customizeJasmine(
   env,
   includedCategory,
   excludedCategory,
-  includedName,
-  excludedName,
   webglValidation,
   webglStub,
   release
@@ -18,104 +16,25 @@ function customizeJasmine(
 
   const originalDescribe = window.describe;
 
-  window.describe = function (name, suite, categories) {
+  window.describe = function (name, suite, category) {
     // exclude this spec if we're filtering by category and it's not the selected category
     // otherwise if we have an excluded category, exclude this test if the category of this spec matches
-    if (includedCategory && categories !== includedCategory) {
-      return;
-    } else if (excludedCategory && categories === excludedCategory) {
-      return;
+    if (
+      includedCategory &&
+      includedCategory !== "" &&
+      includedCategory !== "none" &&
+      category !== includedCategory
+    ) {
+      window.xdescribe(name, suite);
+    } else if (
+      excludedCategory &&
+      excludedCategory !== "" &&
+      category === excludedCategory
+    ) {
+      window.xdescribe(name, suite);
+    } else {
+      originalDescribe(name, suite);
     }
-
-    if (includedName && !name.includes(includedName)) {
-      return;
-    } else if (excludedName && name.includes(excludedName)) {
-      return;
-    }
-
-    originalDescribe(name, suite, categories);
-  };
-
-  // Override beforeEach(), afterEach(), beforeAll(), afterAll(), and it() to automatically
-  // call done() when a returned promise resolves.
-  const originalIt = window.it;
-
-  window.it = function (description, f, timeout, categories) {
-    originalIt(
-      description,
-      function (done) {
-        const result = f(done);
-        Promise.resolve(result)
-          .then(function () {
-            done();
-          })
-          .catch(function (e) {
-            done.fail(`promise rejected: ${e.toString()}`);
-          });
-      },
-      timeout,
-      categories
-    );
-  };
-
-  const originalBeforeEach = window.beforeEach;
-
-  window.beforeEach = function (f) {
-    originalBeforeEach(function (done) {
-      const result = f(done);
-      Promise.resolve(result)
-        .then(function () {
-          done();
-        })
-        .catch(function (e) {
-          done.fail(`promise rejected: ${e.toString()}`);
-        });
-    });
-  };
-
-  const originalAfterEach = window.afterEach;
-
-  window.afterEach = function (f) {
-    originalAfterEach(function (done) {
-      const result = f(done);
-      Promise.resolve(result)
-        .then(function () {
-          done();
-        })
-        .catch(function (e) {
-          done.fail(`promise rejected: ${e.toString()}`);
-        });
-    });
-  };
-
-  const originalBeforeAll = window.beforeAll;
-
-  window.beforeAll = function (f) {
-    originalBeforeAll(function (done) {
-      const result = f(done);
-      Promise.resolve(result)
-        .then(function () {
-          done();
-        })
-        .catch(function (e) {
-          done.fail(`promise rejected: ${e.toString()}`);
-        });
-    });
-  };
-
-  const originalAfterAll = window.afterAll;
-
-  window.afterAll = function (f) {
-    originalAfterAll(function (done) {
-      const result = f(done);
-      Promise.resolve(result)
-        .then(function () {
-          done();
-        })
-        .catch(function (e) {
-          done.fail(`promise rejected: ${e.toString()}`);
-        });
-    });
   };
 
   if (webglValidation) {
@@ -126,8 +45,6 @@ function customizeJasmine(
     window.webglStub = true;
   }
 
-  //env.catchExceptions(true);
-
   env.beforeEach(function () {
     addDefaultMatchers(!release).call(env);
     env.addCustomEqualityTester(equalsMethodEqualityTester);