core(jsonld): add structured data validation

.travis.yml

@@ -20,7 +20,8 @@ install:
   # with the latest puppeteer api so we probably need to run on chromimum
   # @see https://github.com/GoogleChrome/lighthouse/pull/4640/files#r171425004
   - export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1
-  - yarn
+  # jsonld package has native bindings that are optional
+  - yarn --ignore-optional
 before_script:
   - export DISPLAY=:99.0
   # see comment above about puppeteer

package.json

@@ -32,8 +32,9 @@
     "unit-cli": "jest  --runInBand \"lighthouse-cli/\"",
     "unit-cli:ci": "jest --runInBand --coverage --ci \"lighthouse-cli/\"",
     "unit-viewer": "mocha --reporter dot \"lighthouse-viewer/test/**/*-test.js\"",
-    "unit": "yarn unit-core && yarn unit-cli && yarn unit-viewer",
-    "unit:ci": "yarn unit-core:ci && yarn unit-cli:ci && yarn unit-viewer",
+    "unit-sd-validation": "mocha --reporter dot \"sd-validation/test/**/*-test.js\"",
+    "unit": "yarn unit-core && yarn unit-cli && yarn unit-viewer && yarn unit-sd-validation",
+    "unit:ci": "yarn unit-core:ci && yarn unit-cli:ci && yarn unit-viewer && yarn unit-sd-validation",
     "core-unit": "yarn unit-core",
     "cli-unit": "yarn unit-cli",
     "viewer-unit": "yarn unit-viewer",
@@ -148,6 +149,8 @@
     "intl-messageformat-parser": "^1.4.0",
     "jpeg-js": "0.1.2",
     "js-library-detector": "^5.1.0",
+    "jsonld": "^1.2.0",
+    "jsonlint-mod": "^1.7.2",
     "lighthouse-logger": "^1.2.0",
     "lodash.isequal": "^4.5.0",
     "lookup-closest-locale": "6.0.4",

sd-validation/expand.js

@@ -0,0 +1,79 @@
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const {URL} = require('url');
+const jsonld = require('jsonld');
+// @ts-ignore
+const schemaOrgContext = require('./assets/jsonldcontext');
+const SCHEMA_ORG_HOST = 'schema.org';
+
+/**
+ * Custom loader that prevents network calls and allows us to return local version of the
+ * schema.org document
+ * @param {string} schemaUrl
+ * @param {function(null, Object):void} callback
+ */
+function loadDocument(schemaUrl, callback) {
+  let urlObj = null;
+
+  try {
+    urlObj = new URL(schemaUrl, 'http://example.com');
+  } catch (e) {
+    throw new Error('Error parsing URL: ' + schemaUrl);
+  }
+
+  if (urlObj && urlObj.host === SCHEMA_ORG_HOST && urlObj.pathname === '/') {
+    callback(null, {
+      document: schemaOrgContext,
+    });
+  } else {
+    // We only process schema.org, for other schemas we return an empty object
+    callback(null, {
+      document: {},
+    });
+  }
+}
+
+/**
+ * Takes JSON-LD object and normalizes it by following the expansion algorithm
+ * (https://json-ld.org/spec/latest/json-ld-api/#expansion).
+ *
+ * @param {Object} inputObject
+ * @returns {Object}
+ */
+module.exports = function expand(inputObject) {
+  /** @type {function(string):void} */
+  let resolve;
+  /** @type {function(string):void} */
+  let reject;
+  const promise = new Promise((res, rej) => {
+    resolve = res; reject = rej;
+  });
+
+  const documentLoader = (
+      /** @type {string} **/ schemaUrl,
+      /** @type {function(null, Object):void} **/ callback
+  ) => {
+    try {
+      return loadDocument(schemaUrl, callback);
+    } catch (e) {
+      reject(e.message);
+    }
+  };
+
+  jsonld.expand(inputObject, {
+    documentLoader,
+  }, (/** @type {string} */e, /** @type {Object} **/expanded) => {
+    if (e) {
+      reject('Expansion error: ' + e.toString());
+    } else {
+      resolve(expanded);
+    }
+  });
+
+  return promise;
+};

sd-validation/helpers/walkObject.js

@@ -0,0 +1,31 @@
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/**
+ * Recursively (DFS) traverses an object and calls provided function for each field.
+ *
+ * @param {Object} obj
+ * @param {function(String, any, Array<String>, Object): void} callback
+ * @param {Array<String>} path
+ */
+module.exports = function walkObject(obj, callback, path = []) {
+  if (obj === null) {
+    return;
+  }
+
+  Object.keys(obj).forEach(fieldName => {
+    const fieldValue = obj[fieldName];
+    const newPath = Array.from(path);
+    newPath.push(fieldName);
+
+    callback(fieldName, fieldValue, newPath, obj);
+
+    if (typeof fieldValue === 'object') {
+      walkObject(fieldValue, callback, newPath);
+    }
+  });
+};

sd-validation/index.js

@@ -0,0 +1,83 @@
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const parseJSON = require('./json');
+const validateJsonLD = require('./jsonld');
+const promiseExpand = require('./expand');
+const validateSchemaOrg = require('./schema');
+
+/**
+ * Validates JSON-LD input. Returns array of error objects.
+ *
+ * @param {string} textInput
+ * @returns {Promise<Array<{path: ?string, validator: string, message: string}>>}
+ */
+module.exports = async function validate(textInput) {
+  /** @type {Array<{path: ?string, validator: string, message: string}>} */
+  const errors = [];
+
+  // STEP 1: VALIDATE JSON
+  const parseOutput = parseJSON(textInput);
+
+  if (parseOutput.error) {
+    errors.push({
+      validator: 'json',
+      path: parseOutput.error.line,
+      message: parseOutput.error.message,
+    });
+
+    return errors;
+  }
+
+  const inputObject = parseOutput.result;
+
+  // STEP 2: VALIDATE JSONLD
+  const jsonLdErrors = validateJsonLD(inputObject);
+
+  if (jsonLdErrors && jsonLdErrors.length) {
+    jsonLdErrors.forEach(error => {
+      errors.push({
+        validator: 'json-ld',
+        path: error.path,
+        message: error.message.toString(),
+      });
+    });
+
+    return errors;
+  }
+
+  // STEP 3: EXPAND
+  let expandedObj = null;
+  try {
+    expandedObj = await promiseExpand(inputObject);
+  } catch (error) {
+    errors.push({
+      validator: 'json-ld-expand',
+      path: null,
+      message: error && error.toString(),
+    });
+
+    return errors;
+  }
+
+  // STEP 4: VALIDATE SCHEMA
+  const schemaOrgErrors = validateSchemaOrg(expandedObj);
+
+  if (schemaOrgErrors && schemaOrgErrors.length) {
+    schemaOrgErrors.forEach(error => {
+      errors.push({
+        validator: 'schema-org',
+        path: error.path,
+        message: error.message,
+      });
+    });
+
+    return errors;
+  }
+
+  return errors;
+};

sd-validation/json.js

@@ -0,0 +1,59 @@
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const jsonlint = require('jsonlint-mod');
+
+/**
+ * @param {string} input
+ * @returns {{error: {message: string, line: string|null}|null, result: Object|null}}
+ */
+module.exports = function parseJSON(input) {
+  let result;
+  const logError = console.error; // eslint-disable-line no-console
+
+  try {
+    // jsonlint-mod always calls console.error when there's an error
+    // We don't want this behavior, so we stash console.error while it's executing
+    console.error = () => undefined; // eslint-disable-line no-console
+    jsonlint.parse(input);
+    result = JSON.parse(input);
+  } catch (error) {
+    let line = error.at;
+    let message = error.message;
+
+    // extract line number from message
+    if (!line) {
+      const regexLineResult = error.message.match(/Parse error on line (\d+)/);
+
+      if (regexLineResult) {
+        line = regexLineResult[1];
+      }
+    }
+
+    // adjust jsonlint error output to our needs
+    const regexMessageResult = error.message.match(/-+\^\n(.+)$/);
+
+    if (regexMessageResult) {
+      message = regexMessageResult[1];
+    }
+
+    return {
+      error: {
+        message,
+        line,
+      },
+      result,
+    };
+  } finally {
+    console.error = logError; // eslint-disable-line no-console
+  }
+
+  return {
+    error: null,
+    result,
+  };
+};

sd-validation/jsonld.js

@@ -0,0 +1,71 @@
+/**
+ * @license Copyright 2018 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+const walkObject = require('./helpers/walkObject');
+
+// This list comes from the JSON-LD 1.1 spec: https://json-ld.org/spec/latest/json-ld/#syntax-tokens-and-keywords
+const KEYWORDS = [
+  '@base',
+  '@container',
+  '@context',
+  '@graph',
+  '@id',
+  '@index',
+  '@language',
+  '@list',
+  '@nest',
+  '@none',
+  '@prefix',
+  '@reverse',
+  '@set',
+  '@type',
+  '@value',
+  '@version',
+  '@vocab',
+];
+
+/**
+ * @param {string} fieldName
+ * @returns boolean
+ */
+function validKeyword(fieldName) {
+  return KEYWORDS.includes(fieldName);
+}
+
+/**
+ * @param {string} keyName
+ * @returns {string | null} error
+ */
+function validateKey(keyName) {
+  if (keyName[0] === '@' && !validKeyword(keyName)) {
+    return 'Unknown keyword';
+  }
+
+  return null;
+}
+
+/**
+ * @param {Object} json
+ * @returns {Array<{path: string, message: string}>}
+ */
+module.exports = function validateJsonLD(json) {
+  /** @type {Array<{path: string, message: string}>} */
+  const errors = [];
+
+  walkObject(json, (name, value, path, object) => {
+    const error = validateKey.call(null, name, value, path, object);
+
+    if (error) {
+      errors.push({
+        path: path.join('/'),
+        message: error,
+      });
+    }
+  });
+
+  return errors;
+};

sd-validation/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "sd-validation",
+  "private": true,
+  "engines": {
+    "node": ">=8"
+  },
+  "scripts": {
+    "test": "mocha test/**/*-test.js"
+  },
+  "dependencies": {
+    "jsonld": "^1.0.2",
+    "jsonlint-mod": "^1.7.2"
+  },
+  "devDependencies": {
+    "request-promise-native": "^1.0.5"
+  }
+}