Prebid Core: ORTB 2.5 translation utilities (prebid#9263)

* ORTB 2.5 spec definition

* ORTB 2.5 translation

* Only test translation of native reqs if FEATURES.NATIVE is set

libraries/ortb2.5StrictTranslator/dsl.js

@@ -0,0 +1,54 @@
+export const ERR_TYPE = 0; // field has wrong type (only objects, enums, and arrays of objects or enums are checked)
+export const ERR_UNKNOWN_FIELD = 1; // field is not defined in ORTB 2.5 spec
+export const ERR_ENUM = 2; // field is an enum and its value is not one of those listed in the ORTB 2.5 spec
+
+// eslint-disable-next-line symbol-description
+export const extend = Symbol();
+
+export function Obj(primitiveFields, spec = {}) {
+  const scan = (path, parent, field, value, onError) => {
+    if (value == null || typeof value !== 'object') {
+      onError(ERR_TYPE, path, parent, field, value);
+      return;
+    }
+    Object.entries(value).forEach(([k, v]) => {
+      if (v == null) return;
+      const kpath = path == null ? k : `${path}.${k}`;
+      if (spec.hasOwnProperty(k)) {
+        spec[k](kpath, value, k, v, onError);
+        return;
+      }
+      if (k !== 'ext' && !primitiveFields.includes(k)) {
+        onError(ERR_UNKNOWN_FIELD, kpath, value, k, v);
+      }
+    });
+  };
+  scan[extend] = (extraPrimitives, specOverride = {}) =>
+    Obj(primitiveFields.concat(extraPrimitives), Object.assign({}, spec, specOverride));
+  return scan;
+}
+
+export const ID = Obj(['id']);
+export const Named = ID[extend](['name']);
+
+export function Arr(def) {
+  return (path, parent, field, value, onError) => {
+    if (!Array.isArray(value)) {
+      onError(ERR_TYPE, path, parent, field, value);
+    } else {
+      value.forEach((item, i) => def(`${path}.${i}`, value, i, item, onError));
+    }
+  };
+}
+
+export function IntEnum(min, max) {
+  return (path, parent, field, value, onError) => {
+    const errno = (() => {
+      if (typeof value !== 'number') return ERR_TYPE;
+      if (isNaN(value) || value > max || value < min) return ERR_ENUM;
+    })();
+    if (errno != null) {
+      onError(errno, path, parent, field, value);
+    }
+  };
+}

libraries/ortb2.5StrictTranslator/spec.js

@@ -0,0 +1,81 @@
+import {Arr, extend, ID, IntEnum, Named, Obj} from './dsl.js';
+
+const CatDomain = Named[extend](['cat', 'domain']);
+const Segment = Named[extend](['value']);
+const Data = Named[extend]([], {
+  segment: Arr(Segment)
+});
+const Content = ID[extend](['episode', 'title', 'series', 'season', 'artist', 'genre', 'album', 'isrc', 'url', 'cat', 'contentrating', 'userrating', 'keywords', 'livestream', 'sourcerelationship', 'len', 'language', 'embeddable'], {
+  producer: CatDomain,
+  data: Arr(Data),
+  prodq: IntEnum(0, 3),
+  videoquality: IntEnum(0, 3),
+  context: IntEnum(1, 7),
+  qagmediarating: IntEnum(1, 3),
+});
+
+const Client = CatDomain[extend](['sectioncat', 'pagecat', 'privacypolicy', 'keywords'], {
+  publisher: CatDomain, content: Content,
+});
+const Site = Client[extend](['page', 'ref', 'search', 'mobile']);
+const App = Client[extend](['bundle', 'storeurl', 'ver', 'paid']);
+
+const Geo = Obj(['lat', 'lon', 'accuracy', 'lastfix', 'country', 'region', 'regionfips104', 'metro', 'city', 'zip', 'utcoffset'], {
+  type: IntEnum(1, 3),
+  ipservice: IntEnum(1, 4)
+});
+const Device = Obj(['ua', 'dnt', 'lmt', 'ip', 'ipv6', 'make', 'model', 'os', 'osv', 'hwv', 'h', 'w', 'ppi', 'pxratio', 'js', 'geofetch', 'flashver', 'language', 'carrier', 'mccmnc', 'ifa', 'didsha1', 'didmd5', 'dpidsha1', 'dpidmd5', 'macsha1', 'macmd5'], {
+  geo: Geo, devicetype: IntEnum(1, 7), connectiontype: IntEnum(0, 6)
+});
+const User = ID[extend](['buyeruid', 'yob', 'gender', 'keywords', 'customdata'], {
+  geo: Geo, data: Arr(Data),
+});
+
+const Floorable = ID[extend](['bidfloor', 'bidfloorcur']);
+const Deal = Floorable[extend](['at', 'wseat', 'wadomain']);
+const Pmp = Obj(['private_auction'], {
+  deals: Arr(Deal),
+});
+const Format = Obj(['w', 'h', 'wratio', 'hratio', 'wmin']);
+const MediaType = Obj(['mimes'], {
+  api: Arr(IntEnum(1, 6)), battr: Arr(IntEnum(1, 17))
+});
+const Banner = MediaType[extend](['id', 'w', 'h', 'wmax', 'hmax', 'hmin', 'wmin', 'topframe', 'vcm'], {
+  format: Arr(Format), btype: Arr(IntEnum(1, 4)), pos: IntEnum(0, 7), expdir: Arr(IntEnum(1, 5))
+});
+const Native = MediaType[extend](['request', 'ver']);
+const RichMediaType = MediaType[extend](['minduration', 'maxduration', 'startdelay', 'sequence', 'maxextended', 'minbitrate', 'maxbitrate'], {
+  protocols: Arr(IntEnum(1, 10)),
+  delivery: Arr(IntEnum(1, 3)),
+  companionad: Arr(Banner),
+  companiontype: Arr(IntEnum(1, 3)),
+});
+/*
+const Audio = RichMediaType[extend](['maxseq', 'stitched'], {
+  feed: IntEnum(1, 3), nvol: IntEnum(0, 4),
+});
+ */
+const Video = RichMediaType[extend](['w', 'h', 'skip', 'skipmin', 'skipafter', 'boxingallowed'], {
+  pos: IntEnum(0, 7),
+  protocol: IntEnum(1, 10),
+  placement: IntEnum(1, 5),
+  linearity: IntEnum(1, 2),
+  playbackmethod: Arr(IntEnum(1, 6)),
+  playbackend: IntEnum(1, 3),
+});
+const Metric = Obj(['type', 'value', 'vendor']);
+const Imp = (() => {
+  const spec = {
+    metric: Arr(Metric), banner: Banner, video: Video, pmp: Pmp,
+  };
+  if (FEATURES.NATIVE) {
+    spec.native = Native;
+  }
+  return Floorable[extend](['displaymanager', 'displaymanagerver', 'instl', 'tagid', 'clickbrowser', 'secure', 'iframebuster', 'exp'], spec);
+})();
+
+const Regs = Obj(['coppa']);
+const Source = Obj(['fd', 'tid', 'pchain']);
+export const BidRequest = ID[extend](['test', 'at', 'tmax', 'wseat', 'bseat', 'allimps', 'cur', 'wlang', 'bcat', 'badv', 'bapp'], {
+  imp: Arr(Imp), site: Site, app: App, device: Device, user: User, source: Source, regs: Regs
+});

libraries/ortb2.5StrictTranslator/translator.js

@@ -0,0 +1,37 @@
+import {BidRequest} from './spec.js';
+import {logWarn} from '../../src/utils.js';
+import {toOrtb25} from '../ortb2.5Translator/translator.js';
+
+function deleteField(errno, path, obj, field, value) {
+  logWarn(`${path} is not valid ORTB 2.5, field will be removed from request:`, value);
+  Array.isArray(obj) ? obj.splice(field, 1) : delete obj[field];
+}
+
+/**
+ * Translates an ortb request to 2.5, and removes from the result any field that is:
+ *  - not defined in the 2.5 spec, or
+ *  - defined as an enum, but has a value that is not listed in the 2.5 spec.
+ *
+ * `ortb2` is modified in place and returned.
+ *
+ * Note that using this utility will cause your adapter to pull in an additional ~3KB after minification.
+ * If possible, consider making your endpoint tolerant to unrecognized or invalid fields instead.
+ *
+ *
+ * @param ortb2 ORTB request
+ * @param translator translation function. The default moves 2.x fields that have a known standard location in 2.5.
+ *                   See the `ortb2.5Translator` library.
+ * @param onError a function invoked once for each field that is not valid according to the 2.5 spec; it takes
+ *        (errno, path, obj, field, value), where:
+ *         - errno is an error code (defined in dsl.js)
+ *         - path is the JSON path of the offending field, for example `regs.gdpr`
+ *         - obj is the object containing the offending field, for example `ortb2.regs`
+ *         - field is the field name, for example `'gdpr'`
+ *         - value is `obj[field]`.
+ *        The default logs a warning and deletes the offending field.
+ */
+export function toOrtb25Strict(ortb2, translator = toOrtb25, onError = deleteField) {
+  ortb2 = translator(ortb2);
+  BidRequest(null, null, null, ortb2, onError);
+  return ortb2;
+}

libraries/ortb2.5Translator/translator.js

@@ -0,0 +1,82 @@
+import {deepAccess, deepSetValue, logError} from '../../src/utils.js';
+
+export const EXT_PROMOTIONS = [
+  'source.schain',
+  'regs.gdpr',
+  'regs.us_privacy',
+  'regs.gpp',
+  'user.consent',
+  'user.eids'
+];
+
+export function splitPath(path) {
+  const parts = path.split('.');
+  const prefix = parts.slice(0, parts.length - 1).join('.');
+  const field = parts[parts.length - 1];
+  return [prefix, field];
+}
+
+/**
+ * @param sourcePath a JSON path such as `regs.us_privacy`
+ * @param dest {function(String, String): String} a function taking (prefix, field) and returning a destination path;
+ *             for example, ('regs', 'us_privacy') => 'regs.ext.us_privacy'
+ * @return {(function({}): (function(): void|undefined))|*} a function that takes an object and, if it contains
+ *        sourcePath, copies its contents to destinationPath, returning a function that deletes the original sourcePath.
+ */
+export function moveRule(sourcePath, dest = (prefix, field) => `${prefix}.ext.${field}`) {
+  const [prefix, field] = splitPath(sourcePath);
+  dest = dest(prefix, field);
+  return (ortb2) => {
+    const obj = deepAccess(ortb2, prefix);
+    if (obj?.[field] != null) {
+      deepSetValue(ortb2, dest, obj[field]);
+      return () => delete obj[field];
+    }
+  };
+}
+
+function kwarrayRule(section) {
+  // move 2.6 `kwarray` into 2.5 comma-separated `keywords`.
+  return (ortb2) => {
+    const kwarray = ortb2[section]?.kwarray;
+    if (kwarray != null) {
+      let kw = (ortb2[section].keywords || '').split(',');
+      if (Array.isArray(kwarray)) kw.push(...kwarray);
+      ortb2[section].keywords = kw.join(',');
+      return () => delete ortb2[section].kwarray;
+    }
+  };
+}
+
+export const DEFAULT_RULES = Object.freeze([
+  ...EXT_PROMOTIONS.map((f) => moveRule(f)),
+  ...['app', 'content', 'site', 'user'].map(kwarrayRule)
+]);
+
+/**
+ * Factory for ORTB 2.5 translation functions.
+ *
+ * @param deleteFields if true, the translation function will remove fields that have been translated (transferred somewhere else within the request)
+ * @param rules translation rules; an array of functions of the type returned by `moveRule`
+ * @return {function({}): {}} a translation function that takes an ORTB object, modifies it in place, and returns it.
+ */
+export function ortb25Translator(deleteFields = true, rules = DEFAULT_RULES) {
+  return function (ortb2) {
+    rules.forEach(f => {
+      try {
+        const deleter = f(ortb2);
+        if (typeof deleter === 'function' && deleteFields) deleter();
+      } catch (e) {
+        logError('Error translating request to ORTB 2.5', e);
+      }
+    })
+    return ortb2;
+  }
+}
+
+/**
+ * Translate an ortb request to version 2.5 by moving 2.6 (and later) fields that have a standardized 2.5 extension.
+ *
+ * The request is modified in place and returned.
+ */
+export const toOrtb25 = ortb25Translator();

test/spec/ortb2.5StrictTranslator/dsl_spec.js

@@ -0,0 +1,137 @@
+import {Arr, ERR_ENUM, ERR_TYPE, ERR_UNKNOWN_FIELD, IntEnum, Obj} from '../../../libraries/ortb2.5StrictTranslator/dsl.js';
+import {deepClone} from '../../../src/utils.js';
+
+describe('DSL', () => {
+  const spec = (() => {
+    const inner = Obj(['p21', 'p22'], {
+      enum: IntEnum(10, 20),
+      enumArray: Arr(IntEnum(10, 20))
+    });
+    return Obj(['p11', 'p12'], {
+      inner,
+      innerArray: Arr(inner)
+    });
+  })();
+
+  let onError;
+
+  function scan(obj) {
+    spec(null, null, null, obj, onError);
+  }
+
+  beforeEach(() => {
+    onError = sinon.stub();
+  });
+
+  it('checks object type', () => {
+    scan(null);
+    sinon.assert.calledWith(onError, ERR_TYPE, null, null, null, null);
+  });
+  it('ignores known fields and ext', () => {
+    scan({p11: 1, p12: 2, ext: {e1: 1, e2: 2}});
+    sinon.assert.notCalled(onError);
+  });
+  it('detects unknown fields', () => {
+    const obj = {p11: 1, unk: 2};
+    scan(obj);
+    sinon.assert.calledOnce(onError);
+    sinon.assert.calledWith(onError, ERR_UNKNOWN_FIELD, 'unk', obj, 'unk', 2);
+  });
+  describe('when nested', () => {
+    describe('directly', () => {
+      it('detects unknown fields', () => {
+        const obj = {inner: {p21: 1, unk: 2}};
+        scan(obj);
+        sinon.assert.calledOnce(onError);
+        sinon.assert.calledWith(onError, ERR_UNKNOWN_FIELD, 'inner.unk', obj.inner, 'unk', 2);
+      });
+      it('accepts enum values in range', () => {
+        scan({inner: {enum: 12}});
+        sinon.assert.notCalled(onError);
+      });
+      [Infinity, NaN, -Infinity].forEach(val => {
+        it(`does not accept ${val} in enum`, () => {
+          const obj = {inner: {enum: val}};
+          scan(obj);
+          sinon.assert.calledOnce(onError);
+          sinon.assert.calledWith(onError, ERR_ENUM, 'inner.enum', obj.inner, 'enum', val);
+        });
+      });
+      it('accepts arrays of enums that are in range', () => {
+        scan({inner: {enumArray: [12, 13]}});
+        sinon.assert.notCalled(onError);
+      })
+      it('detects enum values out of range', () => {
+        const obj = {inner: {enum: -1}};
+        scan(obj);
+        sinon.assert.calledOnce(onError);
+        sinon.assert.calledWith(onError, ERR_ENUM, 'inner.enum', obj.inner, 'enum', -1);
+      });
+      it('detects enum values that are not numbers', () => {
+        const obj = {inner: {enum: 'err'}};
+        scan(obj);
+        sinon.assert.calledOnce(onError);
+        sinon.assert.calledWith(onError, ERR_TYPE, 'inner.enum', obj.inner, 'enum', 'err');
+      })
+      it('detects arrays of enums that are out of range', () => {
+        const obj = {inner: {enumArray: [12, 13, -1, 14]}};
+        scan(obj);
+        sinon.assert.calledOnce(onError);
+        sinon.assert.calledWith(onError, ERR_ENUM, 'inner.enumArray.2', obj.inner.enumArray, 2, -1);
+      });
+      it('detects when enum arrays are not arrays', () => {
+        const obj = {inner: {enumArray: 'err'}};
+        scan(obj);
+        sinon.assert.calledOnce(onError);
+        sinon.assert.calledWith(onError, ERR_TYPE, 'inner.enumArray', obj.inner, 'enumArray', 'err');
+      });
+      it('detects items within enum arrays that are not numbers', () => {
+        const obj = {inner: {enumArray: [12, 'err', 13]}};
+        scan(obj);
+        sinon.assert.calledOnce(onError);
+        sinon.assert.calledWith(onError, ERR_TYPE, 'inner.enumArray.1', obj.inner.enumArray, 1, 'err');
+      })
+    });
+    describe('into arrays', () => {
+      it('detects if inner array is not an array', () => {
+        const obj = {innerArray: 'err', inner: {p21: 1}};
+        scan(obj);
+        sinon.assert.calledOnce(onError);
+        sinon.assert.calledWith(onError, ERR_TYPE, 'innerArray', obj, 'innerArray', 'err');
+      });
+      it('detects when elements of inner array are not objects', () => {
+        const obj = {innerArray: [{p21: 1}, 'err', {ext: {r: 1}}]};
+        scan(obj);
+        sinon.assert.calledOnce(onError);
+        sinon.assert.calledWith(onError, ERR_TYPE, 'innerArray.1', obj.innerArray, 1, 'err');
+      });
+      const oos = {
+        innerArray: [
+          {p22: 2, unk: 3, enumArray: [-1, 12, 'err']},
+          {p21: 1, enum: -1, ext: {e: 1}},
+        ]
+      };
+      it('detects invalid properties within inner array', () => {
+        const obj = deepClone(oos);
+        scan(obj);
+        sinon.assert.calledWith(onError, ERR_UNKNOWN_FIELD, 'innerArray.0.unk', obj.innerArray[0], 'unk', 3);
+        sinon.assert.calledWith(onError, ERR_ENUM, 'innerArray.0.enumArray.0', obj.innerArray[0].enumArray, 0, -1);
+        sinon.assert.calledWith(onError, ERR_TYPE, 'innerArray.0.enumArray.2', obj.innerArray[0].enumArray, 2, 'err');
+        sinon.assert.calledWith(onError, ERR_ENUM, 'innerArray.1.enum', obj.innerArray[1], 'enum', -1);
+      });
+      it('can remove all invalid properties during scan', () => {
+        onError.callsFake((errno, path, obj, field) => {
+          Array.isArray(obj) ? obj.splice(field, 1) : delete obj[field];
+        });
+        const obj = deepClone(oos);
+        scan(obj);
+        expect(obj).to.eql({
+          innerArray: [
+            {p22: 2, enumArray: [12]},
+            {p21: 1, ext: {e: 1}}
+          ]
+        });
+      })
+    })
+  })
+});