Uid2 module: major implementation change (prebid#9264)

* Complete the UID2 integration. Update docs. Add tests. * Removed some unnecessary code in uid2IdSystem.uid2IdSystem. Improved log messages. Pass through configured baseUrl. Tidied up some in-progress code problems. Added a timer mock to track and clear timers at the end of each test, to prevent interference. Improved testing code and fixed some bugs. * Move cookie cleanup into the after so it doesn't leave a mess behind for subsequent tests. Allow specifying multiple --file options when running/watching tests. * Provide an additional mock object for some test environments which don't provide crypto.subtle. * Improve some documentation for the UID2 module. * Improved UID2 module logging when debug flag is enabled. * Added tests around the api base url config for UID2. Added the new UID2 config to the example. * Update integration example to attempt a token refresh (it will fail due to not being a valid token). * Refactor to avoid duplicating cookie read code. Add a test for the case when the id value is provided directly in config without making use of the new token refresh system. * Fix an incorrect log call. Co-authored-by: Lionell Pack <lionell.pack@thetradedesk.com>
nextmillenniummedia · Feb 8, 2023 · 5139750 · 5139750
1 parent 6e6fff3
commit 5139750
Show file tree

Hide file tree

Showing 6 changed files with 655 additions and 35 deletions.
diff --git a/integrationExamples/gpt/userId_example.html b/integrationExamples/gpt/userId_example.html
@@ -249,7 +249,17 @@
               }
             },
             {
-              "name": "uid2"
+              "name": "uid2",
+              "params": {
+                "uid2Token": {
+                  "advertising_token": "example token",
+                  "refresh_token": "aslkdjaslkjdaslkhj",
+                  "identity_expires": Date.now() + 60*1000,
+                  "refresh_from": Date.now() - 10*1000,
+                  "refresh_expires": Date.now() + 12*60*60*1000,
+                  "refresh_response_key": null
+                }
+              }
             }
             ,
             {

diff --git a/karma.conf.maker.js b/karma.conf.maker.js
@@ -110,7 +110,7 @@ module.exports = function(codeCoverage, browserstack, watchMode, file, disableFe
   var webpackConfig = newWebpackConfig(codeCoverage, disableFeatures);
   var plugins = newPluginsArray(browserstack);
 
-  var files = file ? ['test/test_deps.js', file] : ['test/test_index.js'];
+  var files = file ? ['test/test_deps.js', file].flatMap(f => f) : ['test/test_index.js'];
   // This file opens the /debug.html tab automatically.
   // It has no real value unless you're running --watch, and intend to do some debugging in the browser.
   if (watchMode) {

diff --git a/modules/uid2IdSystem.js b/modules/uid2IdSystem.js
@@ -1,3 +1,4 @@
+/* eslint-disable no-console */
 /**
  * This module adds uid2 ID support to the User ID module
  * The {@link module:modules/userId} module is required.
@@ -10,48 +11,142 @@ import {submodule} from '../src/hook.js';
 import { getStorageManager } from '../src/storageManager.js';
 
 const MODULE_NAME = 'uid2';
+const MODULE_REVISION = `1.0`;
+const PREBID_VERSION = '$prebid.version$';
+const UID2_CLIENT_ID = `PrebidJS-${PREBID_VERSION}-UID2Module-${MODULE_REVISION}`;
 const GVLID = 887;
 const LOG_PRE_FIX = 'UID2: ';
 const ADVERTISING_COOKIE = '__uid2_advertising_token';
 
-function readCookie() {
-  return storage.cookiesAreEnabled() ? storage.getCookie(ADVERTISING_COOKIE) : null;
+// eslint-disable-next-line no-unused-vars
+const UID2_TEST_URL = 'https://operator-integ.uidapi.com';
+const UID2_PROD_URL = 'https://prod.uidapi.com';
+const UID2_BASE_URL = UID2_PROD_URL;
+
+function getStorage() {
+  return getStorageManager({gvlid: GVLID, moduleName: MODULE_NAME});
+}
+
+function createLogInfo(prefix) {
+  return function (...strings) {
+    logInfo(prefix + ' ', ...strings);
+  }
 }
+export const storage = getStorage();
+const _logInfo = createLogInfo(LOG_PRE_FIX);
 
 function readFromLocalStorage() {
   return storage.localStorageIsEnabled() ? storage.getDataFromLocalStorage(ADVERTISING_COOKIE) : null;
 }
 
-function getStorage() {
-  return getStorageManager({gvlid: GVLID, moduleName: MODULE_NAME});
+function readModuleCookie() {
+  const cookie = readCookie(ADVERTISING_COOKIE);
+  if (cookie && cookie.includes('{')) {
+    return JSON.parse(cookie);
+  }
+  return cookie;
 }
 
-const storage = getStorage();
+function readJsonCookie(cookieName) {
+  return JSON.parse(readCookie(cookieName));
+}
 
-const _logInfo = createLogInfo(LOG_PRE_FIX);
+function readCookie(cookieName) {
+  const cookie = storage.cookiesAreEnabled() ? storage.getCookie(cookieName) : null;
+  if (!cookie) {
+    _logInfo(`Attempted to read UID2 from cookie '${cookieName}' but it was empty`);
+    return null;
+  };
+  _logInfo(`Read UID2 from cookie '${cookieName}'`);
+  return cookie;
+}
 
-function createLogInfo(prefix) {
-  return function (...strings) {
-    logInfo(prefix + ' ', ...strings);
+function storeValue(value) {
+  if (storage.cookiesAreEnabled()) {
+    storage.setCookie(ADVERTISING_COOKIE, JSON.stringify(value), Date.now() + 60 * 60 * 24 * 1000);
+  } else if (storage.localStorageIsEnabled()) {
+    storage.setLocalStorage(ADVERTISING_COOKIE, value);
   }
 }
 
-/**
- * Encode the id
- * @param value
- * @returns {string|*}
- */
-function encodeId(value) {
-  const result = {};
-  if (value) {
-    const bidIds = {
-      id: value
+function isValidIdentity(identity) {
+  return !!(typeof identity === 'object' && identity !== null && identity.advertising_token && identity.identity_expires && identity.refresh_from && identity.refresh_token && identity.refresh_expires);
+}
+
+// This is extracted from an in-progress API client. Once it's available via NPM, this class should be replaced with the NPM package.
+class Uid2ApiClient {
+  constructor(opts) {
+    this._baseUrl = opts.baseUrl ? opts.baseUrl : UID2_BASE_URL;
+    this._clientVersion = UID2_CLIENT_ID;
+  }
+  createArrayBuffer(text) {
+    const arrayBuffer = new Uint8Array(text.length);
+    for (let i = 0; i < text.length; i++) {
+      arrayBuffer[i] = text.charCodeAt(i);
     }
-    result.uid2 = bidIds;
-    _logInfo('Decoded value ' + JSON.stringify(result));
-    return result;
+    return arrayBuffer;
+  }
+  hasStatusResponse(response) {
+    return typeof (response) === 'object' && response && response.status;
+  }
+  isValidRefreshResponse(response) {
+    return this.hasStatusResponse(response) && (
+      response.status === 'optout' || response.status === 'expired_token' || (response.status === 'success' && response.body && isValidIdentity(response.body))
+    );
+  }
+  ResponseToRefreshResult(response) {
+    if (this.isValidRefreshResponse(response)) {
+      if (response.status === 'success') { return { status: response.status, identity: response.body }; }
+      return response;
+    } else { return "Response didn't contain a valid status"; }
+  }
+  callRefreshApi(refreshDetails) {
+    const url = this._baseUrl + '/v2/token/refresh';
+    const req = new XMLHttpRequest();
+    req.overrideMimeType('text/plain');
+    req.open('POST', url, true);
+    req.setRequestHeader('X-UID2-Client-Version', this._clientVersion);
+    let resolvePromise;
+    let rejectPromise;
+    const promise = new Promise((resolve, reject) => {
+      resolvePromise = resolve;
+      rejectPromise = reject;
+    });
+    req.onreadystatechange = () => {
+      if (req.readyState !== req.DONE) { return; }
+      try {
+        if (!refreshDetails.refresh_response_key || req.status !== 200) {
+          _logInfo('Error status OR no response decryption key available, assuming unencrypted JSON');
+          const response = JSON.parse(req.responseText);
+          const result = this.ResponseToRefreshResult(response);
+          if (typeof result === 'string') { rejectPromise(result); } else { resolvePromise(result); }
+        } else {
+          _logInfo('Decrypting refresh API response');
+          const encodeResp = this.createArrayBuffer(atob(req.responseText));
+          window.crypto.subtle.importKey('raw', this.createArrayBuffer(atob(refreshDetails.refresh_response_key)), { name: 'AES-GCM' }, false, ['decrypt']).then((key) => {
+            _logInfo('Imported decryption key')
+            // returns the symmetric key
+            window.crypto.subtle.decrypt({
+              name: 'AES-GCM',
+              iv: encodeResp.slice(0, 12),
+              tagLength: 128, // The tagLength you used to encrypt (if any)
+            }, key, encodeResp.slice(12)).then((decrypted) => {
+              const decryptedResponse = String.fromCharCode(...new Uint8Array(decrypted));
+              _logInfo('Decrypted to:', decryptedResponse);
+              const response = JSON.parse(decryptedResponse);
+              const result = this.ResponseToRefreshResult(response);
+              if (typeof result === 'string') { rejectPromise(result); } else { resolvePromise(result); }
+            }, (reason) => console.warn(`Call to UID2 API failed`, reason));
+          }, (reason) => console.warn(`Call to UID2 API failed`, reason));
+        }
+      } catch (err) {
+        rejectPromise(err);
+      }
+    };
+    _logInfo('Sending refresh request', req);
+    req.send(refreshDetails.refresh_token);
+    return promise;
   }
-  return undefined;
 }
 
 /** @type {Submodule} */
@@ -71,27 +166,118 @@ export const uid2IdSubmodule = {
    * decode the stored id value for passing to bid requests
    * @function
    * @param {string} value
-   * @returns {{uid2:{ id: string }} or undefined if value doesn't exists
+   * @returns {{uid2:{ id: string } }} or undefined if value doesn't exists
    */
   decode(value) {
-    return (value) ? encodeId(value) : undefined;
+    const result = decodeImpl(value);
+    _logInfo('UID2 decode returned', result);
+    return result;
   },
 
   /**
    * performs action to obtain id and return a value.
    * @function
-   * @param {SubmoduleConfig} [config]
+   * @param {SubmoduleConfig} [configparams]
    * @param {ConsentData|undefined} consentData
    * @returns {uid2Id}
    */
   getId(config, consentData) {
-    _logInfo('Creating UID 2.0');
-    let value = readCookie() || readFromLocalStorage();
-    _logInfo('The advertising token: ' + value);
-    return {id: value}
+    const result = getIdImpl(config, consentData);
+    _logInfo(`UID2 getId returned`, result);
+    return result;
   },
-
 };
 
+function refreshTokenAndStore(baseUrl, token) {
+  _logInfo('UID2 base url provided: ', baseUrl);
+  const client = new Uid2ApiClient({baseUrl});
+  return client.callRefreshApi(token).then((response) => {
+    _logInfo('Refresh endpoint responded with:', response);
+    const tokens = {
+      originalToken: token,
+      latestToken: response.identity,
+    };
+    storeValue(tokens);
+    return tokens;
+  });
+}
+
+function decodeImpl(value) {
+  if (typeof value === 'string') {
+    _logInfo('Found an old-style ID from an earlier version of the module. Refresh is unavailable for this token.');
+    const result = { uid2: { id: value } };
+    return result;
+  }
+  if (Date.now() < value.latestToken.identity_expires) {
+    return { uid2: { id: value.latestToken.advertising_token } };
+  }
+  return null;
+}
+
+function getIdImpl(config, consentData) {
+  let suppliedToken = null;
+  const uid2BaseUrl = config?.params?.uid2ApiBase ?? UID2_BASE_URL;
+  if (config && config.params) {
+    if (config.params.uid2Token) {
+      suppliedToken = config.params.uid2Token;
+      _logInfo('Read token from params', suppliedToken);
+    } else if (config.params.uid2ServerCookie) {
+      suppliedToken = readJsonCookie(config.params.uid2ServerCookie);
+      _logInfo('Read token from server-supplied cookie', suppliedToken);
+    }
+  }
+  let storedTokens = readModuleCookie() || readFromLocalStorage();
+  _logInfo('Loaded module-stored tokens:', storedTokens);
+
+  if (storedTokens && typeof storedTokens === 'string') {
+    // Legacy value stored, this must be from an old integration. If no token supplied, just use the legacy value.
+
+    if (!suppliedToken) {
+      _logInfo('Returning legacy cookie value.');
+      return { id: storedTokens };
+    }
+    // Otherwise, ignore the legacy value - it should get over-written later anyway.
+    _logInfo('Discarding superseded legacy cookie.');
+    storedTokens = null;
+  }
+
+  if (suppliedToken && storedTokens) {
+    if (storedTokens.originalToken?.advertising_token !== suppliedToken.advertising_token) {
+      _logInfo('Server supplied new token - ignoring stored value.', storedTokens.originalToken?.advertising_token, suppliedToken.advertising_token);
+      // Stored token wasn't originally sourced from the provided token - ignore the stored value. A new user has logged in?
+      storedTokens = null;
+    }
+  }
+  // At this point, any legacy values or superseded stored tokens have been nulled out.
+  const useSuppliedToken = !(storedTokens?.latestToken) || (suppliedToken && suppliedToken.identity_expires > storedTokens.latestToken.identity_expires);
+  const newestAvailableToken = useSuppliedToken ? suppliedToken : storedTokens.latestToken;
+  _logInfo('UID2 module selected latest token', useSuppliedToken, newestAvailableToken);
+  if (!newestAvailableToken || Date.now() > newestAvailableToken.refresh_expires) {
+    _logInfo('Newest available token is expired and not refreshable.');
+    return { id: null };
+  }
+  if (Date.now() > newestAvailableToken.identity_expires) {
+    const promise = refreshTokenAndStore(uid2BaseUrl, newestAvailableToken);
+    _logInfo('Token is expired but can be refreshed, attempting refresh.');
+    return { callback: (cb) => {
+      promise.then((result) => {
+        _logInfo('Refresh reponded, passing the updated token on.', result);
+        cb(result);
+      });
+    } };
+  }
+  // If should refresh (but don't need to), refresh in the background.
+  if (Date.now() > newestAvailableToken.refresh_from) {
+    _logInfo(`Refreshing token in background with low priority.`);
+    refreshTokenAndStore(uid2BaseUrl, newestAvailableToken);
+  }
+  const tokens = {
+    originalToken: suppliedToken ?? storedTokens?.originalToken,
+    latestToken: newestAvailableToken,
+  };
+  storeValue(tokens);
+  return { id: tokens };
+}
+
 // Register submodule for userId
 submodule('userId', uid2IdSubmodule);
diff --git a/modules/uid2IdSystem.md b/modules/uid2IdSystem.md
@@ -6,19 +6,49 @@ UID 2.0 ID Module.
 
 Individual params may be set for the UID 2.0 Submodule. At least one identifier must be set in the params.
 
+The module will handle refreshing the token periodically and storing the updated token using the Prebid.js storage manager. If you provide an expired identity and the module has a valid identity which was refreshed from the identity you provide, it will use the refreshed identity. The module stores the original token used for refreshing the token, and it will use the refreshed tokens as long as the original token matches the one supplied.
+
 ```
 pbjs.setConfig({
     userSync: {
         userIds: [{
-            name: 'uid2'
+            name: 'uid2',
+            params: {
+                // Either:
+                uid2ServerCookie: 'your_UID2_server_set_cookie_name'
+                // Or:
+                uid2Token: {
+                    'advertising_token': '...',
+                    'refresh_token': '...',
+                    // etc. - see the Sample Token below for contents of this object
+                }
+            }
         }]
     }
 });
 ```
 ## Parameter Descriptions for the `usersync` Configuration Section
 The below parameters apply only to the UID 2.0 User ID Module integration.
 
+You should supply either `uid2Token` or `uid2ServerCookie`.
+
+If you provide `uid2Token`, the value should be a JavaScript/JSON object with the decrypted `body` payload response from a call to either `/token/generate` or `/token/refresh`.
+
+If you provide `uid2ServerCookie`, the module will expect that same JSON object to be stored in the cookie - i.e. it will pass the cookie value to `JSON.parse` and expect to receive an object containing similar to what you see in the `Sample token` section below.
+
+The module will make calls to the `/token/refresh` endpoint to update the token it stores internally, so bids may contain an updated token.
+
+If neither of `uid2Token` or `uid2ServerCookie` are supplied, and the module has stored a token using the Prebid.js storage system (typically in a cookie named `__uid2_advertising_token`), it will use that token. This cookie is internal to the module and should not be set directly.
+
+If a new token is supplied which does not match the original token used to generate any refreshed tokens, all stored tokens will be discarded and the new token used instead (refreshed if necessary).
+
+### Sample token
+
+`{`<br />&nbsp;&nbsp;`"advertising_token": "...",`<br />&nbsp;&nbsp;`"refresh_token": "...",`<br />&nbsp;&nbsp;`"identity_expires": 1633643601000,`<br />&nbsp;&nbsp;`"refresh_from": 1633643001000,`<br />&nbsp;&nbsp;`"refresh_expires": 1636322000000,`<br />&nbsp;&nbsp;`"refresh_response_key": "wR5t6HKMfJ2r4J7fEGX9Gw=="`<br />`}`
+
 | Param under userSync.userIds[] | Scope | Type | Description | Example |
 | --- | --- | --- | --- | --- |
 | name | Required | String | ID value for the UID20 module - `"uid2"` | `"uid2"` |
-| value | Optional | Object | Used only if the page has a separate mechanism for storing the UID 2.0 ID. The value is an object containing the values to be sent to the adapters. In this scenario, no URL is called and nothing is added to local storage | `{"uid2": { "id": "eb33b0cb-8d35-4722-b9c0-1a31d4064888"}}` |
+| params.uid2Token | Optional | Object | The initial UID2 token. This should be `body` element of the decrypted response from a call to the `/token/generate` or `/token/refresh` endpoint. | See the sample token above. |
+| params.uid2ServerCookie | Optional | String | The name of a cookie which holds the initial UID2 token, set by the server. The cookie should contain JSON in the same format as the alternative uid2Token param. **If uid2Token is supplied, this param is ignored.** | See the sample token above. |
+| params.uid2ApiBase | Optional | String | Overrides the default UID2 API endpoint. | `https://prod.uidapi.com` _(default)_ |