Prebid core & PBS adapter: Feature tags and optional compilation of n…

…ative support (#8219) * Upgrade webpack to 5; gulp build works * Fix karma, except events * Uniform access to events, import * or require (import * from 'events.js' / import events from 'events.js' return different objects, which is a problem for stubbing) * Fix (?) adapters that use `this` inappropriately * Update webpack-bundle-analyzer * Fix warnings * Enable tree shaking * Set webpack mode 'none' (or else tests fail (!)) * Update coreJS version in babelrc - #7943 * Use babel to translate to commonjs only for unit tests; enable production mode * Define feature flags at compile time - starting with just "NATIVE" * Add build-bundle-verbose * Run tests with all features disabled * Fix build-bundle-verbose * Tag native#nativeAdapters * Merge master * Add fsevents as optional dep * Adjust syntax for node 12
prebid · Jun 17, 2022 · 5b4987d · 5b4987d
1 parent 583c80b
commit 5b4987d
Show file tree

Hide file tree

Showing 23 changed files with 436 additions and 308 deletions.
diff --git a/.eslintrc.js b/.eslintrc.js
@@ -21,7 +21,8 @@ module.exports = {
   globals: {
     '$$PREBID_GLOBAL$$': false,
     'BROWSERSTACK_USERNAME': false,
-    'BROWSERSTACK_KEY': false
+    'BROWSERSTACK_KEY': false,
+    'FEATURES': 'readonly',
   },
   // use babel as parser for fancy syntax
   parser: '@babel/eslint-parser',

diff --git a/features.json b/features.json
@@ -0,0 +1,3 @@
+[
+  "NATIVE"
+]
diff --git a/gulpHelpers.js b/gulpHelpers.js
@@ -206,5 +206,11 @@ module.exports = {
     }
 
     return options;
-  }
+  },
+  getDisabledFeatures() {
+    return (argv.disable || '')
+      .split(',')
+      .map((s) => s.trim())
+      .filter((s) => s);
+  },
 };
diff --git a/gulpfile.js b/gulpfile.js
@@ -9,8 +9,6 @@ var connect = require('gulp-connect');
 var webpack = require('webpack');
 var webpackStream = require('webpack-stream');
 var gulpClean = require('gulp-clean');
-var KarmaServer = require('karma').Server;
-var karmaConfMaker = require('./karma.conf.maker.js');
 var opens = require('opn');
 var webpackConfig = require('./webpack.conf.js');
 var helpers = require('./gulpHelpers.js');
@@ -32,7 +30,8 @@ var prebid = require('./package.json');
 var port = 9999;
 const INTEG_SERVER_HOST = argv.host ? argv.host : 'localhost';
 const INTEG_SERVER_PORT = 4444;
-const { spawn } = require('child_process');
+const { spawn, fork } = require('child_process');
+const TerserPlugin = require('terser-webpack-plugin');
 
 // these modules must be explicitly listed in --modules to be included in the build, won't be part of "all" modules
 var explicitModules = [
@@ -140,21 +139,23 @@ function makeDevpackPkg() {
     .pipe(connect.reload());
 }
 
-function makeWebpackPkg() {
-  var cloned = _.cloneDeep(webpackConfig);
+function makeWebpackPkg(extraConfig = {}) {
+  var cloned = _.merge(_.cloneDeep(webpackConfig), extraConfig);
   if (!argv.sourceMaps) {
     delete cloned.devtool;
   }
 
-  var externalModules = helpers.getArgModules();
+  return function buildBundle() {
+    var externalModules = helpers.getArgModules();
 
-  const analyticsSources = helpers.getAnalyticsSources();
-  const moduleSources = helpers.getModulePaths(externalModules);
+    const analyticsSources = helpers.getAnalyticsSources();
+    const moduleSources = helpers.getModulePaths(externalModules);
 
-  return gulp.src([].concat(moduleSources, analyticsSources, 'src/prebid.js'))
-    .pipe(helpers.nameModules(externalModules))
-    .pipe(webpackStream(cloned, webpack))
-    .pipe(gulp.dest('build/dist'));
+    return gulp.src([].concat(moduleSources, analyticsSources, 'src/prebid.js'))
+      .pipe(helpers.nameModules(externalModules))
+      .pipe(webpackStream(cloned, webpack))
+      .pipe(gulp.dest('build/dist'));
+  }
 }
 
 function getModulesListToAddInBanner(modules) {
@@ -272,9 +273,11 @@ function bundle(dev, moduleArr) {
 
 function testTaskMaker(options = {}) {
   ['watch', 'e2e', 'file', 'browserstack', 'notest'].forEach(opt => {
-    options[opt] = options[opt] || argv[opt];
+    options[opt] = options.hasOwnProperty(opt) ? options[opt] : argv[opt];
   })
 
+  options.disableFeatures = options.disableFeatures || helpers.getDisabledFeatures();
+
   return function test(done) {
     if (options.notest) {
       done();
@@ -295,14 +298,7 @@ function testTaskMaker(options = {}) {
           process.exit(1);
         });
     } else {
-      var karmaConf = karmaConfMaker(false, options.browserstack, options.watch, options.file);
-
-      var browserOverride = helpers.parseBrowserArgs(argv);
-      if (browserOverride.length > 0) {
-        karmaConf.browsers = browserOverride;
-      }
-
-      new KarmaServer(karmaConf, newKarmaCallback(done)).start();
+      runKarma(options, done)
     }
   }
 }
@@ -329,25 +325,24 @@ function runWebdriver({file}) {
   return execa(wdioCmd, wdioOpts, { stdio: 'inherit' });
 }
 
-function newKarmaCallback(done) {
-  return function (exitCode) {
+function runKarma(options, done) {
+  // the karma server appears to leak memory; starting it multiple times in a row will run out of heap
+  // here we run it in a separate process to bypass the problem
+  options = Object.assign({browsers: helpers.parseBrowserArgs(argv)}, options)
+  const child = fork('./karmaRunner.js');
+  child.on('exit', (exitCode) => {
     if (exitCode) {
       done(new Error('Karma tests failed with exit code ' + exitCode));
-      if (argv.browserstack) {
-        process.exit(exitCode);
-      }
     } else {
       done();
-      if (argv.browserstack) {
-        process.exit(exitCode);
-      }
     }
-  }
+  })
+  child.send(options);
 }
 
 // If --file "<path-to-test-file>" is given, the task will only run tests in the specified file.
 function testCoverage(done) {
-  new KarmaServer(karmaConfMaker(true, false, false, argv.file), newKarmaCallback(done)).start();
+  runKarma({coverage: true, browserstack: false, watch: false, file: argv.file}, done);
 }
 
 function coveralls() { // 2nd arg is a dependency: 'test' must be finished
@@ -425,11 +420,30 @@ gulp.task(clean);
 gulp.task(escapePostbidConfig);
 
 gulp.task('build-bundle-dev', gulp.series(makeDevpackPkg, gulpBundle.bind(null, true)));
-gulp.task('build-bundle-prod', gulp.series(makeWebpackPkg, gulpBundle.bind(null, false)));
+gulp.task('build-bundle-prod', gulp.series(makeWebpackPkg(), gulpBundle.bind(null, false)));
+// build-bundle-verbose - prod bundle except names and comments are preserved. Use this to see the effects
+// of dead code elimination.
+gulp.task('build-bundle-verbose', gulp.series(makeWebpackPkg({
+  optimization: {
+    minimizer: [
+      new TerserPlugin({
+        parallel: true,
+        terserOptions: {
+          mangle: false,
+          format: {
+            comments: 'all'
+          }
+        },
+        extractComments: false,
+      }),
+    ],
+  }
+}), gulpBundle.bind(null, false)));
 
 // public tasks (dependencies are needed for each task since they can be ran on their own)
 gulp.task('test-only', test);
-gulp.task('test', gulp.series(clean, lint, 'test-only'));
+gulp.task('test-all-features-disabled', testTaskMaker({disableFeatures: require('./features.json'), oneBrowser: 'chrome', watch: false}))
+gulp.task('test', gulp.series(clean, lint, gulp.series('test-all-features-disabled', 'test-only')));
 
 gulp.task('test-coverage', gulp.series(clean, testCoverage));
 gulp.task(viewCoverage);

diff --git a/karma.conf.maker.js b/karma.conf.maker.js
@@ -7,7 +7,7 @@ var _ = require('lodash');
 var webpackConf = require('./webpack.conf.js');
 var karmaConstants = require('karma').constants;
 
-function newWebpackConfig(codeCoverage) {
+function newWebpackConfig(codeCoverage, disableFeatures) {
   // Make a clone here because we plan on mutating this object, and don't want parallel tasks to trample each other.
   var webpackConfig = _.cloneDeep(webpackConf);
 
@@ -22,7 +22,7 @@ function newWebpackConfig(codeCoverage) {
     .flatMap((r) => r.use)
     .filter((use) => use.loader === 'babel-loader')
     .forEach((use) => {
-      use.options = babelConfig({test: true});
+      use.options = babelConfig({test: true, disableFeatures});
     });
 
   if (codeCoverage) {
@@ -117,8 +117,8 @@ function setBrowsers(karmaConf, browserstack) {
   }
 }
 
-module.exports = function(codeCoverage, browserstack, watchMode, file) {
-  var webpackConfig = newWebpackConfig(codeCoverage);
+module.exports = function(codeCoverage, browserstack, watchMode, file, disableFeatures) {
+  var webpackConfig = newWebpackConfig(codeCoverage, disableFeatures);
   var plugins = newPluginsArray(browserstack);
 
   var files = file ? ['test/test_deps.js', file] : ['test/test_index.js'];

diff --git a/karmaRunner.js b/karmaRunner.js
@@ -0,0 +1,23 @@
+const karma = require('karma');
+const process = require('process');
+const karmaConfMaker = require('./karma.conf.maker.js');
+
+process.on('message', function(options) {
+  try {
+    let cfg = karmaConfMaker(options.coverage, options.browserstack, options.watch, options.file, options.disableFeatures);
+    if (options.browsers && options.browsers.length) {
+      cfg.browsers = options.browsers;
+    }
+    if (options.oneBrowser) {
+      cfg.browsers = [cfg.browsers.find((b) => b.toLowerCase().includes(options.oneBrowser.toLowerCase())) || cfg.browsers[0]]
+    }
+    cfg = karma.config.parseConfig(null, cfg);
+    new karma.Server(cfg, (exitCode) => {
+      process.exit(exitCode);
+    }).start();
+  } catch (e) {
+    // eslint-disable-next-line
+    console.error(e);
+    process.exit(1);
+  }
+});
diff --git a/modules/prebidServerBidAdapter/index.js b/modules/prebidServerBidAdapter/index.js
@@ -438,18 +438,19 @@ let nativeEventTrackerMethodMap = {
   js: 2
 };
 
-// enable reverse lookup
-[
-  nativeDataIdMap,
-  nativeImgIdMap,
-  nativeEventTrackerEventMap,
-  nativeEventTrackerMethodMap
-].forEach(map => {
-  Object.keys(map).forEach(key => {
-    map[map[key]] = key;
+if (FEATURES.NATIVE) {
+  // enable reverse lookup
+  [
+    nativeDataIdMap,
+    nativeImgIdMap,
+    nativeEventTrackerEventMap,
+    nativeEventTrackerMethodMap
+  ].forEach(map => {
+    Object.keys(map).forEach(key => {
+      map[map[key]] = key;
+    });
   });
-});
-
+}
 /*
  * Protocol spec for OpenRTB endpoint
  * e.g., https://<prebid-server-url>/v1/openrtb2/auction
@@ -558,7 +559,7 @@ Object.assign(ORTB2.prototype, {
 
       const nativeParams = adUnit.nativeParams;
       let nativeAssets;
-      if (nativeParams) {
+      if (FEATURES.NATIVE && nativeParams) {
         let idCounter = -1;
         try {
           nativeAssets = nativeAssetCache[impressionId] = Object.keys(nativeParams).reduce((assets, type) => {
@@ -683,7 +684,7 @@ Object.assign(ORTB2.prototype, {
         }
       }
 
-      if (nativeAssets) {
+      if (FEATURES.NATIVE && nativeAssets) {
         try {
           mediaTypes['native'] = {
             request: JSON.stringify({
@@ -1036,7 +1037,7 @@ Object.assign(ORTB2.prototype, {
 
             if (bid.adm) { bidObject.vastXml = bid.adm; }
             if (!bidObject.vastUrl && bid.nurl) { bidObject.vastUrl = bid.nurl; }
-          } else if (deepAccess(bid, 'ext.prebid.type') === NATIVE) {
+          } else if (FEATURES.NATIVE && deepAccess(bid, 'ext.prebid.type') === NATIVE) {
             bidObject.mediaType = NATIVE;
             let adm;
             if (typeof bid.adm === 'string') {

diff --git a/modules/sizeMappingV2.js b/modules/sizeMappingV2.js
@@ -129,7 +129,7 @@ export function checkAdUnitSetupHook(adUnits) {
           Verify that 'config.active' is a 'boolean'.
           If not, return 'false'.
         */
-        if (mediaType === 'native') {
+        if (FEATURES.NATIVE && mediaType === 'native') {
           if (typeof config[propertyName] !== 'boolean') {
             logError(`Ad unit ${adUnitCode}: Invalid declaration of 'active' in 'mediaTypes.${mediaType}.sizeConfig[${index}]'. ${conditionalLogMessages[mediaType]}`);
             isValid = false;
@@ -206,7 +206,7 @@ export function checkAdUnitSetupHook(adUnits) {
       }
     }
 
-    if (mediaTypes.native) {
+    if (FEATURES.NATIVE && mediaTypes.native) {
       // Apply the old native checks
       validatedNative = validatedVideo ? adUnitSetupChecks.validateNativeMediaType(validatedVideo) : validatedBanner ? adUnitSetupChecks.validateNativeMediaType(validatedBanner) : adUnitSetupChecks.validateNativeMediaType(adUnit);
 

diff --git a/plugins/pbjsGlobals.js b/plugins/pbjsGlobals.js
@@ -2,6 +2,20 @@
 let t = require('@babel/core').types;
 let prebid = require('../package.json');
 const path = require('path');
+const allFeatures = new Set(require('../features.json'));
+
+const FEATURES_GLOBAL = 'FEATURES';
+
+function featureMap(disable = []) {
+  disable = disable.map((s) => s.toUpperCase());
+  disable.forEach((f) => {
+    if (!allFeatures.has(f)) {
+      throw new Error(`Unrecognized feature: ${f}`)
+    }
+  });
+  disable = new Set(disable);
+  return Object.fromEntries([...allFeatures.keys()].map((f) => [f, !disable.has(f)]));
+}
 
 function getNpmVersion(version) {
   try {
@@ -13,6 +27,7 @@ function getNpmVersion(version) {
 
 module.exports = function(api, options) {
   const pbGlobal = options.globalVarName || prebid.globalVarName;
+  const features = featureMap(options.disableFeatures);
   let replace = {
     '$prebid.version$': prebid.version,
     '$$PREBID_GLOBAL$$': pbGlobal,
@@ -91,6 +106,17 @@ module.exports = function(api, options) {
             }
           }
         });
+      },
+      MemberExpression(path) {
+        if (
+          t.isIdentifier(path.node.object) &&
+          path.node.object.name === FEATURES_GLOBAL &&
+          !path.scope.hasBinding(FEATURES_GLOBAL) &&
+          t.isIdentifier(path.node.property) &&
+          features.hasOwnProperty(path.node.property.name)
+        ) {
+          path.replaceWith(t.booleanLiteral(features[path.node.property.name]));
+        }
       }
     }
   };

diff --git a/src/adapterManager.js b/src/adapterManager.js
@@ -211,7 +211,9 @@ adapterManager.makeBidRequests = hook('sync', function (adUnits, auctionStart, a
    * @see {@link https://github.com/prebid/Prebid.js/issues/4149|Issue}
    */
   events.emit(CONSTANTS.EVENTS.BEFORE_REQUEST_BIDS, adUnits);
-  decorateAdUnitsWithNativeParams(adUnits);
+  if (FEATURES.NATIVE) {
+    decorateAdUnitsWithNativeParams(adUnits);
+  }
   adUnits = setupAdUnitMediaTypes(adUnits, labels);
 
   let {[PARTITIONS.CLIENT]: clientBidders, [PARTITIONS.SERVER]: serverBidders} = partitionBidders(adUnits, _s2sConfigs);
@@ -425,7 +427,7 @@ adapterManager.callBids = (adUnits, bidRequests, addBidResponse, doneCb, request
 function getSupportedMediaTypes(bidderCode) {
   let supportedMediaTypes = [];
   if (includes(adapterManager.videoAdapters, bidderCode)) supportedMediaTypes.push('video');
-  if (includes(nativeAdapters, bidderCode)) supportedMediaTypes.push('native');
+  if (FEATURES.NATIVE && includes(nativeAdapters, bidderCode)) supportedMediaTypes.push('native');
   return supportedMediaTypes;
 }
 
@@ -439,7 +441,7 @@ adapterManager.registerBidAdapter = function (bidAdapter, bidderCode, {supported
       if (includes(supportedMediaTypes, 'video')) {
         adapterManager.videoAdapters.push(bidderCode);
       }
-      if (includes(supportedMediaTypes, 'native')) {
+      if (FEATURES.NATIVE && includes(supportedMediaTypes, 'native')) {
         nativeAdapters.push(bidderCode);
       }
     } else {

diff --git a/src/adapters/bidderFactory.js b/src/adapters/bidderFactory.js
@@ -542,7 +542,7 @@ export function isValid(adUnitCode, bid, {index = auctionManager.index} = {}) {
     return false;
   }
 
-  if (bid.mediaType === 'native' && !nativeBidIsValid(bid, {index})) {
+  if (FEATURES.NATIVE && bid.mediaType === 'native' && !nativeBidIsValid(bid, {index})) {
     logError(errorMessage('Native bid missing some required properties.'));
     return false;
   }