Initial commit

.editorconfig

@@ -0,0 +1,13 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.txt]
+indent_size = 0
+insert_final_newline = false

.eslintrc.cjs

@@ -0,0 +1,16 @@
+module.exports = {
+  'env': {
+    'node': true,
+    'es2021': true,
+  },
+  'extends': ['eslint:recommended', 'plugin:prettier/recommended'],
+  'overrides': [],
+  'parserOptions': {
+    'ecmaVersion': 'latest',
+    'sourceType': 'module',
+  },
+  'plugins': ['prettier'],
+  'rules': {
+    'prettier/prettier': 'error',
+  },
+};

.gitignore

@@ -0,0 +1,15 @@
+node_modules/
+.DS_Store/
+
+# Ignore all files within a directory
+output/current/idle/**
+output/current/idleWithBrowser/**
+output/current/withGhostery/**
+output/current/withoutGhostery/**
+output/time/withGhostery/**
+output/time/withoutGhostery/**
+profiles/withGhostery/**
+profiles/withoutGhostery/**
+
+# Do not ignore a special file name
+!.gitkeep

.prettierrc

@@ -0,0 +1,9 @@
+{
+  "semi": true,
+  "useTabs": false,
+  "singleQuote": true,
+  "trailingComma": "all",
+  "bracketSpacing": true,
+  "arrowParens": "always",
+  "quoteProps": "preserve"
+}

.tool-versions

@@ -0,0 +1 @@
+nodejs 20.2.0

README.md

@@ -0,0 +1,167 @@
+# Ghostery Benchmarks
+
+## How to start?
+
+Install dependencies
+
+```
+npm ci
+```
+
+To run main script which opens 360 websites one by one please follow commands below. Websites are picked based on https://github.com/whotracksme/whotracks.me
+
+Start benchmarks
+
+```
+npm start
+```
+
+Usage:
+
+```
+--US            site list for US region
+--EU            site list for European region
+--with-ghostery load Ghostery extension
+```
+
+Example output:
+```
+LOG: Addon temp path:
+LOG: Downloading addon
+LOG: Unpacking addon
+LOG: Addon path:
+INFO: Open websites from for region: EU.
+LOG={"index":1,"url":"https://kingoloto.com/subscribe.html","loadTime":955,"loadedAt":"2023-05-19T02:20:32.748Z"}
+LOG={"index":2,"url":"https://bananalotto.fr/subscribe.html","loadTime":860,"loadedAt":"2023-05-19T02:20:35.739Z"}
+LOG={"index":359,"url":"https://krunker.io/","loadTime":2635,"loadedAt":"2023-05-19T02:48:12.686Z"}
+LOG={"index":360,"url":"https://www.virgilio.it/","loadTime":1433,"loadedAt":"2023-05-19T02:48:17.331Z"}
+
+```
+
+## Convert the current to power based on the measurements taken from the benchmark
+
+To measure AC current, Gravity Analog AC Current Sensor was used. All the code to gather current value is available on https://wiki.dfrobot.com/Gravity_Analog_AC_Current_Sensor__SKU_SEN0211_
+
+
+Start calculate
+
+```
+npm run -- currentToPower
+```
+
+
+Example input:
+```
+10:11:50.121	0.091
+10:11:50.121	0.112
+10:11:50.121	0.104
+10:11:50.125	0.104
+10:11:50.135	0.098
+10:11:50.152	0.097
+10:11:50.152	0.095
+```
+
+
+Example output:
+```
+[
+  {
+    dataSet: 1,
+    fileName: 'idle/Idle_Set1_ 2023-05-17 22-00-33.txt',
+    AVG: 9.829,
+    AVG_CTP: 9.826,
+    INTEGRAL: 9.829,
+    INTEGRAL_CTP: 9.826,
+    durationInMin: 59.98
+  },
+  {
+    dataSet: 2,
+    fileName: 'idle/Idle_Set2_ 2023-05-17 22-00-33.txt',
+    AVG: 9.004,
+    AVG_CTP: 9.004,
+    INTEGRAL: 9.004,
+    INTEGRAL_CTP: 9.004,
+    durationInMin: 60.00
+  },
+  {
+    dataSet: 28,
+    fileName: 'withoutGhostery/Without_Set8_2023-05-18 20-17-08.txt',
+    AVG: 22.353,
+    AVG_CTP: 15.852,
+    INTEGRAL: 22.353,
+    INTEGRAL_CTP: 15.852,
+    durationInMin: 42.55
+  }
+]
+```
+
+
+
+
+## Based on the collected data from the benchmark, calculate the average time to load URLs
+
+Start read time
+
+```
+npm run -- readTime
+```
+
+Usage:
+
+```
+--US            site list for US region
+--EU            site list for European region
+```
+
+Example output:
+```
+With Ghostery
+Average urls load time:
+{
+  'https://kingoloto.com/subscribe.html': 1242.125,
+  'https://bananalotto.fr/subscribe.html': 1619,
+  'https://www.vogue.com/': 2542.125,
+  'https://krunker.io/': 2627.625,
+  'https://www.virgilio.it/': 1555.75,
+  'https://velvet.hu/': null
+}
+Duration of urls load time for specific set of measurements:
+[
+  'Set withGhostery/2023-05-19T02_20_04.734Z.txt: Load time duration: 27.74 min.',
+  'Set withGhostery/2023-05-19T09_03_26.047Z.txt: Load time duration: 26.93 min.'
+]
+Urls which are broken or not loaded:
+[
+  { url: 'https://gooutdoors.co.uk/', loadTime: null },
+  { url: 'https://dyson.fr/', loadTime: null },
+  { url: 'https://velvet.hu/', loadTime: null }
+]
+
+Without Ghostery
+Average urls load time:
+{
+  'https://kingoloto.com/subscribe.html': 3554.889,
+  'https://bananalotto.fr/subscribe.html': 2802.333,
+  'https://satkurier.pl/': 5574.571,
+  'https://www.wexphotovideo.com/': 3290.778,
+  'https://velvet.hu/': null
+}
+Duration of urls load time for specific set of measurements:
+[
+  'Set withoutGhostery/2023-05-18T18_17_18.025Z.txt: Load time duration: 40.98 min.',
+  'Set withoutGhostery/2023-05-19T01_10_12.807Z.txt: Load time duration: 41.84 min.'
+]
+Urls which are broken or not loaded:
+[
+  { url: 'https://agar.io/', loadTime: -1684434429752 },
+  { url: 'https://velvet.hu/', loadTime: null }
+]
+```
+
+## Resolve redirect
+
+Many website will redirect when their TLD is entered to the browser. We use simple curl script to resolve most of the redirects:
+
+```sh
+cat domains.txt | ./resolve-links.sh
+```

index.js

@@ -0,0 +1,156 @@
+import { Builder, Browser, By, until } from 'selenium-webdriver';
+import firefox from 'selenium-webdriver/firefox.js';
+import fs from 'fs';
+import { downloadAddon, sleep, switchToWindowWithUrl } from './helpers.js';
+
+const timestamp = new Date().toISOString();
+const isGhosteryEnabled = Boolean(
+  process.argv.find((arg) => arg === '--with-ghostery'),
+);
+const isRegionEU = Boolean(process.argv.find((arg) => arg === '--EU'));
+const isRegionUS = Boolean(process.argv.find((arg) => arg === '--US'));
+
+let region = 'GLOBAL';
+let outputPath = 'output/time';
+let n = 1;
+
+if (isRegionEU && isRegionUS) {
+  throw new Error('Cannot use more than one region at the same time.');
+} else if (isRegionUS) {
+  region = 'US';
+} else if (isRegionEU) {
+  region = 'EU';
+}
+
+const urls = fs
+  .readFileSync(`input/${region}/urls.txt`, { encoding: 'utf8' })
+  .split(/\r?\n/)
+  .map((l) => `${l}`);
+
+const addonUUID = 'd56a5b99-51b6-4e83-ab23-796216679614';
+
+const options = new firefox.Options();
+options.setPreference(
+  'extensions.webextensions.uuids',
+  `{"firefox@ghostery.com": "${addonUUID}"}`,
+);
+// options.addArguments("--headless");
+
+if (isGhosteryEnabled) {
+  options.addArguments('-profile', 'profiles/withGhostery');
+  outputPath += '/withGhostery';
+} else {
+  options.addArguments('-profile', 'profiles/withoutGhostery');
+  outputPath += '/withoutGhostery';
+}
+
+const outputStream = fs.createWriteStream(`${outputPath}/${timestamp}.txt`);
+
+const driver = await new Builder()
+  .forBrowser(Browser.FIREFOX)
+  .setFirefoxOptions(options)
+  .build();
+
+driver.manage().setTimeouts({ pageLoad: 20000 });
+
+console.log = function (msg) {
+  outputStream.write(`${msg}\n`);
+};
+
+const logPageLoadTime = async (n, url, now) => {
+  const navigationStart = await driver.executeScript(
+    'return window.performance.timing.navigationStart',
+  );
+  const domComplete = await driver.executeScript(
+    'return window.performance.timing.domComplete',
+  );
+  const totalTime = domComplete - navigationStart;
+  console.log(
+    `LOG=${JSON.stringify({
+      index: n,
+      url,
+      loadTime: totalTime,
+      loadedAt: now,
+    })}`,
+  );
+};
+
+try {
+  if (isGhosteryEnabled) {
+    const addon = await downloadAddon(
+      'https://github.com/ghostery/ghostery-extension/releases/download/v8.9.15/ghostery-firefox-v8.9.15.zip',
+    );
+
+    await driver.installAddon(addon, true);
+
+    if (!fs.existsSync('profiles/withGhostery/onboarded')) {
+      await driver.wait(
+        async () => (await driver.getAllWindowHandles()).length === 2,
+      );
+      console.log('INFO: Ghostery onboarding opened.');
+
+      await switchToWindowWithUrl(
+        driver,
+        `moz-extension://${addonUUID}/app/templates/onboarding.html`,
+      );
+      await (
+        await driver.wait(until.elementLocated(By.css('ui-button button')))
+      ).click();
+      await driver.wait(
+        until.elementLocated(
+          By.css('ui-onboarding-outro-success-view section'),
+        ),
+      );
+      console.log('INFO: Ghostery onboarding completed.');
+
+      await driver.get(
+        `moz-extension://${addonUUID}/app/templates/autoconsent.html`,
+      );
+      await driver
+        .wait(until.elementLocated(By.css('input[type=radio]:not(:checked)')))
+        .click();
+      await driver
+        .wait(
+          until.elementLocated(
+            By.css('ui-autoconsent-views-home ui-button[type=primary]'),
+          ),
+        )
+        .click();
+      await driver
+        .wait(
+          until.elementLocated(
+            By.css('ui-autoconsent-views-confirm ui-button[type=primary]'),
+          ),
+        )
+        .click();
+      console.log('INFO: Never-Consent enabled for all pages.');
+
+      fs.writeFileSync('profiles/withGhostery/onboarded', '');
+    }
+  }
+
+  console.log(`INFO: Open websites from for region: ${region}.`);
+
+  if (isGhosteryEnabled) {
+    // Wait for Ghostery extension to download fresh Ad-blocking filters
+    await sleep(1000 * 20);
+  }
+
+  for (const url of urls) {
+    const now = new Date().toISOString();
+    try {
+      await driver.get(url);
+      logPageLoadTime(n, url, now);
+
+      await sleep(1000 * 2);
+    } catch (error) {
+      console.error(`LOG=${JSON.stringify({ index: n, url })}`);
+      console.error(error);
+    }
+
+    n++;
+  }
+} finally {
+  await driver.quit();
+  outputStream.close();
+}