More DoubleClick adapters (#68)

* Fixes #66: Adapter for DoubleClick getconfig/pubsetting

* Towards #67: Stub adapter for googleads.g.doubleclick.net/pagead/interaction

research-docs/google/IDE.md

@@ -0,0 +1,3 @@
+According to the [cookie section](https://policies.google.com/technologies/cookies?hl=en-US) of Google's privacy policy ([archived](https://web.archive.org/web/20240425205614/https://policies.google.com/technologies/cookies?hl=en-US)), the `IDE` cookie is “used to show Google ads on non-Google sites” and “to personalize the ads [users] see”. It lasts for 13 months.
+
+According to the [cookie information for Google's ad products](https://business.safety.google/adscookies/) ([archived](https://web.archive.org/web/20240428070401/https://business.safety.google/adscookies/)), it is stored on the `doubleclick.net` domain and used by the following products: Campaign Manager, Display & Video 360, Google Ad Manager, Google Analytics, Search Ads 360

research-docs/google/vnm.md

@@ -0,0 +1,45 @@
+The `vnm` property in the query string always matches the app's version for the requests in our database.
+
+On iOS, it holds the human-readable version. A few samples:
+
+* [`vnm=2.17.0`](https://data.tweasel.org/data/requests/informed-consent,428853) comes from `com.elecont.EWClock@2.17`
+* [`vnm=22.7.2`](https://data.tweasel.org/data/requests/monkey-july-2023,22973) comes from `com.picsart.studio@22.7.2`
+* [`vnm=5.85.2`](https://data.tweasel.org/data/requests/monkey-july-2023,9490) comes from `com.cyberlink.youperfect@5.85.2`
+* [`vnm=1.4.8`](https://data.tweasel.org/data/requests/monkey-july-2023,8708) comes from `com.crazymaplestudio.aurora@1.4.08`
+* [`vnm=1.0.2`](https://data.tweasel.org/data/requests/informed-consent,120876) comes from `com.smmservice.videoforairplay@1.0.2`
+
+On Android, it holds the version code. A few samples:
+
+* [`vnm=140`](https://data.tweasel.org/data/requests/worrying-confessions,32750) comes from `com.translate.talkingtranslator@2.3.9`:
+
+  ```sh
+  ❯ aapt dump badging com.translate.talkingtranslator.apk
+  package: name='com.translate.talkingtranslator' versionCode='140' versionName='2.3.9' compileSdkVersion='31' compileSdkVersionCodename='12'
+  ```
+* [`vnm=43`](https://data.tweasel.org/data/requests/worrying-confessions,44179) comes from `com.bravo.booster@1.3.0.1001`:
+
+  ```sh
+  ❯ aapt dump badging com.bravo.booster.apk
+  package: name='com.bravo.booster' versionCode='43' versionName='1.3.0.1001' compileSdkVersion='31' compileSdkVersionCodename='12'
+  ```
+* [`vnm=40`](https://data.tweasel.org/data/requests/informed-consent,185851) comes from `com.paradyme.citysmash@1.40`:
+
+  ```sh
+  ❯ aapt dump badging com.paradyme.citysmash.apk
+  package: name='com.paradyme.citysmash' versionCode='40' versionName='1.40' compileSdkVersion='30' compileSdkVersionCodename='11'
+  ```
+* [`vnm=2736704`](https://data.tweasel.org/data/requests/informed-consent,358831) comes from `com.outdooractive.Outdooractive@3.9.0`:
+
+  ```sh
+  ❯ aapt dump badging com.outdooractive.Outdooractive.apk
+  package: name='com.outdooractive.Outdooractive' versionCode='2736704' versionName='3.9.0' compileSdkVersion='31' compileSdkVersionCodename='12'
+  ```
+* [`vnm=2070010`](https://data.tweasel.org/data/requests/informed-consent,571951) comes from `com.easybrain.block.puzzle.games@2.7.0`:
+
+  ```sh
+  ❯ aapt dump badging com.easybrain.block.puzzle.games/com.easybrain.block.puzzle.games.apk
+  package: name='com.easybrain.block.puzzle.games' versionCode='2070010' versionName='2.7.0' compileSdkVersion='31' compileSdkVersionCodename='12'
+  ```
+
+
+It's also likely that "vnm" is an acronym for "version number".

src/adapters/google.ts

@@ -96,6 +96,12 @@ const containedDataPathsDoubleclickMadsGma = (context: Context): Adapter['contai
         path: 'dtsdk',
         reasoning: 'obvious property name',
     },
+
+    otherIdentifiers: {
+        context: 'cookie',
+        path: 'IDE',
+        reasoning: 'google/IDE.md',
+    },
 });
 
 export const adapters: Adapter[] = [
@@ -248,7 +254,10 @@ export const adapters: Adapter[] = [
         endpointUrls: ['https://googleads.g.doubleclick.net/mads/gma'],
         match: (r) => !!r.content,
 
-        decodingSteps: [{ function: 'parseQueryString', input: 'body', output: 'res.body' }],
+        decodingSteps: [
+            { function: 'parseQueryString', input: 'body', output: 'res.body' },
+            { function: 'getProperty', input: 'cookie', output: 'res.cookie.IDE', options: { path: 'IDE' } },
+        ],
         containedDataPaths: containedDataPathsDoubleclickMadsGma('body'),
     },
 
@@ -259,8 +268,64 @@ export const adapters: Adapter[] = [
 
         endpointUrls: ['https://googleads.g.doubleclick.net/mads/gma'],
 
-        decodingSteps: [{ function: 'parseQueryString', input: 'query', output: 'res.query' }],
-        containedDataPaths: containedDataPathsDoubleclickMadsGma('query'),
+        decodingSteps: [
+            { function: 'parseQueryString', input: 'query', output: 'res.query' },
+            { function: 'getProperty', input: 'cookie', output: 'res.cookie.IDE', options: { path: 'IDE' } },
+        ],
+        containedDataPaths: {
+            ...containedDataPathsDoubleclickMadsGma('query'),
+
+            appVersion: {
+                context: 'query',
+                path: 'vnm',
+                reasoning: 'google/vnm.md',
+            },
+        },
+    },
+
+    {
+        slug: 'doubleclick-getconfig-pubsetting',
+        name: 'DoubleClick (getconfig/pubsetting)',
+        tracker,
+
+        endpointUrls: ['https://googleads.g.doubleclick.net/getconfig/pubsetting'],
+
+        decodingSteps: [
+            { function: 'parseQueryString', input: 'query', output: 'res.query' },
+            {
+                function: 'getProperty',
+                input: 'header',
+                output: 'res.header.x-requested-with',
+                options: { path: 'x-requested-with' },
+            },
+            { function: 'getProperty', input: 'cookie', output: 'res.cookie.IDE', options: { path: 'IDE' } },
+        ],
+        containedDataPaths: {
+            appId: [
+                {
+                    context: 'query',
+                    path: 'app_name',
+                    reasoning: 'obvious observed values',
+                },
+                {
+                    context: 'header',
+                    path: 'x-requested-with',
+                    reasoning: 'obvious observed values',
+                },
+            ],
+
+            appVersion: {
+                context: 'query',
+                path: 'vnm',
+                reasoning: 'google/vnm.md',
+            },
+
+            otherIdentifiers: {
+                context: 'cookie',
+                path: 'IDE',
+                reasoning: 'google/IDE.md',
+            },
+        },
     },
 
     {
@@ -353,4 +418,36 @@ export const adapters: Adapter[] = [
             },
         },
     },
+
+    {
+        slug: 'doubleclick-pagead-interaction',
+        name: 'DoubleClick (pagead/interaction)',
+        tracker,
+
+        endpointUrls: ['https://googleads.g.doubleclick.net/pagead/interaction/'],
+
+        decodingSteps: [
+            { function: 'parseQueryString', input: 'query', output: 'res.query' },
+            { function: 'getProperty', input: 'cookie', output: 'res.cookie.IDE', options: { path: 'IDE' } },
+            {
+                function: 'getProperty',
+                input: 'header',
+                output: 'res.header.x-requested-with',
+                options: { path: 'x-requested-with' },
+            },
+        ],
+        containedDataPaths: {
+            appId: {
+                context: 'header',
+                path: 'x-requested-with',
+                reasoning: 'obvious observed values',
+            },
+
+            otherIdentifiers: {
+                context: 'cookie',
+                path: 'IDE',
+                reasoning: 'google/IDE.md',
+            },
+        },
+    },
 ];