Merge pull request #919 from 18F/staging

[Bugfix] Top Downloads report updates -- Production
18F · Oct 23, 2024 · 7767459 · 7767459
2 parents 3b3d1db + 60a796e
commit 7767459
Show file tree

Hide file tree

Showing 7 changed files with 187 additions and 26 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,3 +1,5 @@
+name: CI workflow
+
 on:
   push:
   pull_request:

diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -1,3 +1,5 @@
+name: Shared deploy job
+
 on:
   workflow_call:
     inputs:

diff --git a/.github/workflows/manual_deploy_to_dev.yml b/.github/workflows/manual_deploy_to_dev.yml
@@ -0,0 +1,31 @@
+name: Deploy to dev manually
+
+on:
+  workflow_dispatch:
+
+jobs:
+  deploy_dev:
+    uses: 18F/analytics-reporter/.github/workflows/deploy.yml@develop
+    with:
+      ANALYTICS_GA4_CALL_RETRY_COUNT: ${{ vars.ANALYTICS_GA4_CALL_RETRY_COUNT_DEV }}
+      ANALYTICS_KEY_FILE_NAME: ${{ vars.ANALYTICS_KEY_FILE_NAME }}
+      ANALYTICS_LOG_LEVEL: ${{ vars.ANALYTICS_LOG_LEVEL_DEV }}
+      ANALYTICS_REPORT_EMAIL: ${{ vars.ANALYTICS_REPORT_EMAIL }}
+      APP_NAME: ${{ vars.APP_NAME_DEV }}
+      CF_ORGANIZATION_NAME: ${{ vars.CF_ORGANIZATION_NAME }}
+      CF_SPACE_NAME: ${{ vars.CF_SPACE_NAME_DEV }}
+      DB_SERVICE_NAME: ${{ vars.DB_SERVICE_NAME_DEV }}
+      MESSAGE_QUEUE_DATABASE_NAME: ${{ vars.MESSAGE_QUEUE_DATABASE_NAME }}
+      MESSAGE_QUEUE_NAME: ${{ vars.MESSAGE_QUEUE_NAME }}
+      NEW_RELIC_APP_NAME: ${{ vars.NEW_RELIC_APP_NAME_DEV }}
+      PROXY_FQDN: ${{ vars.PROXY_FQDN_DEV }}
+      PROXY_PORT: ${{ vars.PROXY_PORT }}
+      S3_SERVICE_NAME: ${{ vars.S3_SERVICE_NAME_DEV }}
+    secrets:
+      ANALYTICS_CREDENTIALS: ${{ secrets.ANALYTICS_CREDENTIALS }}
+      CF_USERNAME: ${{ secrets.CF_USERNAME_DEV }}
+      CF_PASSWORD: ${{ secrets.CF_PASSWORD_DEV }}
+      GA4_CREDS: ${{ secrets.GA4_CREDS_DEV }}
+      NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY_DEV }}
+      PROXY_USERNAME: ${{ secrets.PROXY_USERNAME_DEV }}
+      PROXY_PASSWORD: ${{ secrets.PROXY_PASSWORD_DEV }}
diff --git a/index.js b/index.js
@@ -143,6 +143,9 @@ async function runQueuePublish(options = {}) {
           ),
           {
             priority: _messagePriority(reportConfig),
+            retryLimit: 2,
+            retryDelay: 10,
+            retryBackoff: true,
             singletonKey: `${appConfig.scriptName}-${agency.agencyName}-${reportConfig.name}`,
           },
         );

diff --git a/manifest.consumer.yml b/manifest.consumer.yml
@@ -4,7 +4,7 @@ applications:
   # 1GB is needed right at startup because many child processes are spawned in
   # parallel. This could be reduced if we limited the number of simultaneous
   # processes.
-  memory: 384M
+  memory: 416M
   disk_quota: 512M
   health-check-type: process
   buildpacks:
@@ -26,7 +26,7 @@ applications:
     NEW_RELIC_APP_NAME: ${NEW_RELIC_APP_NAME}
     NEW_RELIC_LICENSE_KEY: ${NEW_RELIC_LICENSE_KEY}
     NODE_ENV: production
-    NODE_OPTIONS: --max-old-space-size=192
+    NODE_OPTIONS: --max-old-space-size=312
     PGSSLMODE: true
     PROXY_USERNAME: ${PROXY_USERNAME}
     PROXY_PASSWORD: ${PROXY_PASSWORD}

diff --git a/manifest.publisher.yml b/manifest.publisher.yml
@@ -4,7 +4,7 @@ applications:
   # 1GB is needed right at startup because many child processes are spawned in
   # parallel. This could be reduced if we limited the number of simultaneous
   # processes.
-  memory: 256M
+  memory: 192M
   disk_quota: 512M
   health-check-type: process
   buildpacks:

diff --git a/reports/usa.json b/reports/usa.json
@@ -788,7 +788,7 @@
       }
     },
     {
-      "name": "top-10000-pages-and-screens-30-days",
+      "name": "top-100000-pages-and-screens-30-days",
       "frequency": "daily",
       "query": {
         "dimensions": [
@@ -905,11 +905,11 @@
             ]
           }
         },
-        "limit": "10000"
+        "limit": "100000"
       },
       "meta": {
-        "name": "Top 10000 pages and screens people are viewing (30 days)",
-        "description": "The top 10000 pages, measured by active onsite users, for all sites."
+        "name": "Top 100000 pages and screens people are viewing (30 days)",
+        "description": "The top 100000 pages, measured by active onsite users, for all sites."
       }
     },
     {
@@ -1332,13 +1332,10 @@
             "name": "pageTitle"
           },
           {
-            "name": "eventName"
-          },
-          {
-            "name": "fileName"
+            "name": "fullPageUrl"
           },
           {
-            "name": "fullPageUrl"
+            "name": "linkUrl"
           }
         ],
         "metrics": [
@@ -1394,6 +1391,50 @@
                     }
                   }
                 }
+              },
+              {
+                "notExpression": {
+                  "filter": {
+                    "fieldName": "pageTitle",
+                    "stringFilter": {
+                      "value": "(other)",
+                      "caseSensitive": false
+                    }
+                  }
+                }
+              },
+              {
+                "notExpression": {
+                  "filter": {
+                    "fieldName": "pageTitle",
+                    "stringFilter": {
+                      "value": "null",
+                      "caseSensitive": false
+                    }
+                  }
+                }
+              },
+              {
+                "notExpression": {
+                  "filter": {
+                    "fieldName": "pageTitle",
+                    "stringFilter": {
+                      "value": "",
+                      "caseSensitive": false
+                    }
+                  }
+                }
+              },
+              {
+                "notExpression": {
+                  "filter": {
+                    "fieldName": "pageTitle",
+                    "stringFilter": {
+                      "value": "(not set)",
+                      "caseSensitive": false
+                    }
+                  }
+                }
               }
             ]
           }
@@ -1414,13 +1455,10 @@
             "name": "pageTitle"
           },
           {
-            "name": "eventName"
-          },
-          {
-            "name": "fileName"
+            "name": "fullPageUrl"
           },
           {
-            "name": "fullPageUrl"
+            "name": "linkUrl"
           }
         ],
         "metrics": [
@@ -1476,6 +1514,50 @@
                     }
                   }
                 }
+              },
+              {
+                "notExpression": {
+                  "filter": {
+                    "fieldName": "pageTitle",
+                    "stringFilter": {
+                      "value": "(other)",
+                      "caseSensitive": false
+                    }
+                  }
+                }
+              },
+              {
+                "notExpression": {
+                  "filter": {
+                    "fieldName": "pageTitle",
+                    "stringFilter": {
+                      "value": "null",
+                      "caseSensitive": false
+                    }
+                  }
+                }
+              },
+              {
+                "notExpression": {
+                  "filter": {
+                    "fieldName": "pageTitle",
+                    "stringFilter": {
+                      "value": "",
+                      "caseSensitive": false
+                    }
+                  }
+                }
+              },
+              {
+                "notExpression": {
+                  "filter": {
+                    "fieldName": "pageTitle",
+                    "stringFilter": {
+                      "value": "(not set)",
+                      "caseSensitive": false
+                    }
+                  }
+                }
               }
             ]
           }
@@ -1496,13 +1578,10 @@
             "name": "pageTitle"
           },
           {
-            "name": "eventName"
-          },
-          {
-            "name": "fileName"
+            "name": "fullPageUrl"
           },
           {
-            "name": "fullPageUrl"
+            "name": "linkUrl"
           }
         ],
         "metrics": [
@@ -1558,6 +1637,50 @@
                     }
                   }
                 }
+              },
+              {
+                "notExpression": {
+                  "filter": {
+                    "fieldName": "pageTitle",
+                    "stringFilter": {
+                      "value": "(other)",
+                      "caseSensitive": false
+                    }
+                  }
+                }
+              },
+              {
+                "notExpression": {
+                  "filter": {
+                    "fieldName": "pageTitle",
+                    "stringFilter": {
+                      "value": "null",
+                      "caseSensitive": false
+                    }
+                  }
+                }
+              },
+              {
+                "notExpression": {
+                  "filter": {
+                    "fieldName": "pageTitle",
+                    "stringFilter": {
+                      "value": "",
+                      "caseSensitive": false
+                    }
+                  }
+                }
+              },
+              {
+                "notExpression": {
+                  "filter": {
+                    "fieldName": "pageTitle",
+                    "stringFilter": {
+                      "value": "(not set)",
+                      "caseSensitive": false
+                    }
+                  }
+                }
               }
             ]
           }
@@ -1820,15 +1943,15 @@
             ]
           }
         },
-        "limit": "10000"
+        "limit": "100000"
       },
       "meta": {
         "name": "All Pages (Live)",
         "description": "Pages, measured by active onsite users, for all sites."
       }
     },
     {
-      "name": "top-10000-domains-30-days",
+      "name": "top-100000-domains-30-days",
       "frequency": "daily",
       "query": {
         "dimensions": [
@@ -1897,10 +2020,10 @@
             ]
           }
         },
-        "limit": "10000"
+        "limit": "100000"
       },
       "meta": {
-        "name": "Top 10000 Domains (30 Days)",
+        "name": "Top 100000 Domains (30 Days)",
         "description": "Last 30 days' domains, measured by page views and sessions, for the top 10000 sites."
       }
     },