Merge pull request #46 from guardian/users-refresher-lambda

users-refresher-lambda

artifact.json

@@ -16,6 +16,11 @@
       "action": "pinboard-workflow-bridge-lambda",
       "path": "workflow-bridge-lambda/dist",
       "compress": "zip"
+    },
+    {
+      "action": "pinboard-users-refresher-lambda",
+      "path": "users-refresher-lambda/dist",
+      "compress": "zip"
     }
   ]
 }

bootstrapping-lambda/src/appSyncLookup.ts

@@ -1,6 +1,6 @@
 import * as AWS from "aws-sdk";
 import { AppSyncConfig } from "../../shared/AppSyncConfig";
-import { STAGE, standardAwsConfig } from "./awsIntegration";
+import { STAGE, standardAwsConfig } from "../../shared/awsIntegration";
 
 const ONE_HOUR_IN_SECONDS = 60 * 60;
 const TWENTY_FIVE_HOURS_IN_SECONDS = 25 * ONE_HOUR_IN_SECONDS;

bootstrapping-lambda/src/panDomainAuth.ts

@@ -4,23 +4,16 @@ import {
   guardianValidation,
 } from "@guardian/pan-domain-node";
 import { AWS_REGION } from "../../shared/awsRegion";
-
-const pandaKeyFilename = (function () {
-  // TODO consider doing this via Stage tag OR injecting this value directly as env variable
-  const functionName = process.env.AWS_LAMBDA_FUNCTION_NAME;
-  if (functionName?.includes("PROD")) {
-    return "gutools.co.uk.settings.public";
-  } else if (functionName?.includes("CODE")) {
-    return "code.dev-gutools.co.uk.settings.public";
-  }
-  return "local.dev-gutools.co.uk.settings.public";
-})();
+import {
+  pandaPublicConfigFilename,
+  pandaSettingsBucketName,
+} from "../../shared/panda";
 
 const panda = new PanDomainAuthentication(
   "gutoolsAuth-assym", // cookie name
   AWS_REGION, // AWS region
-  "pan-domain-auth-settings", // Settings bucket
-  pandaKeyFilename, // Settings file
+  pandaSettingsBucketName, // Settings bucket
+  pandaPublicConfigFilename, // Settings file
   guardianValidation
 );
 

bootstrapping-lambda/src/permissionCheck.ts

@@ -1,40 +1,11 @@
 import * as AWS from "aws-sdk";
-import { STAGE, standardAwsConfig } from "./awsIntegration";
+import { standardAwsConfig } from "../../shared/awsIntegration";
+import { getPinboardPermissionOverrides } from "../../shared/permissions";
 
-const s3Client = new AWS.S3(standardAwsConfig);
-
-interface Permission {
-  permission: {
-    name: string;
-    app: string;
-  };
-  overrides: Array<{
-    userId: string;
-    active: boolean;
-  }>;
-}
+const S3 = new AWS.S3(standardAwsConfig);
 
 export const userHasPermission = (userEmail: string): Promise<boolean> =>
-  s3Client
-    .getObject({
-      Bucket: "permissions-cache",
-      Key: `${STAGE}/permissions.json`,
-    })
-    .promise()
-    .then(({ Body }) => {
-      if (!Body) {
-        throw Error("could not read permissions");
-      }
-
-      const permissions: Permission[] = JSON.parse(Body.toString());
-
-      // see https://github.com/guardian/permissions/pull/128
-      return !!permissions.find(
-        (_) =>
-          _.permission.app === "pinboard" &&
-          _.permission.name === "pinboard" &&
-          _.overrides.find(
-            (override) => override.userId === userEmail && override.active
-          )
-      );
-    });
+  getPinboardPermissionOverrides(S3).then(
+    (overrides) =>
+      !!overrides?.find(({ userId, active }) => userId === userEmail && active)
+  );

cdk/package.json

@@ -9,16 +9,19 @@
   },
   "devDependencies": {
     "@types/node": "^14.14.7",
-    "aws-cdk": "^1.89.0",
+    "aws-cdk": "^1.95.1",
     "ts-node": "^9.0.0"
   },
   "dependencies": {
-    "@aws-cdk/aws-apigateway": "^1.89.0",
-    "@aws-cdk/aws-appsync": "^1.89.0",
-    "@aws-cdk/aws-dynamodb": "^1.89.0",
-    "@aws-cdk/aws-iam": "^1.89.0",
-    "@aws-cdk/aws-lambda": "^1.89.0",
-    "@aws-cdk/aws-s3": "^1.89.0",
-    "@aws-cdk/core": "^1.89.0"
+    "@aws-cdk/aws-apigateway": "^1.95.1",
+    "@aws-cdk/aws-appsync": "^1.95.1",
+    "@aws-cdk/aws-dynamodb": "^1.95.1",
+    "@aws-cdk/aws-iam": "^1.95.1",
+    "@aws-cdk/aws-lambda": "^1.95.1",
+    "@aws-cdk/aws-s3": "^1.95.1",
+    "@aws-cdk/aws-ec2": "^1.95.1",
+    "@aws-cdk/aws-events": "^1.95.1",
+    "@aws-cdk/aws-events-targets": "^1.95.1",
+    "@aws-cdk/core": "^1.95.1"
   }
 }

cdk/stack.ts

@@ -17,8 +17,11 @@ import * as appsync from "@aws-cdk/aws-appsync";
 import * as db from "@aws-cdk/aws-dynamodb";
 import * as acm from "@aws-cdk/aws-certificatemanager";
 import * as ec2 from "@aws-cdk/aws-ec2";
+import * as events from "@aws-cdk/aws-events";
+import * as eventsTargets from "@aws-cdk/aws-events-targets";
 import { join } from "path";
 import { AWS_REGION } from "../shared/awsRegion";
+import { userTableTTLAttribute } from "../shared/constants";
 
 export class PinBoardStack extends Stack {
   constructor(scope: Construct, id: string, props?: StackProps) {
@@ -134,11 +137,11 @@ export class PinBoardStack extends Stack {
       }
     );
 
-    const pinboardItemTableName = "pinboard-item-table";
+    const pinboardItemTableBaseName = "pinboard-item-table";
 
     const pinboardAppsyncItemTable = new db.Table(
       thisStack,
-      pinboardItemTableName,
+      pinboardItemTableBaseName,
       {
         billingMode: db.BillingMode.PAY_PER_REQUEST,
         partitionKey: {
@@ -153,17 +156,18 @@ export class PinBoardStack extends Stack {
       }
     );
 
-    const pinboardUserTableName = "pinboard-user-table";
+    const pinboardUserTableBaseName = "pinboard-user-table";
 
     const pinboardAppsyncUserTable = new db.Table(
       thisStack,
-      pinboardUserTableName,
+      pinboardUserTableBaseName,
       {
         billingMode: db.BillingMode.PAY_PER_REQUEST,
         partitionKey: {
           name: "email",
           type: db.AttributeType.STRING,
         },
+        timeToLiveAttribute: userTableTTLAttribute,
         encryption: db.TableEncryption.DEFAULT,
       }
     );
@@ -177,15 +181,15 @@ export class PinBoardStack extends Stack {
     );
 
     const pinboardItemDataSource = pinboardAppsyncApi.addDynamoDbDataSource(
-      `${pinboardItemTableName
+      `${pinboardItemTableBaseName
         .replace("pinboard-", "")
         .split("-")
         .join("_")}_datasource`,
       pinboardAppsyncItemTable
     );
 
     const pinboardUserDataSource = pinboardAppsyncApi.addDynamoDbDataSource(
-      `${pinboardUserTableName
+      `${pinboardUserTableBaseName
         .replace("pinboard-", "")
         .split("-")
         .join("_")}_datasource`,
@@ -262,20 +266,66 @@ export class PinBoardStack extends Stack {
       responseMappingTemplate: appsync.MappingTemplate.dynamoDbResultItem(),
     });
 
-    // this allows the lambda to query/create AppSync config/secrets
-    const bootstrappingLambdaAppSyncPolicyStatement = new iam.PolicyStatement({
+    const permissionsFilePolicyStatement = new iam.PolicyStatement({
       effect: iam.Effect.ALLOW,
-      actions: ["appsync:*"],
-      resources: ["arn:aws:appsync:eu-west-1:*"], //TODO tighten up if possible
+      actions: ["s3:GetObject"],
+      resources: [`arn:aws:s3:::permissions-cache/${STAGE}/*`],
     });
 
-    const bootstrappingLambdaPermissionsFilePolicyStatement = new iam.PolicyStatement(
+    const pandaConfigAndKeyPolicyStatement = new iam.PolicyStatement({
+      effect: iam.Effect.ALLOW,
+      actions: ["s3:GetObject"],
+      resources: [`arn:aws:s3:::pan-domain-auth-settings/*`],
+    });
+
+    const usersRefresherLambdaBasename = "pinboard-users-refresher-lambda";
+
+    const usersRefresherLambdaFunction = new lambda.Function(
+      thisStack,
+      usersRefresherLambdaBasename,
       {
-        effect: iam.Effect.ALLOW,
-        actions: ["s3:GetObject"],
-        resources: [`arn:aws:s3:::permissions-cache/${STAGE}/*`],
+        runtime: LAMBDA_NODE_VERSION,
+        memorySize: 128,
+        timeout: Duration.minutes(15),
+        handler: "index.handler",
+        environment: {
+          STAGE,
+          STACK,
+          APP,
+          USERS_TABLE_NAME: pinboardAppsyncUserTable.tableName,
+        },
+        functionName: `${usersRefresherLambdaBasename}-${STAGE}`,
+        code: lambda.Code.fromBucket(
+          deployBucket,
+          `${STACK}/${STAGE}/${usersRefresherLambdaBasename}/${usersRefresherLambdaBasename}.zip`
+        ),
+        initialPolicy: [
+          permissionsFilePolicyStatement,
+          pandaConfigAndKeyPolicyStatement,
+        ],
       }
     );
+    pinboardAppsyncUserTable.grantReadWriteData(usersRefresherLambdaFunction);
+
+    const usersRefresherLambdaSchedule = new events.Rule(
+      thisStack,
+      `${usersRefresherLambdaBasename}-schedule`,
+      {
+        description: `Runs the ${usersRefresherLambdaFunction.functionName} every 6 hours.`,
+        enabled: true,
+        targets: [
+          new eventsTargets.LambdaFunction(usersRefresherLambdaFunction),
+        ],
+        schedule: events.Schedule.rate(Duration.hours(6)),
+      }
+    );
+
+    // this allows the lambda to query/create AppSync config/secrets
+    const bootstrappingLambdaAppSyncPolicyStatement = new iam.PolicyStatement({
+      effect: iam.Effect.ALLOW,
+      actions: ["appsync:*"],
+      resources: ["arn:aws:appsync:eu-west-1:*"], //TODO tighten up if possible
+    });
 
     const bootstrappingLambdaBasename = "pinboard-bootstrapping-lambda";
     const bootstrappingLambdaApiBaseName = `${bootstrappingLambdaBasename}-api`;
@@ -300,7 +350,7 @@ export class PinBoardStack extends Stack {
         ),
         initialPolicy: [
           bootstrappingLambdaAppSyncPolicyStatement,
-          bootstrappingLambdaPermissionsFilePolicyStatement,
+          permissionsFilePolicyStatement,
         ],
       }
     );

package.json

@@ -7,7 +7,8 @@
     "bootstrapping-lambda",
     "cdk",
     "client",
-    "workflow-bridge-lambda"
+    "workflow-bridge-lambda",
+    "users-refresher-lambda"
   ],
   "scripts": {
     "graphql-refresh": "graphql-codegen --config graphql-refresh.yml",

riff-raff.yaml

@@ -27,3 +27,11 @@ deployments:
       bucket: workflow-dist
       fileName: pinboard-workflow-bridge-lambda.zip
       functionNames: [pinboard-workflow-bridge-lambda-]
+
+  pinboard-users-refresher-lambda:
+    type: aws-lambda
+    parameters:
+      prefixStack: false
+      bucket: workflow-dist
+      fileName: pinboard-users-refresher-lambda.zip
+      functionNames: [pinboard-users-refresher-lambda-]

scripts/ci.sh

@@ -35,6 +35,9 @@ yarn --cwd 'client' build
 # build workflow-bridge-lambda into a single file
 yarn --cwd 'workflow-bridge-lambda' build
 
+# build users-refresher-lambda into a single file
+yarn --cwd 'users-refresher-lambda' build
+
 # create a top level dist directory and copy in the built stuff
 mkdir -p dist/client
 cp bootstrapping-lambda/dist/index.js dist/

bootstrapping-lambda/src/awsIntegration.ts → shared/awsIntegration.ts

@@ -1,5 +1,5 @@
 import * as AWS from "aws-sdk";
-import { AWS_REGION } from "../../shared/awsRegion";
+import { AWS_REGION } from "./awsRegion";
 
 const PROFILE = "workflow";
 

shared/constants.ts

@@ -0,0 +1 @@
+export const userTableTTLAttribute = "ttlEpochSeconds";

shared/panda.ts

@@ -0,0 +1,14 @@
+const STAGE = process.env.STAGE || "LOCAL";
+
+const pandaConfigFilenameLookup: { [stage: string]: string } = {
+  PROD: "gutools.co.uk.settings",
+  CODE: "code.dev-gutools.co.uk.settings",
+  LOCAL: "local.dev-gutools.co.uk.settings",
+} as const;
+
+export const pandaSettingsBucketName = "pan-domain-auth-settings";
+
+export const pandaConfigFilename =
+  pandaConfigFilenameLookup[STAGE] || pandaConfigFilenameLookup["LOCAL"];
+
+export const pandaPublicConfigFilename = `${pandaConfigFilename}.public`;

shared/permissions.ts

@@ -0,0 +1,35 @@
+import { STAGE } from "./awsIntegration";
+import * as AWS from "aws-sdk";
+
+interface Override {
+  userId: string;
+  active: boolean;
+}
+
+interface Permission {
+  permission: {
+    name: string;
+    app: string;
+  };
+  overrides: Override[];
+}
+
+export const getPinboardPermissionOverrides = (S3: AWS.S3) =>
+  S3.getObject({
+    Bucket: "permissions-cache",
+    Key: `${STAGE}/permissions.json`,
+  })
+    .promise()
+    .then(({ Body }) => {
+      if (!Body) {
+        throw Error("could not read permissions");
+      }
+
+      const allPermissions = JSON.parse(Body.toString()) as Permission[];
+
+      return allPermissions.find(
+        ({ permission }) =>
+          // see https://github.com/guardian/permissions/pull/128
+          permission.app === "pinboard" && permission.name === "pinboard"
+      )?.overrides;
+    });

users-refresher-lambda/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "users-refresher-lambda",
+  "version": "1.0.0",
+  "description": "Lambda to keep the users Dynamo table(s) up to date (run on a schedule)",
+  "repository": "https://github.com/guardian/editorial-tools-pinboard",
+  "scripts": {
+    "watch": "ts-node-dev --respawn run.ts",
+    "build": "ncc build src/index.ts -o dist -m -e aws-sdk"
+  },
+  "devDependencies": {
+    "@types/iniparser": "^0.0.29",
+    "@types/node-forge": "^0.9.7",
+    "@vercel/ncc": "^0.25.1",
+    "aws-sdk": "^2.840.0",
+    "ts-node-dev": "^1.0.0"
+  },
+  "dependencies": {
+    "@googleapis/admin": "^0.2.0",
+    "@googleapis/people": "^0.2.1",
+    "iniparser": "^1.0.5",
+    "node-forge": "^0.10.0"
+  }
+}

users-refresher-lambda/run.ts

@@ -0,0 +1,3 @@
+import { handler } from "./src";
+
+handler().then(console.log).catch(console.error);