diff --git a/docs/authorization_spec.md b/docs/authorization_spec.md index 2e7cb854..3c9702de 100644 --- a/docs/authorization_spec.md +++ b/docs/authorization_spec.md @@ -51,12 +51,12 @@ metadata: name: my-user Spec: type: js-expression - code: { "result": (data.jwt.sub === data.args.userId) ? "allow" : "deny"} + code: { "result": (input.jwt.sub === input.args.userId) ? "allow" : "deny"} args: userId: ID! ``` -The `args` are available to use on the data object +The `args` are available to use on the input object _Note the js-expression type is an example of a possible type and not planned to be implemented at this time._ @@ -72,13 +72,13 @@ Spec: code: | allow = false allow = { - data.args.userId == data.queries.familyQuery.family.members[_].id + input.args.userId == input.queries.familyQuery.family.members[_].id } args: userId: ID! queries: - type: graphql - paramName: familyQuery + name: familyQuery graphql: query: | { @@ -104,10 +104,10 @@ metadata: Spec: type: opa code: | - query = sprintf(“graphql { user(%s) {family { members { id} } } }”, data.jwt.sub) + query = sprintf(“graphql { user(%s) {family { members { id} } } }”, input.jwt.sub) allow = false allow = { - data.args.userId == data.query.family.members[_].id + input.args.userId == input.query.family.members[_].id } args: userId: ID! @@ -127,13 +127,13 @@ Spec: code: | allow = false allow = { - data.queries.myUserPolicy == true + input.queries.myUserPolicy == true } args: userId: ID! queries: - type: policy - paramName: myUserPolicy + name: myUserPolicy policy: policyName: my-user args: @@ -267,7 +267,7 @@ Spec: code: | allow = false allow { - data.jwt.claims[data.args.claims[i]] == data.args.values[i] + input.jwt.claims[input.args.claims[i]] == input.args.values[i] } args: claims: [String] diff --git a/services/package-lock.json b/services/package-lock.json index 7e0c7e2f..9de3ea5b 100644 --- a/services/package-lock.json +++ b/services/package-lock.json @@ -749,6 +749,11 @@ } } }, + "@open-policy-agent/opa-wasm": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@open-policy-agent/opa-wasm/-/opa-wasm-1.1.0.tgz", + "integrity": "sha512-ZWxOyyZC9NoSJALVZYcPK9hG9moGg3zwxAu93PCX1JdIBr2YujDeksEyhAT/KnK9PVt/rDJtkzVYXAO7JJbHnA==" + }, "@panva/asn1.js": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@panva/asn1.js/-/asn1.js-1.0.0.tgz", diff --git a/services/package.json b/services/package.json index d070a3c7..062f71a9 100644 --- a/services/package.json +++ b/services/package.json @@ -13,6 +13,7 @@ "license": "ISC", "dependencies": { "@apollo/federation": "^0.12.1", + "@open-policy-agent/opa-wasm": "^1.1.0", "apollo-datasource-rest": "^0.7.0", "apollo-link-context": "^1.0.19", "apollo-link-http": "^1.5.16", diff --git a/services/src/gateway.ts b/services/src/gateway.ts index bdfb74e6..17e6dce4 100644 --- a/services/src/gateway.ts +++ b/services/src/gateway.ts @@ -12,16 +12,21 @@ import { ResourceRepository, CompositeResourceRepository, } from './modules/resource-repository'; +import {PolicyExecutor} from './modules/directives/policy/policy-executor'; async function run() { logger.info('Stitch gateway booting up...'); + const resourceRepository = getResourceRepository(); + const {server, dispose} = createStitchGateway({ - resourceGroups: pollForUpdates(getResourceRepository(), config.resourceUpdateInterval), + resourceGroups: pollForUpdates(resourceRepository, config.resourceUpdateInterval), tracing: config.enableGraphQLTracing, playground: config.enableGraphQLPlayground, introspection: config.enableGraphQLIntrospection, }); + await resourceRepository.initializePolicyAttachments(); + PolicyExecutor.repo = resourceRepository; const app = fastify(); app.register(fastifyMetrics, {endpoint: '/metrics'}); diff --git a/services/src/modules/directives/index.ts b/services/src/modules/directives/index.ts index 9d6252b8..52b6668a 100644 --- a/services/src/modules/directives/index.ts +++ b/services/src/modules/directives/index.ts @@ -5,6 +5,7 @@ import {sdl as restSdl, RestDirective} from './rest'; import {sdl as gqlSdl, GqlDirective} from './gql'; import {sdl as exportSdl, ExportDirective} from './export'; import {sdl as selectSdl, SelectDirective} from './select'; +import {sdl as policySdl, PolicyDirective} from './policy/policy'; export const directiveMap: {[visitorName: string]: typeof SchemaDirectiveVisitor} = { stub: StubDirective, @@ -12,6 +13,7 @@ export const directiveMap: {[visitorName: string]: typeof SchemaDirectiveVisitor gql: GqlDirective, export: ExportDirective, select: SelectDirective, + policy: PolicyDirective, }; -export const sdl = concatAST([stubSdl, restSdl, gqlSdl, exportSdl, selectSdl]); +export const sdl = concatAST([stubSdl, restSdl, gqlSdl, exportSdl, selectSdl, policySdl]); diff --git a/services/src/modules/directives/policy/opa.ts b/services/src/modules/directives/policy/opa.ts new file mode 100644 index 00000000..ab51d17f --- /dev/null +++ b/services/src/modules/directives/policy/opa.ts @@ -0,0 +1,38 @@ +// @ts-ignore opa-wasm already has TS typings merged, but not yet published on npm +import * as Rego from '@open-policy-agent/opa-wasm'; +import {getCompiledFilename} from '../../opaHelper'; +import {PolicyExecutionContext, PolicyExecutionResult, QueriesResults, JwtInput} from './types'; +import {PolicyArgsObject} from '../../resource-repository'; + +export async function evaluate(ctx: PolicyExecutionContext): Promise { + const policy = await getWasmPolicy(ctx); + const input = getInput(ctx); + + const result = policy.evaluate(input)?.[0]?.result; + + return {done: true, allow: result?.allow}; +} + +async function getWasmPolicy(ctx: PolicyExecutionContext): Promise { + const filename = getCompiledFilename({namespace: ctx.namespace, name: ctx.name}); + const wasm = ctx.repo.getPolicyAttachment(filename); + + const rego = new Rego(); + return rego.load_policy(wasm); +} + +function getInput(ctx: PolicyExecutionContext): PolicyInput { + const input: PolicyInput = {}; + + if (ctx.jwt) input.jwt = ctx.jwt; + if (ctx.args) input.args = ctx.args; + if (ctx.queries) input.queries = ctx.queries; + + return input; +} + +type PolicyInput = { + jwt?: JwtInput; + args?: PolicyArgsObject; + queries?: QueriesResults; +}; diff --git a/services/src/modules/directives/policy/policy-executor.ts b/services/src/modules/directives/policy/policy-executor.ts new file mode 100644 index 00000000..9ab559b5 --- /dev/null +++ b/services/src/modules/directives/policy/policy-executor.ts @@ -0,0 +1,72 @@ +import {GraphQLResolveInfo} from 'graphql'; +import {RequestContext} from '../../context'; +import {Policy, GraphQLArguments} from './types'; +import {ResourceRepository, Policy as PolicyDefinition, PolicyArgsObject} from '../../resource-repository'; +import {evaluate as evaluateOpa} from './opa'; +import {injectParameters} from '../../paramInjection'; + +const typeEvaluators = { + opa: evaluateOpa, +}; + +export class PolicyExecutor { + static repo: ResourceRepository; + private policyDefinitions: PolicyDefinition[]; + + constructor( + protected policies: Policy[], + protected parent: any, + protected args: GraphQLArguments, + protected context: RequestContext, + protected info: GraphQLResolveInfo + ) { + // TODO: add jwt data + this.policyDefinitions = PolicyExecutor.repo.getResourceGroup().policies; + } + + async validatePolicies() { + await Promise.all(this.policies.map(r => this.validatePolicy(r))); + } + + async validatePolicy(policy: Policy) { + const policyDefinition = this.getPolicyDefinition(policy.namespace, policy.name); + + const args = policyDefinition.args && this.preparePolicyArgs(policyDefinition.args, policy); + // TODO: evaluate queries + + const evaluate = typeEvaluators[policyDefinition.type]; + if (!evaluate) throw new Error(`Unsupported policy type ${policyDefinition.type}`); + + const {done, allow} = await evaluate({...policy, args, repo: PolicyExecutor.repo}); + if (!done) throw new Error('in-line query evaluation not yet supported'); + + if (!allow) throw new Error(`Unauthorized by policy ${policy.name} in namespace ${policy.namespace}`); + } + + private preparePolicyArgs(supportedPolicyArgs: PolicyArgsObject, policy: Policy): PolicyArgsObject { + return Object.keys(supportedPolicyArgs).reduce((policyArgs, policyArgName) => { + if (policy?.args?.[policyArgName] === undefined) + throw new Error( + `Missing arg ${policyArgName} for policy ${policy.name} in namespace ${policy.namespace}` + ); + + let policyArgValue = policy.args[policyArgName]; + if (typeof policyArgValue === 'string') { + policyArgValue = injectParameters(policyArgValue, this.parent, this.args, this.context, this.info) + .value; + } + + policyArgs[policyArgName] = policyArgValue; + return policyArgs; + }, {}); + } + + private getPolicyDefinition(namespace: string, name: string) { + const policyDefinition = this.policyDefinitions.find(({metadata}) => { + return metadata.namespace === namespace && metadata.name === name; + }); + + if (!policyDefinition) throw new Error(`The policy ${name} in namespace ${namespace} was not found`); + return policyDefinition; + } +} diff --git a/services/src/modules/directives/policy/policy.ts b/services/src/modules/directives/policy/policy.ts new file mode 100644 index 00000000..02cd3eb3 --- /dev/null +++ b/services/src/modules/directives/policy/policy.ts @@ -0,0 +1,30 @@ +import {GraphQLResolveInfo} from 'graphql'; +import {RequestContext} from '../../context'; +import {SchemaDirectiveVisitor} from 'graphql-tools'; +import {GraphQLField, defaultFieldResolver} from 'graphql'; +import {gql} from 'apollo-server-core'; +import {PolicyExecutor} from './policy-executor'; + +export class PolicyDirective extends SchemaDirectiveVisitor { + visitFieldDefinition(field: GraphQLField) { + const originalResolve = field.resolve || defaultFieldResolver; + const policies = this.args.policies; + + field.resolve = async (parent: any, args: any, context: RequestContext, info: GraphQLResolveInfo) => { + const executor = new PolicyExecutor(policies, parent, args, context, info); + await executor.validatePolicies(); + + return originalResolve.call(field, parent, args, context, info); + }; + } +} + +export const sdl = gql` + input PolicyDirectivePolicy { + namespace: String! + name: String! + args: JSONObject + } + + directive @policy(policies: [PolicyDirectivePolicy!]!) on FIELD_DEFINITION +`; diff --git a/services/src/modules/directives/policy/types.ts b/services/src/modules/directives/policy/types.ts new file mode 100644 index 00000000..e028565b --- /dev/null +++ b/services/src/modules/directives/policy/types.ts @@ -0,0 +1,38 @@ +import {PolicyArgsObject, ResourceRepository} from '../../resource-repository/types'; + +export type Policy = { + namespace: string; + name: string; + args?: PolicyArgsObject; +}; + +// args here contain the final values after param injection +export type PolicyExecutionContext = { + namespace: string; + name: string; + repo: ResourceRepository; + jwt?: JwtInput; + args?: PolicyArgsObject; + queries?: QueriesResults; +}; + +export type QueriesResults = { + [name: string]: string; +}; + +export type JwtInput = { + [name: string]: string; +}; + +export type PolicyExecutionResult = { + done: boolean; + allow?: boolean; + query?: { + type: string; + code: string; + }; +}; + +export type GraphQLArguments = { + [name: string]: any; +}; diff --git a/services/src/modules/resource-repository/composite.ts b/services/src/modules/resource-repository/composite.ts index cd817478..21b589b1 100644 --- a/services/src/modules/resource-repository/composite.ts +++ b/services/src/modules/resource-repository/composite.ts @@ -1,4 +1,4 @@ -import {ResourceRepository, FetchLatestResult} from './types'; +import {ResourceRepository, FetchLatestResult, ResourceGroup} from './types'; import {applyResourceGroupUpdates} from './updates'; export class CompositeResourceRepository implements ResourceRepository { @@ -13,6 +13,12 @@ export class CompositeResourceRepository implements ResourceRepository { })); } + getResourceGroup(): ResourceGroup { + const rgs = this.repositories.map(r => r.getResourceGroup()); + + return rgs.reduce((rg1, rg2) => applyResourceGroupUpdates(rg1, rg2)); + } + async update(): Promise { throw new Error('Multiplexed resource repository cannot handle updates'); } @@ -20,4 +26,17 @@ export class CompositeResourceRepository implements ResourceRepository { async writePolicyAttachment(): Promise { throw new Error('Multiplexed resource repository cannot handle updates'); } + + public getPolicyAttachment(filename: string): Buffer { + for (const repo of this.repositories) { + const policyAttachment = repo.getPolicyAttachment(filename); + if (policyAttachment) return policyAttachment; + } + + throw new Error(`Policy attachment with the filename ${filename} was not found`); + } + + public async initializePolicyAttachments() { + await Promise.all(this.repositories.map(repo => repo.initializePolicyAttachments())); + } } diff --git a/services/src/modules/resource-repository/fs.spec.ts b/services/src/modules/resource-repository/fs.spec.ts new file mode 100644 index 00000000..10de6a33 --- /dev/null +++ b/services/src/modules/resource-repository/fs.spec.ts @@ -0,0 +1,126 @@ +import {promises as fs} from 'fs'; +import * as path from 'path'; +import {FileSystemResourceRepository} from './fs'; + +let repo: FileSystemResourceRepository; +const pathToResourcesFile = '/var/stitch/resources.json'; +const policyAttachmentsFolderPath = '/var/stitch/policy-attachments'; +let fsMocks: jest.Mock[]; + +beforeEach(() => { + repo = new FileSystemResourceRepository(pathToResourcesFile, policyAttachmentsFolderPath); + fsMocks = []; +}); + +afterEach(() => { + for (const mock of fsMocks) (mock as any).__restore(); +}); + +describe('shouldRefreshPolicyAttachment', () => { + let shouldRefreshPolicyAttachment: Function; + + beforeEach(() => { + shouldRefreshPolicyAttachment = repo['shouldRefreshPolicyAttachment'].bind(repo); + }); + + it('returns true if the given file does not currently exist in memory', () => { + repo['policyAttachmentsRefreshedAt'] = new Date(); + + const result = shouldRefreshPolicyAttachment({filename: 'file', updatedAt: minutesAgo(5)}); + expect(result).toBe(true); + }); + + it('returns true if this is the first refresh for this process', () => { + repo['policyAttachments']['file'] = Buffer.from('content'); + + const result = shouldRefreshPolicyAttachment({filename: 'file', updatedAt: minutesAgo(5)}); + expect(result).toBe(true); + }); + + it('returns true if the attachment was last updated after the last refresh', () => { + repo['policyAttachments']['file'] = Buffer.from('content'); + repo['policyAttachmentsRefreshedAt'] = minutesAgo(5); + + const result = shouldRefreshPolicyAttachment({filename: 'file', updatedAt: new Date()}); + expect(result).toBe(true); + }); + + it('returns false if the attachment was last updated before the last refresh', () => { + repo['policyAttachments']['file'] = Buffer.from('content'); + repo['policyAttachmentsRefreshedAt'] = new Date(); + + const result = shouldRefreshPolicyAttachment({filename: 'file', updatedAt: minutesAgo(5)}); + expect(result).toBe(false); + }); +}); + +describe('refreshPolicyAttachments', () => { + let refreshPolicyAttachments: Function; + + beforeEach(() => { + refreshPolicyAttachments = repo['refreshPolicyAttachments'].bind(repo); + }); + + it('refreshes the local copy of all policy attachments that should be refreshed', async () => { + repo['policyAttachments'] = { + upToDateFile: Buffer.from('up to date'), + needsUpdating: Buffer.from('needs updating'), + }; + repo['policyAttachmentsRefreshedAt'] = minutesAgo(5); + + const upToDateFilePath = path.resolve(policyAttachmentsFolderPath, 'upToDateFile'); + const needsUpdatingPath = path.resolve(policyAttachmentsFolderPath, 'needsUpdating'); + const newFilePath = path.resolve(policyAttachmentsFolderPath, 'newFile'); + + const readdirMock = mockFsFunction('readdir'); + readdirMock.mockReturnValue(Promise.resolve(['upToDateFile', 'needsUpdating', 'newFile'])); + + const statParamBasedReturnValues = { + [upToDateFilePath]: {mtime: minutesAgo(10)}, + [needsUpdatingPath]: {mtime: minutesAgo(2)}, + [newFilePath]: {mtime: minutesAgo(3)}, + }; + const statMock = mockFsFunction('stat'); + statMock.mockImplementation(filePath => Promise.resolve(statParamBasedReturnValues[filePath])); + + const readFileParamBasedReturnValues = { + [needsUpdatingPath]: Buffer.from('needs updating - updated'), + [newFilePath]: Buffer.from('new file'), + }; + const readFileMock = mockFsFunction('readFile'); + readFileMock.mockImplementation(filePath => Promise.resolve(readFileParamBasedReturnValues[filePath])); + + await refreshPolicyAttachments(); + + expect(repo['policyAttachments']).toMatchObject({ + upToDateFile: Buffer.from('up to date'), + needsUpdating: Buffer.from('needs updating - updated'), + newFile: Buffer.from('new file'), + }); + expect(repo['policyAttachmentsRefreshedAt'].getTime()).toBeGreaterThan(minutesAgo(1).getTime()); + + expect(readdirMock).toHaveBeenCalledTimes(1); + expect(readdirMock).toHaveBeenCalledWith(policyAttachmentsFolderPath); + + expect(statMock).toHaveBeenCalledTimes(3); + expect(statMock).toHaveBeenCalledWith(upToDateFilePath); + expect(statMock).toHaveBeenCalledWith(needsUpdatingPath); + expect(statMock).toHaveBeenCalledWith(newFilePath); + + expect(readFileMock).toHaveBeenCalledTimes(2); + expect(readFileMock).toHaveBeenCalledWith(needsUpdatingPath); + expect(readFileMock).toHaveBeenCalledWith(newFilePath); + }); +}); + +function mockFsFunction(functionName: string): jest.Mock { + const realFunction = (fs as any)[functionName]; + const mock: any = jest.fn(); + mock.__restore = () => ((fs as any)[functionName] = realFunction); + (fs as any)[functionName] = mock; + + fsMocks.push(mock); + return mock; +} + +const minutesAgo = (m: number) => new Date(Date.now() - m * 60000); diff --git a/services/src/modules/resource-repository/fs.ts b/services/src/modules/resource-repository/fs.ts index 020b239b..37d15a30 100644 --- a/services/src/modules/resource-repository/fs.ts +++ b/services/src/modules/resource-repository/fs.ts @@ -2,11 +2,16 @@ import {ResourceGroup, ResourceRepository} from '.'; import * as envVar from 'env-var'; import {promises as fs} from 'fs'; import * as path from 'path'; +import pLimit from 'p-limit'; +import * as config from '../config'; +import logger from '../logger'; import {FetchLatestResult} from './types'; export class FileSystemResourceRepository implements ResourceRepository { protected current?: {mtime: number; rg: ResourceGroup}; protected policyAttachmentsDirInitialized = false; + protected policyAttachments: {[filename: string]: Buffer} = {}; + protected policyAttachmentsRefreshedAt?: Date; constructor(protected pathToFile: string, protected policyAttachmentsFolderPath: string) {} @@ -24,6 +29,10 @@ export class FileSystemResourceRepository implements ResourceRepository { return {isNew: true, resourceGroup: rg}; } + getResourceGroup(): ResourceGroup { + return this.current!.rg; + } + async update(rg: ResourceGroup): Promise { await fs.writeFile(this.pathToFile, JSON.stringify(rg)); } @@ -35,6 +44,85 @@ export class FileSystemResourceRepository implements ResourceRepository { await fs.writeFile(filePath, content); } + public getPolicyAttachment(filename: string): Buffer { + return this.policyAttachments[filename]; + } + + public async initializePolicyAttachments() { + try { + await this.refreshPolicyAttachments(); + } catch (err) { + logger.fatal({err}, 'Failed fetching fs policy attachments on startup'); + throw err; + } + + setInterval(async () => { + try { + await this.refreshPolicyAttachments(); + } catch (err) { + logger.error( + {err}, + `Failed refreshing fs policy attachments, last successful refresh was at ${this.policyAttachmentsRefreshedAt}` + ); + } + }, config.resourceUpdateInterval); + } + + private async refreshPolicyAttachments() { + const newRefreshedAt = new Date(); + + const allAttachments = await this.getPolicyAttachmentsList(); + const attachmentsToRefresh = allAttachments + .filter(a => this.shouldRefreshPolicyAttachment(a)) + .map(a => a.filename); + + if (attachmentsToRefresh.length > 0) { + const newAttachments = await this.getPolicyAttachments(attachmentsToRefresh); + newAttachments.forEach(a => (this.policyAttachments[a.filename] = a.content)); + } + + this.policyAttachmentsRefreshedAt = newRefreshedAt; + } + + private shouldRefreshPolicyAttachment({filename, updatedAt}: {filename: string; updatedAt: Date}) { + if (!this.policyAttachments[filename]) return true; + if (!this.policyAttachmentsRefreshedAt) return true; + + return updatedAt > this.policyAttachmentsRefreshedAt; + } + + private async getPolicyAttachmentsList(): Promise<{filename: string; updatedAt: Date}[]> { + const filenames = await fs.readdir(this.policyAttachmentsFolderPath); + const limit = pLimit(10); + + return Promise.all( + filenames.map(filename => { + return limit(async () => { + const filePath = path.resolve(this.policyAttachmentsFolderPath, filename); + const stats = await fs.stat(filePath); + + const updatedAt = stats.mtime; + return {filename, updatedAt}; + }); + }) + ); + } + + private async getPolicyAttachments(filenames: string[]): Promise<{filename: string; content: Buffer}[]> { + const limit = pLimit(10); + + return Promise.all( + filenames.map(filename => { + return limit(async () => { + const filePath = path.resolve(this.policyAttachmentsFolderPath, filename); + const content = await fs.readFile(filePath); + + return {filename, content}; + }); + }) + ); + } + private async initializePolicyAttachmentsDir() { if (this.policyAttachmentsDirInitialized) return; diff --git a/services/src/modules/resource-repository/s3.spec.ts b/services/src/modules/resource-repository/s3.spec.ts new file mode 100644 index 00000000..52c12556 --- /dev/null +++ b/services/src/modules/resource-repository/s3.spec.ts @@ -0,0 +1,181 @@ +import {S3ResourceRepository} from './s3'; + +let repo: S3ResourceRepository; +let s3Mock: any; +const policyAttachmentsKeyPrefix = 'policyAttachments/'; +const bucketName = 'bucket'; + +beforeEach(() => { + s3Mock = { + getObject: jest.fn(() => ({promise: () => Promise.resolve()})), + listObjectsV2: jest.fn(() => ({promise: () => Promise.resolve()})), + }; + + repo = new S3ResourceRepository({ + s3: s3Mock as any, + bucketName, + objectKey: 'key', + policyAttachmentsKeyPrefix, + }); +}); + +describe('shouldRefreshPolicyAttachment', () => { + let shouldRefreshPolicyAttachment: Function; + + beforeEach(() => { + shouldRefreshPolicyAttachment = repo['shouldRefreshPolicyAttachment'].bind(repo); + }); + + it('returns true if the given file does not currently exist in memory', () => { + repo['policyAttachmentsRefreshedAt'] = new Date(); + + const result = shouldRefreshPolicyAttachment({filename: 'file', updatedAt: minutesAgo(5)}); + expect(result).toBe(true); + }); + + it('returns true if this is the first refresh for this process', () => { + repo['policyAttachments']['file'] = Buffer.from('content'); + + const result = shouldRefreshPolicyAttachment({filename: 'file', updatedAt: minutesAgo(5)}); + expect(result).toBe(true); + }); + + it('returns true if the attachment was last updated after the last refresh', () => { + repo['policyAttachments']['file'] = Buffer.from('content'); + repo['policyAttachmentsRefreshedAt'] = minutesAgo(5); + + const result = shouldRefreshPolicyAttachment({filename: 'file', updatedAt: new Date()}); + expect(result).toBe(true); + }); + + it('returns false if the attachment was last updated before the last refresh', () => { + repo['policyAttachments']['file'] = Buffer.from('content'); + repo['policyAttachmentsRefreshedAt'] = new Date(); + + const result = shouldRefreshPolicyAttachment({filename: 'file', updatedAt: minutesAgo(5)}); + expect(result).toBe(false); + }); +}); + +describe('refreshPolicyAttachments', () => { + let refreshPolicyAttachments: Function; + + beforeEach(() => { + refreshPolicyAttachments = repo['refreshPolicyAttachments'].bind(repo); + }); + + it('refreshes the local copy of all policy attachments that should be refreshed', async () => { + repo['policyAttachments'] = { + upToDateFile: Buffer.from('up to date'), + needsUpdating: Buffer.from('needs updating'), + }; + repo['policyAttachmentsRefreshedAt'] = minutesAgo(5); + + s3Mock.listObjectsV2 = jest.fn().mockReturnValue({ + promise: () => + Promise.resolve({ + Contents: [ + {Key: `${policyAttachmentsKeyPrefix}upToDateFile`, LastModified: minutesAgo(10)}, + {Key: `${policyAttachmentsKeyPrefix}needsUpdating`, LastModified: minutesAgo(2)}, + {Key: `${policyAttachmentsKeyPrefix}newFile`, LastModified: minutesAgo(3)}, + ], + IsTruncated: false, + }), + }); + + const paramBasedReturnValues = { + [`${policyAttachmentsKeyPrefix}needsUpdating`]: {Body: Buffer.from('needs updating - updated')}, + [`${policyAttachmentsKeyPrefix}newFile`]: {Body: Buffer.from('new file')}, + }; + s3Mock.getObject = jest.fn(({Key}) => ({ + promise: () => { + return Promise.resolve(paramBasedReturnValues[Key]); + }, + })); + + await refreshPolicyAttachments(); + + expect(repo['policyAttachments']).toMatchObject({ + upToDateFile: Buffer.from('up to date'), + needsUpdating: Buffer.from('needs updating - updated'), + newFile: Buffer.from('new file'), + }); + expect(repo['policyAttachmentsRefreshedAt'].getTime()).toBeGreaterThan(minutesAgo(1).getTime()); + + expect(s3Mock.listObjectsV2).toHaveBeenCalledTimes(1); + expect(s3Mock.listObjectsV2).toHaveBeenCalledWith({ + Bucket: bucketName, + MaxKeys: 1000, + Prefix: policyAttachmentsKeyPrefix, + }); + + expect(s3Mock.getObject).toHaveBeenCalledTimes(2); + expect(s3Mock.getObject).toHaveBeenCalledWith({ + Bucket: bucketName, + Key: `${policyAttachmentsKeyPrefix}needsUpdating`, + }); + expect(s3Mock.getObject).toHaveBeenCalledWith({ + Bucket: bucketName, + Key: `${policyAttachmentsKeyPrefix}newFile`, + }); + }); +}); + +describe('getPolicyAttachmentsList', () => { + let getPolicyAttachmentsList: Function; + + beforeEach(() => { + getPolicyAttachmentsList = repo['getPolicyAttachmentsList'].bind(repo); + }); + + it('correctly handles truncated results with a continuation token', async () => { + const expectedResult = [ + {filename: 'f1', updatedAt: minutesAgo(10)}, + {filename: 'f2', updatedAt: minutesAgo(7)}, + {filename: 'f3', updatedAt: minutesAgo(5)}, + ]; + + const listMock = jest + .fn() + .mockReturnValueOnce({ + promise: () => + Promise.resolve({ + Contents: [{Key: `${policyAttachmentsKeyPrefix}f1`, LastModified: expectedResult[0].updatedAt}], + IsTruncated: true, + ContinuationToken: 'ct1', + }), + }) + .mockReturnValueOnce({ + promise: () => + Promise.resolve({ + Contents: [{Key: `${policyAttachmentsKeyPrefix}f2`, LastModified: expectedResult[1].updatedAt}], + IsTruncated: true, + ContinuationToken: 'ct2', + }), + }) + .mockReturnValueOnce({ + promise: () => + Promise.resolve({ + Contents: [{Key: `${policyAttachmentsKeyPrefix}f3`, LastModified: expectedResult[2].updatedAt}], + IsTruncated: false, + }), + }); + s3Mock.listObjectsV2 = listMock; + + const result = await getPolicyAttachmentsList(); + + expect(result).toEqual(expectedResult); + + const listParams: any = { + Bucket: bucketName, + MaxKeys: 1000, + Prefix: policyAttachmentsKeyPrefix, + }; + expect(listMock).toHaveBeenCalledTimes(3); + expect(listMock).toHaveBeenNthCalledWith(1, listParams); + expect(listMock).toHaveBeenNthCalledWith(2, {...listParams, ContinuationToken: 'ct1'}); + expect(listMock).toHaveBeenNthCalledWith(3, {...listParams, ContinuationToken: 'ct2'}); + }); +}); + +const minutesAgo = (m: number) => new Date(Date.now() - m * 60000); diff --git a/services/src/modules/resource-repository/s3.ts b/services/src/modules/resource-repository/s3.ts index eefab919..297f8eb0 100644 --- a/services/src/modules/resource-repository/s3.ts +++ b/services/src/modules/resource-repository/s3.ts @@ -1,5 +1,8 @@ import * as AWS from 'aws-sdk'; import * as envVar from 'env-var'; +import pLimit from 'p-limit'; +import * as config from '../config'; +import logger from '../logger'; import {ResourceRepository, ResourceGroup, FetchLatestResult} from './types'; interface S3ResourceRepositoryConfig { @@ -8,8 +11,12 @@ interface S3ResourceRepositoryConfig { objectKey: string; policyAttachmentsKeyPrefix: string; } +type FileDetails = {filename: string; updatedAt: Date}; + export class S3ResourceRepository implements ResourceRepository { protected current?: {etag?: string; rg: ResourceGroup}; + protected policyAttachments: {[filename: string]: Buffer} = {}; + protected policyAttachmentsRefreshedAt?: Date; constructor(protected config: S3ResourceRepositoryConfig) {} @@ -47,6 +54,10 @@ export class S3ResourceRepository implements ResourceRepository { }; } + getResourceGroup(): ResourceGroup { + return this.current!.rg; + } + async update(rg: ResourceGroup): Promise { await this.config.s3 .putObject({ @@ -69,10 +80,109 @@ export class S3ResourceRepository implements ResourceRepository { .promise(); } + public getPolicyAttachment(filename: string): Buffer { + return this.policyAttachments[filename]; + } + + public async initializePolicyAttachments() { + try { + await this.refreshPolicyAttachments(); + } catch (err) { + logger.fatal({err}, 'Failed fetching s3 policy attachments on startup'); + throw err; + } + + setInterval(async () => { + try { + await this.refreshPolicyAttachments(); + } catch (err) { + logger.error( + {err}, + `Failed refreshing s3 policy attachments, last successful refresh was at ${this.policyAttachmentsRefreshedAt}` + ); + } + }, config.resourceUpdateInterval); + } + + private async refreshPolicyAttachments() { + const newRefreshedAt = new Date(); + + const allAttachments = await this.getPolicyAttachmentsList(); + const attachmentsToRefresh = allAttachments + .filter(a => this.shouldRefreshPolicyAttachment(a)) + .map(a => a.filename); + + if (attachmentsToRefresh.length > 0) { + const newAttachments = await this.getPolicyAttachments(attachmentsToRefresh); + newAttachments.forEach(a => (this.policyAttachments[a.filename] = a.content)); + } + + this.policyAttachmentsRefreshedAt = newRefreshedAt; + } + + private shouldRefreshPolicyAttachment({filename, updatedAt}: FileDetails) { + if (!this.policyAttachments[filename]) return true; + if (!this.policyAttachmentsRefreshedAt) return true; + + return updatedAt > this.policyAttachmentsRefreshedAt; + } + + private async getPolicyAttachmentsList(): Promise { + const attachments: FileDetails[] = []; + let isTruncated = true; + let continuationToken; + + while (isTruncated) { + const params: AWS.S3.Types.ListObjectsV2Request = { + Bucket: this.config.bucketName, + MaxKeys: 1000, + Prefix: this.config.policyAttachmentsKeyPrefix, + }; + if (continuationToken) params.ContinuationToken = continuationToken; + + const listResult = await this.config.s3.listObjectsV2(params).promise(); + const keys = listResult.Contents || []; + const newAttachments: FileDetails[] = keys.map(k => ({ + filename: this.getPolicyAttachmentFilenameByKey(k.Key!), + updatedAt: k.LastModified!, + })); + attachments.push(...newAttachments); + + isTruncated = listResult.IsTruncated!; + if (isTruncated) continuationToken = listResult.ContinuationToken; + } + + return attachments; + } + + private async getPolicyAttachments(filenames: string[]): Promise<{filename: string; content: Buffer}[]> { + const limit = pLimit(10); + + return Promise.all( + filenames.map(filename => { + return limit(async () => { + const key = this.getPolicyAttachmentKey(filename); + const params = {Bucket: this.config.bucketName, Key: key}; + const res = await this.config.s3.getObject(params).promise(); + + return {filename, content: res.Body as Buffer}; + }); + }) + ); + } + private getPolicyAttachmentKey(filename: string) { + return `${this.getPolicyAttachmentPrefix()}${filename}`; + } + + private getPolicyAttachmentPrefix() { return this.config.policyAttachmentsKeyPrefix.endsWith('/') - ? `${this.config.policyAttachmentsKeyPrefix}${filename}` - : `${this.config.policyAttachmentsKeyPrefix}/${filename}`; + ? this.config.policyAttachmentsKeyPrefix + : `${this.config.policyAttachmentsKeyPrefix}/`; + } + + private getPolicyAttachmentFilenameByKey(key: string) { + return key.replace(this.getPolicyAttachmentPrefix(), ''); } static fromEnvironment() { diff --git a/services/src/modules/resource-repository/types.ts b/services/src/modules/resource-repository/types.ts index 55f6250b..3a229b2c 100644 --- a/services/src/modules/resource-repository/types.ts +++ b/services/src/modules/resource-repository/types.ts @@ -47,7 +47,7 @@ export interface Policy extends Resource { interface PolicyQuery { type: PolicyQueryType; - paramName: string; + name: string; graphql?: PolicyQueryGraphQL; policy?: PolicyQueryPolicy; } @@ -72,8 +72,11 @@ export interface FetchLatestResult { export interface ResourceRepository { fetchLatest(): Promise; + getResourceGroup(): ResourceGroup; update(rg: ResourceGroup): Promise; writePolicyAttachment(filename: string, content: Buffer): Promise; + getPolicyAttachment(filename: string): Buffer; + initializePolicyAttachments(): Promise; } enum AuthType { diff --git a/services/src/registry.ts b/services/src/registry.ts index c3d92e75..9a9c8ff0 100644 --- a/services/src/registry.ts +++ b/services/src/registry.ts @@ -126,10 +126,9 @@ const typeDefs = gql` GraphQL doesn't support unions for input types, otherwise this would be a union of different policy query types. Instead, the PolicyQueryType enum indicates which policy query type is needed, and there's a property which corresponds to each policy query type, which we validate in the registry. """ - input # The query result will be available to the policy code in a parameter named as chosen in paramName, under the "data.queries" object. - PolicyQueryInput { + input PolicyQueryInput { type: PolicyQueryType! - paramName: String! + name: String! graphql: PolicyQueryGraphQLInput policy: PolicyQueryPolicyInput } @@ -199,7 +198,7 @@ interface PolicyQueryPolicyInput { interface PolicyQueryInput { type: PolicyQueryType; - paramName: string; + name: string; graphql?: PolicyQueryGraphQLInput; policy?: PolicyQueryPolicyInput; } diff --git a/services/src/tests/e2e/tests/authorization.spec.ts b/services/src/tests/e2e/tests/authorization.spec.ts new file mode 100644 index 00000000..c5478b68 --- /dev/null +++ b/services/src/tests/e2e/tests/authorization.spec.ts @@ -0,0 +1,45 @@ +import {GraphQLClient} from 'graphql-request'; +import {sleep} from '../../helpers/utility'; +import { + getSchema, + getUserQuery, + createSchemaMutation, + createPolicyMutation, + onlyAdminPolicy, +} from '../../helpers/authzSchema'; + +const gatewayClient = new GraphQLClient('http://localhost:8080/graphql'); +const registryClient = new GraphQLClient('http://localhost:8090/graphql'); + +describe('authorization', () => { + // This is kind of both the "before" section and a test, but it was weird putting a test in an actual before section + it('creates the policy and schema resources', async () => { + const policyResponse = await registryClient.request(createPolicyMutation, {policy: onlyAdminPolicy()}); + expect(policyResponse.updatePolicies.success).toBe(true); + + const schemaResponse = await registryClient.request(createSchemaMutation, {schema: getSchema()}); + expect(schemaResponse.updateSchemas.success).toBe(true); + + // Wait for gateway to update before next tests + await sleep(500); + }); + + it('allows access to a field based on an argument using param injection from source', async () => { + const response = await gatewayClient.request(getUserQuery('userAdmin')); + expect(response.userAdmin).toEqual({firstName: 'John', lastName: 'Smith', role: 'admin'}); + }); + + it('rejects access to a field when policy test fails, but still returns the other fields', async () => { + let response; + try { + await gatewayClient.request(getUserQuery('user')); + } catch (err) { + response = err.response; + } + + expect(response.errors).toHaveLength(1); + expect(response.errors[0].message).toBe('Unauthorized by policy onlyAdmin in namespace ns'); + expect(response.errors[0].path).toEqual(['user', 'lastName']); + expect(response.data.user).toEqual({firstName: 'John', lastName: null, role: 'normal'}); + }); +}); diff --git a/services/src/tests/e2e/tests/hello-world.spec.ts b/services/src/tests/e2e/tests/hello-world.spec.ts index af6419e7..9d7bffa4 100644 --- a/services/src/tests/e2e/tests/hello-world.spec.ts +++ b/services/src/tests/e2e/tests/hello-world.spec.ts @@ -1,6 +1,5 @@ -const {GraphQLClient} = require('graphql-request'); -const dockerCompose = require('docker-compose'); -const waitFor = require('../waitFor'); +import {GraphQLClient} from 'graphql-request'; +import {sleep} from '../../helpers/utility'; const gatewayClient = new GraphQLClient('http://localhost:8080/graphql'); const registryClient = new GraphQLClient('http://localhost:8090/graphql'); @@ -17,8 +16,6 @@ mutation CreateSchema($schema: SchemaInput!) { } }`; -const sleep = (timeout: number) => new Promise(r => setTimeout(r, timeout)); - describe('Basic flow', () => { test('Default schema works', async () => { const response = await gatewayClient.request(`query {default}`); diff --git a/services/src/tests/helpers/authzSchema.ts b/services/src/tests/helpers/authzSchema.ts new file mode 100644 index 00000000..1e7afb61 --- /dev/null +++ b/services/src/tests/helpers/authzSchema.ts @@ -0,0 +1,60 @@ +export const onlyAdminPolicy = () => ({ + metadata: {namespace: 'ns', name: 'onlyAdmin'}, + type: 'opa', + code: ` + default allow = false + allow { + input.args.role == "admin" + } + `, + args: { + role: 'String', + }, +}); + +export const createPolicyMutation = ` +mutation CreatePolicy($policy: PolicyInput!) { + updatePolicies(input: [$policy]) { + success + } +}`; + +export const getSchema = () => ({ + metadata: {namespace: 'ns', name: 'user'}, + schema: ` + type User { + firstName: String + lastName: String @policy(policies: [ + { namespace: "ns", name: "onlyAdmin", args: { role: "{source.role}" } } + ]) + role: String + } + type Query { + user: User! @stub(value: ${userQueryStub('normal')}) + userAdmin: User! @stub(value: ${userQueryStub('admin')}) + } + `, +}); + +const userQueryStub = (userRole: string) => `{ + firstName: "John" + lastName: "Smith" + role: "${userRole}" +}`; + +export const createSchemaMutation = ` +mutation CreateSchema($schema: SchemaInput!) { + updateSchemas(input: [$schema]) { + success + } +}`; + +export const getUserQuery = (queryType: string) => ` + query { + ${queryType} { + firstName + lastName + role + } + } +`; diff --git a/services/src/tests/helpers/utility.ts b/services/src/tests/helpers/utility.ts new file mode 100644 index 00000000..d2e7d47f --- /dev/null +++ b/services/src/tests/helpers/utility.ts @@ -0,0 +1 @@ +export const sleep = (timeout: number) => new Promise(r => setTimeout(r, timeout)); diff --git a/services/src/tests/integration/registry/create-resources.spec.ts b/services/src/tests/integration/registry/create-resources.spec.ts index e7179e9f..319b7519 100644 --- a/services/src/tests/integration/registry/create-resources.spec.ts +++ b/services/src/tests/integration/registry/create-resources.spec.ts @@ -56,12 +56,12 @@ const policy = { queries: [ { type: PolicyQueryType.graphql, - paramName: 'someGraphqlQuery', + name: 'someGraphqlQuery', graphql: {query: 'actual gql'}, }, { type: PolicyQueryType.policy, - paramName: 'somePolicyQuery', + name: 'somePolicyQuery', policy: {policyName: 'someOtherPolicy', args: {some: 'arg for the other policy'}}, }, ], diff --git a/services/src/tests/integration/registry/update-resources.spec.ts b/services/src/tests/integration/registry/update-resources.spec.ts index e4953a64..f362f30f 100644 --- a/services/src/tests/integration/registry/update-resources.spec.ts +++ b/services/src/tests/integration/registry/update-resources.spec.ts @@ -57,10 +57,10 @@ const policy = { another: 'one!', }, queries: [ - {type: PolicyQueryType.graphql, paramName: 'someGraphqlQuery', graphql: {query: 'actual gql'}}, + {type: PolicyQueryType.graphql, name: 'someGraphqlQuery', graphql: {query: 'actual gql'}}, { type: PolicyQueryType.policy, - paramName: 'somePolicyQuery', + name: 'somePolicyQuery', policy: {policyName: 'someOtherPolicy', args: {some: 'arg for the other policy'}}, }, ],