exampleApplicationConfig.js

/**
 * @fileOverview An example configuration file, with documentation.
 *
 * It is very important to note that the configuration object will, after it is
 * loaded by the deployment code, be:
 *
 * - Regenerated into code and added as a file in Lambda function modules.
 * - Loaded in Lambda by all Lambda functions in a Lambda Complex application.
 *
 * This means that almost everything that results in a valid configuration
 * object will just work, but there are things you can do that will result in
 * failure to run in Lambda.
 *
 * In particular note the restrictions on routing functions, described below in
 * the component inline comments.
 */

module.exports = {
  // The application name. It must be unique for a given account.
  //
  // When deploying an application the queue names and Lambda function names
  // are prefixed with the application name.
  name: 'example',

  // The application version. This is used in tags and some names.
  version: '1.0.0',

  // A deployment ID that is unique to this deployment of this application. For
  // example the BUILD_NUMBER value generated by Jenkins tasks, or a unix
  // timestamp, or similar. It can be alphanumeric.
  //
  // This is included in most tags, paths, and names.
  deployId: 15,

  // AWS deployment settings.
  //
  // The application is deployed as a CloudFormation stack, and after successful
  // completion any prior deployed stacks for this application are destroyed. On
  // a failure to deploy the new stack is destroyed.
  deployment: {

    // ----------------------------------------------------------------------
    // Required.
    // ----------------------------------------------------------------------

    // AWS region.
    //
    // This must match whatever region is configured in the deployment
    // environment.
    //
    // TODO: having this and the environment configuration is clumsy. Look at
    // cutting that down to one declaration one way or another.
    region: 'us-east-1',

    // The S3 bucket into which packaged Lambda functions will be uploaded.
    s3Bucket: 'lambda-complex',

    // All packaged Lambda function keys will be prefixed with this string.
    s3KeyPrefix: 'applications/',

    // ----------------------------------------------------------------------
    // Optional.
    // ----------------------------------------------------------------------

    // Any additional tags to add to the CloudFormation stack.
    tags: {
      tagName: 'tagValue'
    },

    // An asynchronous function invoked after the new stack is deployed but
    // before the old stack is destroyed. This should perform any functions
    // needed to switch over to use the new stack and then call back.
    //
    // Generally this will mean switching to use new queues to introduce data
    // into the application and then waiting for the old queues to clear, but
    // the precise details will depend on the application.
    //
    // This function will not be invoked in Lambda functions when deployed, so
    // it can require() and invoke any resources available to Lambda Complex
    // during deployment.
    //
    // @param {Object} stackDescription Stack description, including outputs.
    // @param {Object} config This configuration object.
    // @param {Function} callback Of the form function (error).
    switchoverFunction: function (stackDescription, config, callback) {
      callback();
    },

    // If set true then CloudFormation stacks created in prior deployments of
    // this application are NOT destroyed on a successful deployment.
    //
    // This can be helpful during development.
    skipPriorCloudFormationStackDeletion: false,

    // If set true, then CloudWatch log groups created by the Lambda function
    // instances from prior deployments of this application are NOT destroyed
    // on a successful deployment.
    //
    // CloudWatch log groups are clutter for some people, others want to keep
    // them.
    skipPriorCloudWatchLogGroupsDeletion: false,

    // If set true, then the CloudFormation stack for this deployment is NOT
    // destroyed on deployment failure.
    //
    // This can be helpful during development.
    skipCloudFormationStackDeletionOnFailure: false
  },

  // The coordinator is a lambda function package that is expected to consume
  // this configuration and manage the invocation of invoker lambda functions in
  // the required concurrency.
  coordinator: {
    // The maximum number of concurrent coordinators to run.
    //
    // Each coordinator will carry out its fraction of the overall work to be
    // done. E.g. for 2 concurrent coordinators, each will undertake half of the
    // necessary invocations.
    //
    // More than one coordinator provides a little redundancy against the random
    // slings and arrows of AWS API failure. The application relies on
    // coordinators invoking themselves, and while the coordinator code tries to
    // be bulletproof, there are still a number of points at which failure can
    // occur.
    //
    // So long as one coordinator is running, it will invoke as many other
    // instances as are needed to maintain this specified concurrency.
    coordinatorConcurrency: 2,

    // How many AWS API requests can the coordinator make at any one time?
    // Too many and you'll tend to see errors.
    maxApiConcurrency: 10,

    // How many Lambda functions, excluding the coordinator itself, can a
    // single coordinator instance launch? Too many and it can get timed out.
    // Too few and concurrency won't be maxed out. When making invocation
    // requests, the <maxApiConcurrency> setting is obeyed, so it will limit
    // how many requests are made at one time.
    maxInvocationCount: 50,

    // Minimum interval in seconds between invocations. Has to be equal to
    // or less than the execution time limit for a lambda function, see:
    // http://docs.aws.amazon.com/lambda/latest/dg/limits.html
    //
    // A coordinator may well run longer than this interval before invoking the
    // next coordinator if it has a lot to do.
    //
    // The shorter the interval the more frequently that new component Lambda
    // function instances will be launched in response to messages in their
    // queues. This goes a long way towards determining the pace of the
    // application for small applications and small amounts of data.
    minInterval: 10
  },

  // Every Lambda function, and thus every component, is associated with a
  // single IAM role. This role provides it with all of the necessary
  // permissions to access AWS resources. E.g. S3 buckets, etc.
  //
  // Component definitions each specify the use of one of these roles.
  //
  roles: [
    // In the rare case that a Lambda function needs no permissions, it is
    // possible to assign an empty role. Lambda Complex will add the necessary
    // internal permissions to it, such as queue access and logging.
    {
      name: 'default',
      statements: []
    },
    // More commonly a role will contain one or more statements. These follow
    // the normal form for policy statements inside a CloudFormation role
    // definition (but with lowercase property names, for consistency with the
    // rest of this config file).
    //
    // In addition to the statements provided here, Lambda Complex will add
    // necessary internal permissions, such as queue access and logging.
    {
      name: 's3Read',
      statements: [
        {
          effect: 'Allow',
          action: [
            's3:get*',
            's3:list*'
          ],
          resource: [
            'arn:aws:s3:::exampleBucket1/*',
            'arn:aws:s3:::exampleBucket2/*'
          ]
        }
      ]
    }
  ],

  // A component consists of a lambda function package that will be triggered in
  // response to circumstances, such as a message in a queue or the completion
  // of another Lambda function.
  //
  // There are several different types of component.
  //
  // Any number of components can be defined.
  components: [

    // Example of a lambda function that consumes a message from a queue when
    // invoked, and the message data becomes the event passed to the lambda
    // function handler.
    //
    // This type of component can be run at a given concurrency, controlled by
    // the coordinator.
    {
      // The name of the lambda function definition to be created in
      // AWS.
      name: 'eventFromMessageExample',

      // Type of the component.
      type: 'eventFromMessage',

      // The maximum number of Lambda function instances for this component that
      // will run at any one time. This, coupled with coordinator.minInterval,
      // goes towards determining the rate at which this component's message
      // queue will drain.
      maxConcurrency: 10,

      // Seconds to wait for a message to show up in the queue before giving up.
      // This should not be high as it cuts into short time limit for a Lambda.
      // function. Setting a few seconds may smooth out the operation of an
      // application in future versions of Lambda Complex, but at the present
      // time should make no difference either way.
      queueWaitTime: 5,

      // Define where the results from this Lambda function are sent. The data
      // passed to context.succeed(data) or context.done(null, data) will be
      // send on to another component as the event passed to the handle of its
      // Lambda function.
      //
      // This is an optional property. If not defined, then data is not passed
      // on to any other component.
      //
      // If a string, then results on success will always be sent to an
      // invocation of the specified component's Lambda function. On failure
      // there is no invocation and no data sent.
      routing: 'eventFromInvocationExample',
      //
      // If an array of strings, then a copy of the results on success is sent
      // to an invocation of the Lambda function for each of the specified
      // components.
      //routing: ['componentA', 'componentB'],
      //
      // If a function, then the destination components and the data sent can
      // be specified as desired.
      //
      // Note that this function will be called inside a Lambda function context
      // and so cannot rely on any NPM modules not in that context. Further,
      // the require() statements for resources must be inside the routing
      // function context, as this config file will be loaded in other contexts
      // that do not have those modules available.
      //
      //routing: function (error, data) {
      //  // This is safe provided that the Lambda function for this component
      //  // includes the underscore module.
      //  var _ = require('underscore');
      //
      //  // Don't send on any data if an error resulted.
      //  if (error) {
      //    return [];
      //  }
      //
      //  // Otherwise split up or manipulate data and send it to other
      //  // components as desired.
      //  return [
      //    { name: 'componentA', data: _.keys(data) },
      //    { name: 'componentA', data: _.values(data) }
      //  ];
      //},

      // Detailing the lambda function to be used by this component.
      lambda: {
        // If an NPM package, this is the package name or location for use when
        // installing: npm install <lambdaFunctionPackage>.
        //
        // Note that you can specify a local absolute path instead of a package
        // name. This is the usual methology when constructing an application
        // from multiple components: all of the lambda function definitions are
        // kept locally.
        npmPackage: 'package-name',
        // npmPackage: '/absolute/path/to/local/package/dir',

        // The name of the lambda function handle exported in the package code.
        // This doesn't have to be the same as the name of the lambda function
        // definition.
        handler: 'index.handler',

        // Memory size in MB, with a minimum of 128.
        memorySize: 128,

        // Timeout for the function in seconds.
        timeout: 60,

        // The role used by this function.
        role: 's3Read'
      }
    },

    // Example of a lambda function that is directly invoked with data from a
    // preceding invocation.
    //
    // Note that there is no concept of concurrency here; this is invoked
    // directly by a prior lambda function execution, so operates at the same
    // concurrency.
    {
      // The name of the lambda function definition to be created in
      // AWS.
      name: 'eventFromInvocationExample',

      // Type of the component.
      type: 'eventFromInvocation',

      // Define where the results from this Lambda function are sent. The data
      // passed to context.succeed(data) or context.done(null, data) will be
      // send on to another component as the event passed to the handle of its
      // Lambda function.
      //
      // In this case, no destination is defined so no other function will be
      // invoked as a result of this function completing.
      routing: undefined,

      // Detailing the lambda function to be used by this component.
      lambda: {
        // If an NPM package, this is the package name or location for use when
        // installing: npm install <lambdaFunctionPackage>.
        //
        // Note that you can specify a local absolute path instead of a package
        // name. This is the usual methology when constructing an application
        // from multiple components: all of the lambda function definitions are
        // kept locally.
        npmPackage: 'package-name',
        // npmPackage: '/absolute/path/to/local/package/dir',

        // The lambda function file and function property exported in the
        // package code. For this example that is index.js and the exported
        // property name 'handle'.
        handler: 'index.handler',

        // Memory size in MB, with a minimum of 128.
        memorySize: 128,

        // Timeout for the function in seconds.
        timeout: 60,

        // The role used by this function.
        role: 'default'
      }
    }
  ]

};