feat: allow horizontal scalability of lighthouse prover

.github/workflows/build-test-deploy.yml

@@ -61,9 +61,11 @@ jobs:
           node-version: "18"
           cache: "yarn"
 
-      # - name: Validate using commitlint
-      #   if: github.ref != 'refs/heads/testnet-prod' || github.ref != 'refs/heads/prod'
-      #   uses: wagoid/commitlint-github-action@v5
+      - name: Validate using commitlint
+        if: github.ref != 'refs/heads/testnet-prod' || github.ref != 'refs/heads/prod'
+        uses: wagoid/commitlint-github-action@v5
+        with:
+          commitDepth: 1
 
       - name: Yarn install
         run: yarn install
@@ -609,10 +611,52 @@ jobs:
       - name: Build, tag, and push docker image to Amazon ECR Public
 
         run: |
-          DOCKER_BUILDKIT=1 docker build -f docker/lighthouse/Dockerfile -t $REGISTRY/$REPOSITORY:$IMAGE_TAG -t ghcr.io/connext/lighthouse:$IMAGE_TAG .
+          DOCKER_BUILDKIT=1 docker build -f docker/lighthouse/lambda/Dockerfile -t $REGISTRY/$REPOSITORY:$IMAGE_TAG -t ghcr.io/connext/lighthouse:$IMAGE_TAG .
           docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
           docker push ghcr.io/connext/lighthouse:$IMAGE_TAG
 
+  build-and-push-lighthouse-prover-subscriber-image:
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/testnet-prod' || github.ref == 'refs/heads/prod'
+    env:
+      REGISTRY: ghcr.io
+      IMAGE_NAME: connext/lighthouse-subscriber
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=sha,format=short
+            type=semver,pattern={{raw}}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          file: docker/lighthouse/subscriber/Dockerfile
+
+    outputs:
+      json: ${{ steps.meta.outputs.json }}
+
   build-and-push-relayer-image:
     if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/testnet-prod' || github.ref == 'refs/heads/prod'
     env:
@@ -773,6 +817,7 @@ jobs:
         build-and-push-sequencer-subscriber-image,
         build-and-push-cartographer-image,
         build-and-push-lighthouse-image,
+        build-and-push-lighthouse-prover-subscriber-image,
         build-and-push-relayer-image,
         build-and-push-watcher-image,
         build-and-push-sdk-server-image,
@@ -786,6 +831,7 @@ jobs:
       SEQUENCER_SUBSCRIBER_IMAGE: ${{ fromJSON(needs.build-and-push-sequencer-subscriber-image.outputs.json).tags[0] }}
       CARTOGRAPHER_IMAGE: ghcr.io/connext/cartographer:${{ github.sha }}
       LIGHTHOUSE_IMAGE: ghcr.io/connext/lighthouse:${{ github.sha }}
+      LIGHTHOUSE_PROVER_SUBSCRIBER_IMAGE: ${{ fromJSON(needs.build-and-push-lighthouse-prover-subscriber-image.outputs.json).tags[0] }}
       RELAYER_IMAGE: ${{ fromJSON(needs.build-and-push-relayer-image.outputs.json).tags[0] }}
       WATCHER_IMAGE: ${{ fromJSON(needs.build-and-push-watcher-image.outputs.json).tags[0] }}
       SDK_SERVER_IMAGE: ${{ fromJSON(needs.build-and-push-sdk-server-image.outputs.json).tags[0] }}
@@ -814,6 +860,7 @@ jobs:
       sequencer-server-tags: ${{ needs.build-and-push-sequencer-server-image.outputs.json }}
       sequencer-publisher-tags: ${{ needs.build-and-push-sequencer-publisher-image.outputs.json }}
       sequencer-subscriber-tags: ${{ needs.build-and-push-sequencer-subscriber-image.outputs.json }}
+      lighthouse-prover-subscriber-tags: ${{ needs.build-and-push-lighthouse-prover-subscriber-image.outputs.json }}
       relayer-tags: ${{ needs.build-and-push-relayer-image.outputs.json }}
       watcher-tags: ${{ needs.build-and-push-watcher-image.outputs.json }}
       sdk-server-tags: ${{ needs.build-and-push-sdk-server-image.outputs.json }}
@@ -832,6 +879,7 @@ jobs:
         build-and-push-sequencer-subscriber-image,
         build-and-push-cartographer-image,
         build-and-push-lighthouse-image,
+        build-and-push-lighthouse-prover-subscriber-image,
         build-and-push-relayer-image,
         build-and-push-watcher-image,
         build-and-push-sdk-server-image,
@@ -844,6 +892,7 @@ jobs:
       SEQUENCER_PUBLISHER_IMAGE: ${{ fromJSON(needs.build-and-push-sequencer-publisher-image.outputs.json).tags[0] }}
       SEQUENCER_SUBSCRIBER_IMAGE: ${{ fromJSON(needs.build-and-push-sequencer-subscriber-image.outputs.json).tags[0] }}
       LIGHTHOUSE_IMAGE: ghcr.io/connext/lighthouse:${{ github.sha }}
+      LIGHTHOUSE_PROVER_SUBSCRIBER_IMAGE: ${{ fromJSON(needs.build-and-push-lighthouse-prover-subscriber-image.outputs.json).tags[0] }}
       RELAYER_IMAGE: ${{ fromJSON(needs.build-and-push-relayer-image.outputs.json).tags[0] }}
       WATCHER_IMAGE: ${{ fromJSON(needs.build-and-push-watcher-image.outputs.json).tags[0] }}
       WEB3_SIGNER_PRIVATE_KEY_ROUTER: "0xc87509a1c067bbde78beb793e6fa76530b6382a4c0241e5e4a9ec0a0f44dc0d3"
@@ -878,6 +927,7 @@ jobs:
       sequencer-server-tags: ${{ needs.build-and-push-sequencer-server-image.outputs.json }}
       sequencer-publisher-tags: ${{ needs.build-and-push-sequencer-publisher-image.outputs.json }}
       sequencer-subscriber-tags: ${{ needs.build-and-push-sequencer-subscriber-image.outputs.json }}
+      lighthouse-prover-subscriber-tags: ${{ needs.build-and-push-lighthouse-prover-subscriber-image.outputs.json }}
       relayer-tags: ${{ needs.build-and-push-relayer-image.outputs.json }}
       watcher-tags: ${{ needs.build-and-push-watcher-image.outputs.json }}
       sdk-server-tags: ${{ needs.build-and-push-sdk-server-image.outputs.json }}
@@ -989,6 +1039,7 @@ jobs:
       TF_VAR_full_image_name_sequencer_subscriber: ${{ fromJSON(needs.smoke-tests.outputs.sequencer-subscriber-tags).tags[0] }}
       TF_VAR_full_image_name_watcher: ${{ fromJSON(needs.smoke-tests.outputs.watcher-tags).tags[0] }}
       TF_VAR_full_image_name_relayer: ${{ fromJSON(needs.smoke-tests.outputs.relayer-tags).tags[0] }}
+      TF_VAR_full_image_name_lighthouse_prover_subscriber: ${{ fromJSON(needs.smoke-tests.outputs.lighthouse-prover-subscriber-tags).tags[0] }}
       TF_VAR_lighthouse_image_tag: ${{ github.sha }}
 
     runs-on: ubuntu-latest
@@ -1045,6 +1096,7 @@ jobs:
       sequencer-server-tags: ${{ needs.smoke-tests.outputs.sequencer-server-tags }}
       sequencer-publisher-tags: ${{ needs.smoke-tests.outputs.sequencer-publisher-tags }}
       sequencer-subscriber-tags: ${{ needs.smoke-tests.outputs.sequencer-subscriber-tags }}
+      lighthouse-prover-subscriber-tags: ${{ needs.smoke-tests.outputs.lighthouse-prover-subscriber-tags }}
       router-publisher-tags: ${{ needs.smoke-tests.outputs.router-publisher-tags }}
       router-subscriber-tags: ${{ needs.smoke-tests.outputs.router-subscriber-tags }}
       router-executor-tags: ${{ needs.smoke-tests.outputs.router-executor-tags }}
@@ -1117,6 +1169,7 @@ jobs:
       sequencer-server-tags: ${{ needs.e2e-tests.outputs.sequencer-server-tags }}
       sequencer-publisher-tags: ${{ needs.e2e-tests.outputs.sequencer-publisher-tags }}
       sequencer-subscriber-tags: ${{ needs.e2e-tests.outputs.sequencer-subscriber-tags }}
+      lighthouse-prover-subscriber-tags: ${{ needs.e2e-tests.outputs.lighthouse-prover-subscriber-tags }}
       router-publisher-tags: ${{ needs.e2e-tests.outputs.router-publisher-tags }}
       router-subscriber-tags: ${{ needs.e2e-tests.outputs.router-subscriber-tags }}
       router-executor-tags: ${{ needs.e2e-tests.outputs.router-executor-tags }}
@@ -1188,6 +1241,7 @@ jobs:
       router-publisher-tags: ${{ needs.e2e-tests.outputs.router-publisher-tags }}
       router-subscriber-tags: ${{ needs.e2e-tests.outputs.router-subscriber-tags }}
       router-executor-tags: ${{ needs.e2e-tests.outputs.router-executor-tags }}
+      lighthouse-prover-subscriber-tags: ${{ fromJSON(needs.e2e-tests.outputs.lighthouse-prover-subscriber-tags).tags[0] }}
       relayer-tags: ${{ needs.e2e-tests.outputs.relayer-tags }}
       watcher-tags: ${{ needs.e2e-tests.outputs.watcher-tags }}
       sdk-server-tags: ${{ needs.e2e-tests.outputs.sdk-server-tags }}
@@ -1262,6 +1316,7 @@ jobs:
       TF_VAR_full_image_name_sequencer_subscriber: ${{ fromJSON(needs.terraform-services-backend-prod-testnet.outputs.sequencer-subscriber-tags).tags[0] }}
       TF_VAR_full_image_name_relayer: ${{ fromJSON(needs.terraform-services-backend-prod-testnet.outputs.relayer-tags).tags[0] }}
       TF_VAR_full_image_name_watcher: ${{ fromJSON(needs.terraform-services-backend-prod-testnet.outputs.watcher-tags).tags[0] }}
+      TF_VAR_full_image_name_lighthouse_prover_subscriber: ${{ fromJSON(needs.terraform-services-backend-prod-testnet.outputs.lighthouse-prover-subscriber-tags).tags[0] }}
       TF_VAR_lighthouse_image_tag: ${{ github.sha }}
 
     runs-on: ubuntu-latest

docker/lighthouse/subscriber/Dockerfile

@@ -0,0 +1,85 @@
+FROM node:18-alpine as node
+
+# ----------------------------------------
+# Builds Docker container for nxtp-router package.
+# @dev configuraion located ./config.json (relative to this file)
+
+
+# ----------------------------------------
+
+FROM node as build
+RUN apk update
+RUN apk add git
+
+ENV HOME=/tmp/build \
+    PATH=/tmp/build/node_modules/.bin:./node_modules/.bin:${PATH}
+
+WORKDIR /tmp/build
+
+ARG TEMP_DEPS_DIR
+
+# ----- Copy only the files that affect yarn install -----
+# Allows docker to use cache and skip install if dependencies are unchanged.
+# Assumes that ./packages/*/package.json files have been copied to TEMP_DEPS_DIR
+# with that same directory structure. build.sh does this.
+COPY .yarn /tmp/build/.yarn/
+COPY .yarnrc.yml /tmp/build/
+COPY package.json /tmp/build/
+COPY packages/agents/lighthouse/package.json /tmp/build/packages/agents/lighthouse/
+COPY packages/agents/relayer/package.json /tmp/build/packages/agents/relayer/
+COPY packages/adapters/txservice/package.json /tmp/build/packages/adapters/txservice/
+COPY packages/adapters/cache/package.json /tmp/build/packages/adapters/cache/
+COPY packages/adapters/web3signer/package.json /tmp/build/packages/adapters/web3signer/
+COPY packages/deployments/contracts/package.json /tmp/build/packages/deployments/contracts/
+COPY packages/adapters/database/package.json /tmp/build/packages/adapters/database/
+COPY packages/adapters/relayer/package.json /tmp/build/packages/adapters/relayer/
+COPY packages/adapters/subgraph/package.json /tmp/build/packages/adapters/subgraph/
+COPY packages/adapters/web3signer/package.json /tmp/build/packages/adapters/web3signer/
+COPY packages/utils/package.json /tmp/build/packages/utils/
+COPY yarn.lock /tmp/build/
+
+# ----- Install dependencies -----
+# Install dependencies exactly as in the yarn.lock file - no updates.
+RUN yarn install
+
+# ----- Copy source and all other files that affect lint, test, build -----
+COPY config config/
+COPY packages/agents/lighthouse/ /tmp/build/packages/agents/lighthouse/
+COPY packages/agents/relayer /tmp/build/packages/agents/relayer
+COPY packages/adapters/txservice /tmp/build/packages/adapters/txservice
+COPY packages/adapters/cache /tmp/build/packages/adapters/cache
+COPY packages/adapters/web3signer /tmp/build/packages/adapters/web3signer
+COPY packages/deployments/contracts /tmp/build/packages/deployments/contracts/
+COPY packages/adapters/database /tmp/build/packages/adapters/database
+COPY packages/adapters/relayer /tmp/build/packages/adapters/relayer
+COPY packages/adapters/subgraph /tmp/build/packages/adapters/subgraph
+COPY packages/utils /tmp/build/packages/utils
+COPY .eslintignore /tmp/build/
+COPY .eslintrc.js /tmp/build/
+
+# ----- Lint, test and build -----
+RUN yarn build
+
+# ----------------------------------------
+# Copy files to the deployment image.
+# ----------------------------------------
+
+FROM node as runtime
+
+ENV NODE_ENV=production \
+    PORT=8081
+
+ARG COMMIT_HASH
+ENV COMMIT_HASH ${COMMIT_HASH:-unknown}
+
+# ----- Copy files required at runtime by the app -----
+COPY --from=build --chown=node:node /tmp/build /home/node
+
+# This user is created in the base image with uid and gid = 1000.
+USER node
+
+WORKDIR /home/node/packages/agents/lighthouse
+
+EXPOSE 8081
+
+CMD ["node", "--trace-warnings", "dist/tasks/run.js"]

ops/mainnet/prod/core/config.tf

@@ -275,6 +275,10 @@ locals {
   })
 
   local_lighthouse_config = jsonencode({
+    redis = {
+      host = module.lighthouse_cache.redis_instance_address,
+      port = module.lighthouse_cache.redis_instance_port
+    }
     logLevel = "debug"
     chains = {
       "6648936" = {

ops/mainnet/prod/core/main.tf

@@ -293,6 +293,44 @@ module "sequencer_web3signer" {
   container_env_vars       = local.sequencer_web3signer_env_vars
 }
 
+module "lighthouse_prover_subscriber" {
+  source                   = "../../../modules/service"
+  stage                    = var.stage
+  environment              = var.environment
+  domain                   = var.domain
+  region                   = var.region
+  dd_api_key               = var.dd_api_key
+  zone_id                  = data.aws_route53_zone.primary.zone_id
+  execution_role_arn       = data.aws_iam_role.ecr_admin_role.arn
+  cluster_id               = module.ecs.ecs_cluster_id
+  vpc_id                   = module.network.vpc_id
+  private_subnets          = module.network.private_subnets
+  lb_subnets               = module.network.public_subnets
+  internal_lb              = false
+  docker_image             = var.full_image_name_lighthouse_prover_subscriber
+  container_family         = "lighthouse-prover-subscriber"
+  health_check_path        = "/ping"
+  container_port           = 7072
+  loadbalancer_port        = 80
+  cpu                      = 1024
+  memory                   = 2048
+  instance_count           = 10
+  timeout                  = 180
+  ingress_cdir_blocks      = ["0.0.0.0/0"]
+  ingress_ipv6_cdir_blocks = []
+  service_security_groups  = flatten([module.network.allow_all_sg, module.network.ecs_task_sg])
+  cert_arn                 = var.certificate_arn_testnet
+  container_env_vars       = concat(local.lighthouse_prover_subscriber_env_vars, [{ name = "LIGHTHOUSE_SERVICE", value = "prover-sub" }])
+}
+module "lighthouse_prover_subscriber_auto_scaling" {
+  source           = "../../../modules/auto-scaling"
+  stage            = var.stage
+  environment      = var.environment
+  domain           = var.domain
+  ecs_service_name = module.lighthouse_prover_subscriber.service_name
+  ecs_cluster_name = module.ecs.ecs_cluster_name
+}
+
 module "lighthouse_prover_cron" {
   source              = "../../../modules/lambda"
   ecr_repository_name = "nxtp-lighthouse"
@@ -526,3 +564,13 @@ module "relayer_cache" {
   vpc_id                        = module.network.vpc_id
   cache_subnet_group_subnet_ids = module.network.public_subnets
 }
+
+module "lighthouse_cache" {
+  source                        = "../../../modules/redis"
+  stage                         = var.stage
+  environment                   = var.environment
+  family                        = "lighthouse"
+  sg_id                         = module.network.ecs_task_sg
+  vpc_id                        = module.network.vpc_id
+  cache_subnet_group_subnet_ids = module.network.public_subnets
+}

ops/modules/amq/outputs.tf

@@ -5,3 +5,7 @@ output "aws_mq_broker_console" {
 output "aws_mq_amqp_endpoint" {
   value = trimprefix(aws_mq_broker.default.instances[0].endpoints[0], "amqps://")
 }
+
+output "aws_mq_amqp_arn" {
+  value = aws_mq_broker.default.arn
+}

ops/modules/auto-scaling/main.tf

@@ -1,6 +1,6 @@
 
 resource "aws_appautoscaling_target" "service_task_scaling" {
-  max_capacity       = 50
+  max_capacity       = 100
   min_capacity       = 10
   resource_id        = "service/${var.ecs_cluster_name}/${var.ecs_service_name}"
   scalable_dimension = "ecs:service:DesiredCount"
@@ -34,6 +34,6 @@ resource "aws_appautoscaling_policy" "service_avg_cpu_scaling" {
       predefined_metric_type = "ECSServiceAverageCPUUtilization"
     }
 
-    target_value = 50
+    target_value = 25
   }
 }