Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Multistage Docker + Test in Docker #6345

Merged
merged 1 commit into from
Oct 1, 2020
Merged

Multistage Docker + Test in Docker #6345

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
249 changes: 163 additions & 86 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ orbs:
horizon: artsy/release@0.0.1
node: artsy/node@1.0.0
yarn: artsy/yarn@5.1.3
artsy-remote-docker: artsy/remote-docker@0.1.8

jobs:
run_deepcrawl_automator:
Expand All @@ -20,14 +21,6 @@ jobs:
- run:
name: Start automator crawl
command: chmod +x automator.sh && ./automator.sh
acceptance_cypress:
docker:
- image: circleci/node:12-stretch-browsers
steps:
- yarn/setup
- run:
name: Cypress Tests
command: yarn test:smoke

validate_production_schema:
executor: node/build
Expand All @@ -37,29 +30,6 @@ jobs:
name: Validate Production Schema
command: node scripts/validateSchemas.js production

build:
executor: node/build
steps:
- yarn/setup
- run:
name: Build force assets
command: yarn assets
- store_artifacts:
path: ~/project/.artifacts
- run:
name: Duplicates Report
command: curl "https://artsy-dupe-report.now.sh/packages/dupe-report/now.js?owner=artsy&repo=force&buildNum=$CIRCLE_BUILD_NUM"

# FIXME: Reenable after https://github.com/artsy/force/pull/5673 is addressed
# danger:
# executor: node/build
# steps:
# - yarn/setup
# - run:
# name: Danger
# # Formatted this way to prevent GitHub's token detection. This is intended to be committed and public.
# command: DANGER_GITHUB_API_TOKEN="3f715685d9d032e17""48c3368dc8f22c672849136" yarn danger ci

create_or_update_review_app:
executor: hokusai/deploy
steps:
Expand All @@ -76,15 +46,81 @@ jobs:
name: "Create or update review app"
command: |
review_app_name=$(echo $CIRCLE_BRANCH | sed 's/review-app-//')

kubectl config use-context staging

if $(kubectl get namespace | grep -qi $review_app_name); then
./scripts/update_review_app.sh $review_app_name
else
./scripts/build_review_app.sh $review_app_name
fi

mocha:
docker:
- image: 585031190124.dkr.ecr.us-east-1.amazonaws.com/force:$CIRCLE_SHA1-builder
aws_auth:
aws_access_key_id: $AWS_ACCESS_KEY_ID
aws_secret_access_key: $AWS_SECRET_ACCESS_KEY
working_directory: /app
environment:
- NODE_ENV: test
steps:
- run: yarn test:mocha

jest-v1:
docker:
- image: 585031190124.dkr.ecr.us-east-1.amazonaws.com/force:$CIRCLE_SHA1-builder
aws_auth:
aws_access_key_id: $AWS_ACCESS_KEY_ID
aws_secret_access_key: $AWS_SECRET_ACCESS_KEY
working_directory: /app
environment:
- NODE_ENV: test
steps:
- run: yarn test:jest:v1 -w 2

jest-v2:
docker:
- image: 585031190124.dkr.ecr.us-east-1.amazonaws.com/force:$CIRCLE_SHA1-builder
aws_auth:
aws_access_key_id: $AWS_ACCESS_KEY_ID
aws_secret_access_key: $AWS_SECRET_ACCESS_KEY
working_directory: /app
environment:
- NODE_ENV: test
steps:
- run: yarn test:jest:v2 -w 2

type-check:
docker:
- image: 585031190124.dkr.ecr.us-east-1.amazonaws.com/force:$CIRCLE_SHA1-builder
aws_auth:
aws_access_key_id: $AWS_ACCESS_KEY_ID
aws_secret_access_key: $AWS_SECRET_ACCESS_KEY
working_directory: /app
environment:
- NODE_ENV: test
steps:
- run: yarn type-check

acceptance:
docker:
- image: 585031190124.dkr.ecr.us-east-1.amazonaws.com/force:$CIRCLE_SHA1-electron-runner
aws_auth:
aws_access_key_id: $AWS_ACCESS_KEY_ID
aws_secret_access_key: $AWS_SECRET_ACCESS_KEY
working_directory: /app
steps:
- run: /app/scripts/xvfb-run.sh /usr/local/bin/yarn test:acceptance

acceptance-cypress:
docker:
- image: 585031190124.dkr.ecr.us-east-1.amazonaws.com/force:$CIRCLE_SHA1-electron-runner
aws_auth:
aws_access_key_id: $AWS_ACCESS_KEY_ID
aws_secret_access_key: $AWS_SECRET_ACCESS_KEY
working_directory: /app
steps:
- run: /app/scripts/xvfb-run.sh /usr/local/bin/yarn test:smoke

not_master_or_staging_or_release: &not_master_or_staging_or_release
filters:
branches:
Expand Down Expand Up @@ -115,87 +151,128 @@ only_release: &only_release
workflows:
default:
jobs:
- horizon/block:
<<: *only_release
context: horizon
project_id: 11
# Main build
- artsy-remote-docker/buildkit-build:
<<: *not_staging_or_release
context: hokusai
name: builder-image-build
pre-steps:
- run:
command: echo 'export BUILD_TARGET="builder";' >> $BASH_ENV

# Pre-staging
- yarn/jest:
- artsy-remote-docker/buildkit-push:
<<: *not_staging_or_release
args: --runInBand
context: hokusai
name: builder-image-push
requires:
- builder-image-build
pre-steps:
- run:
command: echo 'export BUILD_TARGET="builder";' >> $BASH_ENV

# Electron Runner build
- artsy-remote-docker/buildkit-build:
<<: *not_staging_or_release
context: hokusai
name: electron-runner-image-build
requires:
- builder-image-build
pre-steps:
- run:
command: echo 'export BUILD_TARGET="electron-runner";' >> $BASH_ENV

- artsy-remote-docker/buildkit-push:
<<: *not_staging_or_release
context: hokusai
name: electron-runner-image-push
requires:
- electron-runner-image-build
pre-steps:
- run:
command: echo 'export BUILD_TARGET="electron-runner";' >> $BASH_ENV

- yarn/run:
# Test steps
- mocha:
<<: *not_staging_or_release
name: test:mocha
script: "test:mocha"

# TODO: Disabled due to memory issues. Can we use our workflows above and
# upload coverage that way (similar to reaction). We save a lot of time
# by running outside of an unnecessary docker context.

# - hokusai/test:
# name: test
# <<: *not_staging_or_release
# post-steps:
# - run: mkdir -p ./coverage ./.nyc_output ./reports
# - run:
# name: Copy jest coverage artifacts
# command: docker cp hokusai_force_1:/app/coverage ./
# when: always
# - codecov/upload:
# file: ./coverage/lcov.info
# - run:
# name: Copy mocha coverage artifacts
# command: docker cp hokusai_force_1:/app/.nyc_output ./
# when: always
# - codecov/upload:
# file: ./.nyc_output/lcov.info
# - run:
# name: Copy coverage reports
# command: docker cp hokusai_force_1:/app/reports ./
# when: always
# - store_test_results:
# path: ./reports

- yarn/update-cache:
context: hokusai
requires:
- builder-image-push

- jest-v1:
<<: *not_staging_or_release
context: hokusai
requires:
- builder-image-push

- jest-v2:
<<: *not_staging_or_release
context: hokusai
requires:
- builder-image-push

- type-check:
<<: *not_staging_or_release
- yarn/type-check:
context: hokusai
requires:
- builder-image-push

- acceptance:
<<: *not_staging_or_release
- acceptance_cypress:
<<: *not_master_or_staging_or_release
- build:
context: hokusai
requires:
- electron-runner-image-push

- acceptance-cypress:
<<: *not_staging_or_release
# - danger:
# <<: *not_staging_or_release
context: hokusai
requires:
- electron-runner-image-push

# Staging
- hokusai/push:
name: push-staging-image
- artsy-remote-docker/buildkit-build:
<<: *only_master
context: hokusai
name: production-image-build
requires:
- builder-image-build

- artsy-remote-docker/buildkit-push:
<<: *only_master
context: hokusai
name: production-image-push
requires:
- yarn/type-check
- yarn/jest
- test:mocha
- build
- mocha
- jest-v1
- jest-v2
- type-check
- acceptance
- acceptance-cypress
- production-image-build

- hokusai/deploy-staging:
<<: *only_master
name: deploy-staging
project-name: force
requires:
- push-staging-image
- production-image-push

# Release
- horizon/block:
<<: *only_release
context: horizon
project_id: 11

- validate_production_schema:
<<: *only_release

- hokusai/deploy-production:
<<: *only_release
name: deploy-production
requires:
- horizon/block
- validate_production_schema

# Other
- run_deepcrawl_automator:
context: deepcrawl-automator
requires:
Expand Down
51 changes: 32 additions & 19 deletions .dockerignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,23 +1,36 @@
.git
.gitignore
.dockerignore
Dockerfile
hokusai
# Prefer excluding all assets by default. It prevents large artifacts from
# accidentally becomming part of the Docker context.
*

README.md
LICENSE
VERSION
docs
# CI
!scripts/

.DS_Store
.AppleDouble
.LSOverride
# Source
!data/
!patches/
!src/
!webpack/

.env
.env.*
# Testing
!__mocks__/
!cypress/
!.env.oss
log
tmp
node_modules
.vscode
redis.conf
!.env.test
!cypress.json
!jest.config.v1.js
!jest.config.v2.js
!test.config.js
!test.mocha.js

# Build Configuration
!.eslintrc.js
!.nvmrc
!.prettierignore
!apollo.config.js
!babel.config.js
!coffeelint.json
!dangerfile.ts
!package.json
!relay.config.js
!tsconfig.json
!yarn.lock
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is one of those things that may surprise developers with unexpected CI failures when "everything worked locally." It's acceptable but something to monitor...

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree, however as this pattern is implemented in more places it should become second nature to check this when adding new folders. I would expect most new files to end up in src.

Loading