Rework UI build to improve caching and fix packaging issue

.circleci/config.yml

@@ -62,26 +62,52 @@ jobs:
     <<: *client-defaults
     steps:
       - checkout
-      # Convoluted set of steps here to mimic Makefile actions of bind-mounting different dirs into container
-      - run: mv client/app /home/weave/
-      - run: cd /home/weave; mkdir build         ; yarn run build         ; mv build          scope/client
-      - run: cd /home/weave; mkdir build-external; yarn run build-external; mv build-external scope/client
-      - run: cd /home/weave; mkdir tmp           ; yarn run bundle        ; mv tmp            scope
+      - restore_cache:
+          name: Restoring Yarn Cache
+          key: yarn-cache-{{ checksum "client/yarn.lock" }}
+      - restore_cache:
+          name: Restoring client/node_modules
+          key: node-modules-{{ checksum "client/yarn.lock" }}-{{ checksum ".circleci/config.yml" }}
+      - run: cd client; yarn install
+      - save_cache:
+          name: Saving Yarn Cache
+          key: yarn-cache-{{ checksum "client/yarn.lock" }}
+          paths:
+            - "/home/weave/scope/.cache/yarn"
+      - save_cache:
+          name: Saving client/node_modules
+          # include the CI config in the checksum because it will change when the docker image changes
+          key: node-modules-{{ checksum "client/yarn.lock" }}-{{ checksum ".circleci/config.yml" }}
+          paths:
+            - "/home/weave/scope/client/node_modules"
+      - run: |
+          cd client
+          yarn run build
+          yarn run build-external
+          yarn run bundle
       - persist_to_workspace:
           root: /home/weave/scope
           paths:
           - client/build/
           - client/build-external/
           - tmp/weave-scope.tgz
 
+
   client-test:
     <<: *client-defaults
     steps:
       - checkout
+      - restore_cache:
+          name: Restoring Yarn Cache
+          key: yarn-cache-{{ checksum "client/yarn.lock" }}
+      - restore_cache:
+          name: Restoring client/node_modules
+          key: node-modules-{{ checksum "client/yarn.lock" }}-{{ checksum ".circleci/config.yml" }}
       - run: |
-          mv client/app client/test /home/weave/
-          cd /home/weave; yarn run lint
-          cd /home/weave; yarn test
+          cd client
+          yarn install
+          yarn run lint
+          yarn test
 
   xplatform-build:
     <<: *defaults

Makefile

@@ -143,51 +143,90 @@ endif
 
 ifeq ($(BUILD_IN_CONTAINER),true)
 
-client/build/index.html: $(shell find client/app -type f) $(SCOPE_UI_BUILD_UPTODATE)
+SCOPE_UI_TOOLCHAIN=.cache/build_node_modules
+SCOPE_UI_TOOLCHAIN_UPTODATE=$(SCOPE_UI_TOOLCHAIN)/.uptodate
+
+$(SCOPE_UI_TOOLCHAIN_UPTODATE): client/yarn.lock $(SCOPE_UI_BUILD_UPTODATE)
+	if test "true" != "$(SCOPE_SKIP_UI_ASSETS)"; then \
+		$(SUDO) docker run $(RM) $(RUN_FLAGS) \
+			-v $(shell pwd)/.cache:/home/weave/scope/.cache \
+			-v $(shell pwd)/client:/home/weave/scope/client \
+			-v $(shell pwd)/$(SCOPE_UI_TOOLCHAIN):/home/weave/scope/client/node_modules \
+			-w /home/weave/scope/client \
+			$(SCOPE_UI_BUILD_IMAGE) yarn install; \
+	fi
+	touch $(SCOPE_UI_TOOLCHAIN_UPTODATE)
+
+client/build/index.html: $(shell find client/app -type f) $(SCOPE_UI_TOOLCHAIN_UPTODATE)
 	mkdir -p client/build
 	if test "true" != "$(SCOPE_SKIP_UI_ASSETS)"; then \
-		$(SUDO) docker run $(RM) $(RUN_FLAGS) -v $(shell pwd)/client/app:/home/weave/app \
-			-v $(shell pwd)/client/build:/home/weave/build \
+		$(SUDO) docker run $(RM) $(RUN_FLAGS) \
+			-v $(shell pwd)/.cache:/home/weave/scope/.cache \
+			-v $(shell pwd)/client:/home/weave/scope/client \
+			-v $(shell pwd)/$(SCOPE_UI_TOOLCHAIN):/home/weave/scope/client/node_modules \
+			-w /home/weave/scope/client \
 			$(SCOPE_UI_BUILD_IMAGE) yarn run build; \
 	fi
 
-client/build-external/index.html: $(shell find client/app -type f) $(SCOPE_UI_BUILD_UPTODATE)
+client/build-external/index.html: $(shell find client/app -type f) $(SCOPE_UI_TOOLCHAIN_UPTODATE)
 	mkdir -p client/build-external
 	if test "true" != "$(SCOPE_SKIP_UI_ASSETS)"; then \
-		$(SUDO) docker run $(RM) $(RUN_FLAGS) -v $(shell pwd)/client/app:/home/weave/app \
-			-v $(shell pwd)/client/build-external:/home/weave/build-external \
+		$(SUDO) docker run $(RM) $(RUN_FLAGS) \
+			-v $(shell pwd)/.cache:/home/weave/scope/.cache \
+			-v $(shell pwd)/client:/home/weave/scope/client \
+			-v $(shell pwd)/$(SCOPE_UI_TOOLCHAIN):/home/weave/scope/client/node_modules \
+			-w /home/weave/scope/client \
 			$(SCOPE_UI_BUILD_IMAGE) yarn run build-external; \
 	fi
 
-client-test: $(shell find client/app/scripts -type f) $(SCOPE_UI_BUILD_UPTODATE)
-	$(SUDO) docker run $(RM) $(RUN_FLAGS) -v $(shell pwd)/client/app:/home/weave/app \
-		-v $(shell pwd)/client/test:/home/weave/test \
+client-test: $(shell find client/app/scripts -type f) $(SCOPE_UI_TOOLCHAIN_UPTODATE)
+	$(SUDO) docker run $(RM) $(RUN_FLAGS) \
+		-v $(shell pwd)/.cache:/home/weave/scope/.cache \
+		-v $(shell pwd)/client/client:/home/weave/scope/client \
+		-v $(shell pwd)/$(SCOPE_UI_TOOLCHAIN):/home/weave/scope/client/node_modules \
+		-w /home/weave/scope/client \
 		$(SCOPE_UI_BUILD_IMAGE) yarn test
 
-client-lint: $(SCOPE_UI_BUILD_UPTODATE)
-	$(SUDO) docker run $(RM) $(RUN_FLAGS) -v $(shell pwd)/client/app:/home/weave/app \
-		-v $(shell pwd)/client/test:/home/weave/test \
+client-lint: $(SCOPE_UI_TOOLCHAIN_UPTODATE)
+	$(SUDO) docker run $(RM) $(RUN_FLAGS) \
+		-v $(shell pwd)/.cache:/home/weave/scope/.cache \
+		-v $(shell pwd)/client:/home/weave/scope/client \
+		-v $(shell pwd)/$(SCOPE_UI_TOOLCHAIN):/home/weave/scope/client/node_modules \
+		-w /home/weave/scope/client \
 		$(SCOPE_UI_BUILD_IMAGE) yarn run lint
 
-client-start: $(SCOPE_UI_BUILD_UPTODATE)
-	$(SUDO) docker run $(RM) $(RUN_FLAGS) --net=host -v $(shell pwd)/client/app:/home/weave/app \
-		-v $(shell pwd)/client/build:/home/weave/build -e WEBPACK_SERVER_HOST \
+client-start: $(SCOPE_UI_TOOLCHAIN_UPTODATE)
+	$(SUDO) docker run $(RM) $(RUN_FLAGS) --net=host \
+		-v $(shell pwd)/.cache:/home/weave/scope/.cache \
+		-v $(shell pwd)/client:/home/weave/scope/client \
+		-v $(shell pwd)/$(SCOPE_UI_TOOLCHAIN):/home/weave/scope/client/node_modules \
+		-e WEBPACK_SERVER_HOST \
+		-w /home/weave/scope/client \
 		$(SCOPE_UI_BUILD_IMAGE) yarn start
 
-tmp/weave-scope.tgz: $(shell find client/app -type f) $(SCOPE_UI_BUILD_UPTODATE)
+tmp/weave-scope.tgz: $(shell find client/app -type f) $(SCOPE_UI_TOOLCHAIN_UPTODATE)
 	$(sudo) docker run $(RUN_FLAGS) \
-	-v $(shell pwd)/client/app:/home/weave/app \
-	-v $(shell pwd)/tmp:/home/weave/tmp \
-	$(SCOPE_UI_BUILD_IMAGE) \
-	yarn run bundle
+		-v $(shell pwd)/.cache:/home/weave/scope/.cache \
+		-v $(shell pwd)/client:/home/weave/scope/client \
+		-v $(shell pwd)/$(SCOPE_UI_TOOLCHAIN):/home/weave/scope/client/node_modules \
+		-v $(shell pwd)/tmp:/home/weave/tmp \
+		-w /home/weave/scope/client \
+		$(SCOPE_UI_BUILD_IMAGE) yarn run bundle
 
 else
 
-client/build/index.html:
+SCOPE_UI_TOOLCHAIN=client/node_modules
+SCOPE_UI_TOOLCHAIN_UPTODATE=$(SCOPE_UI_TOOLCHAIN)/.uptodate
+
+$(SCOPE_UI_TOOLCHAIN_UPTODATE): client/yarn.lock
+	if test "true" = "$(SCOPE_SKIP_UI_ASSETS)"; then mkdir -p $(SCOPE_UI_TOOLCHAIN); else cd client && yarn install; fi
+	touch $(SCOPE_UI_TOOLCHAIN_UPTODATE)
+
+client/build/index.html: $(SCOPE_UI_TOOLCHAIN_UPTODATE)
 	mkdir -p client/build
 	if test "true" != "$(SCOPE_SKIP_UI_ASSETS)"; then cd client && yarn run build; fi
 
-client/build-external/index.html:
+client/build-external/index.html: $(SCOPE_UI_TOOLCHAIN_UPTODATE)
 	mkdir -p client/build-external
 	if test "true" != "$(SCOPE_SKIP_UI_ASSETS)"; then cd client && yarn run build-external; fi
 
@@ -220,7 +259,7 @@ ui-pkg-upload: tmp/weave-scope.tgz
 # rmi'ng images is desirable sometimes. Invoke `realclean` for that.
 clean:
 	$(GO) clean ./...
-	rm -rf $(SCOPE_EXPORT) $(SCOPE_UI_BUILD_UPTODATE) $(SCOPE_BACKEND_BUILD_UPTODATE) \
+	rm -rf $(SCOPE_EXPORT) $(SCOPE_UI_BUILD_UPTODATE) $(SCOPE_UI_TOOLCHAIN_UPTODATE) $(SCOPE_BACKEND_BUILD_UPTODATE) \
 		$(SCOPE_EXE) $(RUNSVINIT) prog/staticui/staticui.go prog/externalui/externalui.go client/build/*.js client/build-external/*.js docker/weave .pkg \
 		$(CODECGEN_TARGETS) $(CODECGEN_DIR)/bin
 
@@ -236,6 +275,7 @@ clean-codecgen:
 #
 # Doing this is important for release builds.
 realclean: clean
+	rm -rf $(SCOPE_UI_TOOLCHAIN)
 	$(SUDO) docker rmi -f $(SCOPE_UI_BUILD_IMAGE) $(SCOPE_BACKEND_BUILD_IMAGE) \
 		$(DOCKERHUB_USER)/scope $(DOCKERHUB_USER)/cloud-agent \
 		$(DOCKERHUB_USER)/scope:$(IMAGE_TAG) $(DOCKERHUB_USER)/cloud-agent:$(IMAGE_TAG) \

client/Dockerfile

@@ -1,13 +1,14 @@
+# Changes to this file will not take effect in CI
+# until the image version in the CI config is updated. See
+# https://github.com/weaveworks/scope/blob/master/.circleci/config.yml#L11
 FROM node:8.11
-WORKDIR /home/weave
-COPY package.json yarn.lock /home/weave/
-ENV NPM_CONFIG_LOGLEVEL=warn NPM_CONFIG_PROGRESS=false
-RUN yarn --pure-lockfile
-COPY webpack.local.config.js webpack.production.config.js server.js .babelrc .eslintrc .eslintignore .stylelintrc .sass-lint.yml /home/weave/
+ENV NPM_CONFIG_LOGLEVEL=warn
+ENV NPM_CONFIG_PROGRESS=false
+ENV XDG_CACHE_HOME=/home/weave/scope/.cache
 
 ARG revision
 LABEL maintainer="Weaveworks <help@weave.works>" \
-      org.opencontainers.image.title="client" \
+      org.opencontainers.image.title="scope-ui-build" \
       org.opencontainers.image.source="https://github.com/weaveworks/scope" \
       org.opencontainers.image.revision="${revision}" \
       org.opencontainers.image.vendor="Weaveworks"

client/package.json

@@ -102,7 +102,7 @@
     "build-external": "EXTERNAL=true webpack --config webpack.production.config.js",
     "copy-pkg-files": "cp package.json build-pkg/ && cp -R app/styles build-pkg/",
     "build-pkg": "mkdir -p build-pkg && node node_modules/.bin/babel app/scripts --ignore __tests__ --out-dir build-pkg && yarn run copy-pkg-files",
-    "bundle": "yarn run build-pkg && cd ./build-pkg && yarn pack --filename ../tmp/weave-scope.tgz",
+    "bundle": "mkdir -p tmp && yarn run build-pkg && cd ./build-pkg && yarn pack --filename ../tmp/weave-scope.tgz",
     "start": "node server.js",
     "start-production": "NODE_ENV=production node server.js",
     "test": "jest",