From c03e6a0e50930fc5f7358177a32a092fc45272ca Mon Sep 17 00:00:00 2001 From: Marcus Cobden Date: Mon, 24 Sep 2018 14:26:27 +0100 Subject: [PATCH] Rework UI build to improve caching and fix packaging issue Stop baking the toolchain and dependencies into the build image. Instead, run the install step each time, but use volume mounts or CircleCI caching to keep the happy path fast. Previously, some parts of the client (UI) directory were baked into the build image, and some parts were mounted or copied into the build environment. As a result, files baked into the build image require a two-step update for changes to take effect in CI. Now, for dockerised builds, we pre-install very little into the build image and mount the whole directory into the build environment. However, we do overlay a volume on the node_modules folder to allow the standard build toolchain to be separate from the host build toolchain. Non-dockerised builds (e.g. CI) are now more similar to the dockerised versions. --- .circleci/config.yml | 29 +++++++++++++++++++++-------- Makefile | 43 ++++++++++++++++++++++++++++--------------- client/Dockerfile | 14 ++++++++------ client/package.json | 2 +- 4 files changed, 58 insertions(+), 30 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index abbf4c81c..f5f11e2c8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -62,11 +62,19 @@ jobs: <<: *client-defaults steps: - checkout - # Convoluted set of steps here to mimic Makefile actions of bind-mounting different dirs into container - - run: mv client/app /home/weave/ - - run: cd /home/weave; mkdir build ; yarn run build ; mv build scope/client - - run: cd /home/weave; mkdir build-external; yarn run build-external; mv build-external scope/client - - run: cd /home/weave; mkdir tmp ; yarn run bundle ; mv tmp scope + - restore_cache: + keys: + - yarn-cache-{{ checksum "client/yarn.lock" }} + - run: cd client; yarn install + - save_cache: + key: yarn-cache-{{ checksum "client/yarn.lock" }} + paths: + - "/home/weave/scope/.cache/yarn" + - run: | + cd client + yarn run build + yarn run build-external + yarn run bundle - persist_to_workspace: root: /home/weave/scope paths: @@ -74,14 +82,19 @@ jobs: - client/build-external/ - tmp/weave-scope.tgz + client-test: <<: *client-defaults steps: - checkout + - restore_cache: + keys: + - yarn-cache-{{ checksum "client/yarn.lock" }} - run: | - mv client/app client/test /home/weave/ - cd /home/weave; yarn run lint - cd /home/weave; yarn test + cd client + yarn install + yarn run lint + yarn test xplatform-build: <<: *defaults diff --git a/Makefile b/Makefile index f42be4121..5930f8055 100644 --- a/Makefile +++ b/Makefile @@ -146,39 +146,52 @@ ifeq ($(BUILD_IN_CONTAINER),true) client/build/index.html: $(shell find client/app -type f) $(SCOPE_UI_BUILD_UPTODATE) mkdir -p client/build if test "true" != "$(SCOPE_SKIP_UI_ASSETS)"; then \ - $(SUDO) docker run $(RM) $(RUN_FLAGS) -v $(shell pwd)/client/app:/home/weave/app \ - -v $(shell pwd)/client/build:/home/weave/build \ - $(SCOPE_UI_BUILD_IMAGE) yarn run build; \ + $(SUDO) docker run $(RM) $(RUN_FLAGS) \ + -v $(shell pwd)/.cache:/home/weave/scope/.cache \ + -v $(shell pwd)/client:/home/weave/scope/client \ + -v $(shell pwd)/.cache/build_node_modules:/home/weave/scope/client/node_modules \ + $(SCOPE_UI_BUILD_IMAGE) sh -c 'yarn install && yarn run build'; \ fi client/build-external/index.html: $(shell find client/app -type f) $(SCOPE_UI_BUILD_UPTODATE) mkdir -p client/build-external if test "true" != "$(SCOPE_SKIP_UI_ASSETS)"; then \ - $(SUDO) docker run $(RM) $(RUN_FLAGS) -v $(shell pwd)/client/app:/home/weave/app \ - -v $(shell pwd)/client/build-external:/home/weave/build-external \ - $(SCOPE_UI_BUILD_IMAGE) yarn run build-external; \ + $(SUDO) docker run $(RM) $(RUN_FLAGS) \ + -v $(shell pwd)/.cache:/home/weave/scope/.cache \ + -v $(shell pwd)/client:/home/weave/scope/client \ + -v $(shell pwd)/.cache/build_node_modules:/home/weave/scope/client/node_modules \ + $(SCOPE_UI_BUILD_IMAGE) sh -c 'yarn install && yarn run build-external'; \ fi client-test: $(shell find client/app/scripts -type f) $(SCOPE_UI_BUILD_UPTODATE) - $(SUDO) docker run $(RM) $(RUN_FLAGS) -v $(shell pwd)/client/app:/home/weave/app \ - -v $(shell pwd)/client/test:/home/weave/test \ + $(SUDO) docker run $(RM) $(RUN_FLAGS) \ + -v $(shell pwd)/.cache:/home/weave/scope/.cache \ + -v $(shell pwd)/client/client:/home/weave/scope/client \ + -v $(shell pwd)/.cache/build_node_modules:/home/weave/scope/client/node_modules \ $(SCOPE_UI_BUILD_IMAGE) yarn test client-lint: $(SCOPE_UI_BUILD_UPTODATE) - $(SUDO) docker run $(RM) $(RUN_FLAGS) -v $(shell pwd)/client/app:/home/weave/app \ - -v $(shell pwd)/client/test:/home/weave/test \ + $(SUDO) docker run $(RM) $(RUN_FLAGS) \ + -v $(shell pwd)/.cache:/home/weave/scope/.cache \ + -v $(shell pwd)/client:/home/weave/scope/client \ + -v $(shell pwd)/.cache/build_node_modules:/home/weave/scope/client/node_modules \ $(SCOPE_UI_BUILD_IMAGE) yarn run lint client-start: $(SCOPE_UI_BUILD_UPTODATE) - $(SUDO) docker run $(RM) $(RUN_FLAGS) --net=host -v $(shell pwd)/client/app:/home/weave/app \ - -v $(shell pwd)/client/build:/home/weave/build -e WEBPACK_SERVER_HOST \ + $(SUDO) docker run $(RM) $(RUN_FLAGS) --net=host \ + -v $(shell pwd)/.cache:/home/weave/scope/.cache \ + -v $(shell pwd)/client:/home/weave/scope/client \ + -v $(shell pwd)/.cache/build_node_modules:/home/weave/scope/client/node_modules \ + -e WEBPACK_SERVER_HOST \ $(SCOPE_UI_BUILD_IMAGE) yarn start tmp/weave-scope.tgz: $(shell find client/app -type f) $(SCOPE_UI_BUILD_UPTODATE) $(sudo) docker run $(RUN_FLAGS) \ - -v $(shell pwd)/client/app:/home/weave/app \ - -v $(shell pwd)/tmp:/home/weave/tmp \ - $(SCOPE_UI_BUILD_IMAGE) \ + -v $(shell pwd)/.cache:/home/weave/scope/.cache \ + -v $(shell pwd)/client:/home/weave/scope/client \ + -v $(shell pwd)/.cache/build_node_modules:/home/weave/scope/client/node_modules \ + -v $(shell pwd)/tmp:/home/weave/tmp \ + $(SCOPE_UI_BUILD_IMAGE) \ yarn run bundle else diff --git a/client/Dockerfile b/client/Dockerfile index 4c19dc2f2..f13d44c73 100644 --- a/client/Dockerfile +++ b/client/Dockerfile @@ -1,13 +1,15 @@ +# Changes to this file will not take effect in CI +# until the image version in the CI config is updated. See +# https://github.com/weaveworks/scope/blob/master/.circleci/config.yml#L11 FROM node:8.11 -WORKDIR /home/weave -COPY package.json yarn.lock /home/weave/ -ENV NPM_CONFIG_LOGLEVEL=warn NPM_CONFIG_PROGRESS=false -RUN yarn --pure-lockfile -COPY webpack.local.config.js webpack.production.config.js server.js .babelrc .eslintrc .eslintignore .stylelintrc .sass-lint.yml /home/weave/ +ENV NPM_CONFIG_LOGLEVEL=warn +ENV NPM_CONFIG_PROGRESS=false +ENV XDG_CACHE_HOME=/home/weave/scope/.cache +WORKDIR /home/weave/scope/client ARG revision LABEL maintainer="Weaveworks " \ - org.opencontainers.image.title="client" \ + org.opencontainers.image.title="scope-ui-build" \ org.opencontainers.image.source="https://github.com/weaveworks/scope" \ org.opencontainers.image.revision="${revision}" \ org.opencontainers.image.vendor="Weaveworks" diff --git a/client/package.json b/client/package.json index 84358a277..97169f7ba 100644 --- a/client/package.json +++ b/client/package.json @@ -102,7 +102,7 @@ "build-external": "EXTERNAL=true webpack --config webpack.production.config.js", "copy-pkg-files": "cp package.json build-pkg/ && cp -R app/styles build-pkg/", "build-pkg": "mkdir -p build-pkg && node node_modules/.bin/babel app/scripts --ignore __tests__ --out-dir build-pkg && yarn run copy-pkg-files", - "bundle": "yarn run build-pkg && cd ./build-pkg && yarn pack --filename ../tmp/weave-scope.tgz", + "bundle": "mkdir -p tmp && yarn run build-pkg && cd ./build-pkg && yarn pack --filename ../tmp/weave-scope.tgz", "start": "node server.js", "start-production": "NODE_ENV=production node server.js", "test": "jest",