From 6b4a51563f0b6d755d884a4fa6dbbada4c49c8d7 Mon Sep 17 00:00:00 2001 From: guyfedwards Date: Wed, 8 Aug 2018 13:48:37 +0100 Subject: [PATCH 01/22] initial --- .circleci/config.yml | 127 +++++++++++++++++++++++++++++++++++++++++++ circle.yml | 117 --------------------------------------- 2 files changed, 127 insertions(+), 117 deletions(-) create mode 100644 .circleci/config.yml delete mode 100644 circle.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..3ab10a4d5 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,127 @@ +version: 2 + +defaults: &defaults + steps: + - run: + name: Setup Environment Variables + command: | + echo 'export GOPATH="/home/ubuntu"' >> $BASH_ENV + echo 'export SRCDIR="/home/ubuntu/src/github.com/weaveworks/scope"' >> $BASH_ENV + echo 'export PATH="$PATH:$HOME/.local/bin"' >> $BASH_ENV + echo 'export CLOUDSDK_CORE_DISABLE_PROMPTS="1"' >> $BASH_ENV + echo 'export SCOPE_UI_BUILD="$HOME/docker/scope_ui_build.tar"' >> $BASH_ENV + echo 'export IMAGES="scope cloud-agent"' >> $BASH_ENV +jobs: + build: + <<: *defaults + machine: true + steps: + - checkout + + - run: curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0 + - run: pip install --upgrade requests + - run: | + sudo apt-get update && + sudo apt-get install jq pv && + sudo chmod a+wr --recursive /usr/local/go/pkg && + (curl https://sdk.cloud.google.com | bash) && + (test -z "$SECRET_PASSWORD" || bin/setup-circleci-secrets "$SECRET_PASSWORD") && + make deps && + mkdir -p $(dirname $SRCDIR) && + cp -r $(pwd)/ $SRCDIR + - run: "cd $SRCDIR/client; ../tools/rebuild-image weaveworks/scope-ui-build . Dockerfile package.json webpack.production.config.js .eslintrc .babelrc && touch $SRCDIR/.scope_ui_build.uptodate" + - run: "cd $SRCDIR/backend; ../tools/rebuild-image weaveworks/scope-backend-build . Dockerfile build.sh && touch $SRCDIR/.scope_backend_build.uptodate" + - run: sudo apt-get update && sudo apt-get install python-pip && sudo pip install awscli + + - run: test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh make_template) + - run: cd $SRCDIR; make RM= lint + - run: cd $SRCDIR; COVERDIR=./coverage make RM= CODECGEN_UID=23 tests + - run: cd $SRCDIR; make RM= client-test static + - run: cd $SRCDIR; make RM= client-lint static + - run: cd $SRCDIR; rm -f prog/scope; if [ "$CIRCLE_NODE_INDEX" = "0" ]; then GOARCH=arm make GO_BUILD_INSTALL_DEPS= RM= prog/scope; else GOOS=darwin make GO_BUILD_INSTALL_DEPS= RM= prog/scope; fi + - run: cd $SRCDIR; rm -f prog/scope; make RM= + - run: cd $SRCDIR/extras; ./build_on_circle.sh + - run: "test -z \"$SECRET_PASSWORD\" || (cd $SRCDIR/integration; ./gce.sh setup && eval $(./gce.sh hosts); ./setup.sh)" + - run: + command: test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; eval $(./gce.sh hosts); ./run_all.sh) + no_output_timeout: 5m + - run: test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh destroy) + - run: test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR; ./tools/cover/gather_coverage.sh ./coverage $SRCDIR/coverage) + - run: test "$CIRCLE_NODE_INDEX" != "0" || (goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/profile.cov -service=circleci || true) + - run: test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR; cp */*.codecgen.go $CIRCLE_ARTIFACTS) + + # machine: + # pre: + # - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0 + # services: + # - docker + + deploy: + <<: *defaults + steps: + - checkout + - run: | + test -z "${DOCKER_USER}" || ( + docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && + (test "${DOCKER_ORGANIZATION:-$DOCKER_USER}" = "weaveworks" || ( + for IMAGE in $IMAGES; do + docker tag weaveworks/$IMAGE:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:latest && + docker tag weaveworks/$IMAGE:$(./tools/image-tag) ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:$(./tools/image-tag) + done + )) && + for IMAGE in $IMAGES; do + docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE && + docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:$(./tools/image-tag) + done + ) + - run: | + test -z "${QUAY_USER}" || ( + docker login -e '.' -u "$QUAY_USER" -p "$QUAY_PASSWORD" quay.io && + docker tag weaveworks/scope:$(./tools/image-tag) "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)" && + docker push "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)" + ) + - run: test -z "${UI_BUCKET_KEY_ID}" || (cd $SRCDIR && make ui-upload && make ui-pkg-upload) + + deploy-dev: + <<: *defaults + steps: + - checkout + - run: > + test -z "${DEPLOY_BRANCH}" || test -z "${DOCKER_USER}" || ( + docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && + docker tag weaveworks/scope:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope:${CIRCLE_BRANCH//\//-} && + docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope:${CIRCLE_BRANCH//\//-} + ) +workflows: + version: 2 + test_and_deploy: + jobs: + - build: + filters: + branches: + ignore: gh-pages + - deploy: + filters: + branches: + only: master + requires: + - build + - deploy-dev: + filters: + branches: + ignore: master + requires: + - build + + # release: + # branch: /release-[0-9]+\.[0-9]+/ + # owner: weaveworks + # commands: + # - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get + # - cd $SRCDIR; PRODUCT=scope tools/publish-site "$WP_LIVE_URL" "$WP_LIVE_USER" "$WP_LIVE_PASSWORD" + # issues: + # branch: /.*/ + # owner: weaveworks + # commands: + # - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get + # - cd $SRCDIR; PRODUCT=scope tools/publish-site "$WP_DEV_URL" "$WP_DEV_USER" "$WP_DEV_PASSWORD" diff --git a/circle.yml b/circle.yml deleted file mode 100644 index b334fa972..000000000 --- a/circle.yml +++ /dev/null @@ -1,117 +0,0 @@ -general: - branches: - ignore: - - gh-pages - -machine: - pre: - - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0 - services: - - docker - environment: - GOPATH: /home/ubuntu - SRCDIR: /home/ubuntu/src/github.com/weaveworks/scope - PATH: $PATH:$HOME/.local/bin - CLOUDSDK_CORE_DISABLE_PROMPTS: 1 - SCOPE_UI_BUILD: $HOME/docker/scope_ui_build.tar - IMAGES: scope cloud-agent - -dependencies: - pre: - - pip install --upgrade requests - cache_directories: - - "~/docker" - override: - - | - sudo apt-get update && - sudo apt-get install jq pv && - sudo chmod a+wr --recursive /usr/local/go/pkg && - sudo chown ubuntu:ubuntu "$HOME/.bashrc.backup" - (curl https://sdk.cloud.google.com | bash) && - (test -z "$SECRET_PASSWORD" || bin/setup-circleci-secrets "$SECRET_PASSWORD") && - make deps && - mkdir -p $(dirname $SRCDIR) && - cp -r $(pwd)/ $SRCDIR - - "cd $SRCDIR/client; ../tools/rebuild-image weaveworks/scope-ui-build . Dockerfile package.json webpack.production.config.js .eslintrc .babelrc && touch $SRCDIR/.scope_ui_build.uptodate" - - "cd $SRCDIR/backend; ../tools/rebuild-image weaveworks/scope-backend-build . Dockerfile build.sh && touch $SRCDIR/.scope_backend_build.uptodate" - - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh make_template): - parallel: false - - sudo apt-get update && sudo apt-get install python-pip && sudo pip install awscli - -test: - override: - - cd $SRCDIR; make RM= lint: - parallel: true - - cd $SRCDIR; COVERDIR=./coverage make RM= CODECGEN_UID=23 tests: - parallel: true - - cd $SRCDIR; make RM= client-test static: - parallel: true - - cd $SRCDIR; make RM= client-lint static: - parallel: true - - cd $SRCDIR; rm -f prog/scope; if [ "$CIRCLE_NODE_INDEX" = "0" ]; then GOARCH=arm make GO_BUILD_INSTALL_DEPS= RM= prog/scope; else GOOS=darwin make GO_BUILD_INSTALL_DEPS= RM= prog/scope; fi: - parallel: true - - cd $SRCDIR; rm -f prog/scope; make RM=: - parallel: true - - cd $SRCDIR/extras; ./build_on_circle.sh: - parallel: true - - "test -z \"$SECRET_PASSWORD\" || (cd $SRCDIR/integration; ./gce.sh setup && eval $(./gce.sh hosts); ./setup.sh)": - parallel: true - - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; eval $(./gce.sh hosts); ./run_all.sh): - parallel: true - timeout: 300 - post: - - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh destroy): - parallel: true - - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR; ./tools/cover/gather_coverage.sh ./coverage $SRCDIR/coverage): - parallel: true - - test "$CIRCLE_NODE_INDEX" != "0" || (goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/profile.cov -service=circleci || true): - parallel: true - - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR; cp */*.codecgen.go $CIRCLE_ARTIFACTS): - parallel: true - -deployment: - hub: - branch: master - commands: - - | - test -z "${DOCKER_USER}" || ( - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && - (test "${DOCKER_ORGANIZATION:-$DOCKER_USER}" = "weaveworks" || ( - for IMAGE in $IMAGES; do - docker tag weaveworks/$IMAGE:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:latest && - docker tag weaveworks/$IMAGE:$(./tools/image-tag) ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:$(./tools/image-tag) - done - )) && - for IMAGE in $IMAGES; do - docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE && - docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:$(./tools/image-tag) - done - ) - - | - test -z "${QUAY_USER}" || ( - docker login -e '.' -u "$QUAY_USER" -p "$QUAY_PASSWORD" quay.io && - docker tag weaveworks/scope:$(./tools/image-tag) "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)" && - docker push "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)" - ) - - test -z "${UI_BUCKET_KEY_ID}" || (cd $SRCDIR && make ui-upload && make ui-pkg-upload) - hub-dev: - branch: /^((?!master).)*$/ # not the master branch - commands: - - > - test -z "${DEPLOY_BRANCH}" || test -z "${DOCKER_USER}" || ( - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && - docker tag weaveworks/scope:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope:${CIRCLE_BRANCH//\//-} && - docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope:${CIRCLE_BRANCH//\//-} - ) -# release: -# branch: /release-[0-9]+\.[0-9]+/ -# owner: weaveworks -# commands: -# - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get -# - cd $SRCDIR; PRODUCT=scope tools/publish-site "$WP_LIVE_URL" "$WP_LIVE_USER" "$WP_LIVE_PASSWORD" -# issues: -# branch: /.*/ -# owner: weaveworks -# commands: -# - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get -# - cd $SRCDIR; PRODUCT=scope tools/publish-site "$WP_DEV_URL" "$WP_DEV_USER" "$WP_DEV_PASSWORD" From fc7e5bf9f8f46a10c764d0df6202dc0b9c49cf2b Mon Sep 17 00:00:00 2001 From: guyfedwards Date: Wed, 15 Aug 2018 12:05:11 +0100 Subject: [PATCH 02/22] fix env defaults --- .circleci/config.yml | 86 ++++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3ab10a4d5..7cd83312c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,16 +1,14 @@ version: 2 defaults: &defaults - steps: - - run: - name: Setup Environment Variables - command: | - echo 'export GOPATH="/home/ubuntu"' >> $BASH_ENV - echo 'export SRCDIR="/home/ubuntu/src/github.com/weaveworks/scope"' >> $BASH_ENV - echo 'export PATH="$PATH:$HOME/.local/bin"' >> $BASH_ENV - echo 'export CLOUDSDK_CORE_DISABLE_PROMPTS="1"' >> $BASH_ENV - echo 'export SCOPE_UI_BUILD="$HOME/docker/scope_ui_build.tar"' >> $BASH_ENV - echo 'export IMAGES="scope cloud-agent"' >> $BASH_ENV + environment: + GOPATH: /home/ubuntu + SRCDIR: /home/ubuntu/src/github.com/weaveworks/scope + PATH: $PATH:$HOME/.local/bin + CLOUDSDK_CORE_DISABLE_PROMPTS: 1 + SCOPE_UI_BUILD: $HOME/docker/scope_ui_build.tar + IMAGES: scope cloud-agent + jobs: build: <<: *defaults @@ -56,42 +54,44 @@ jobs: # services: # - docker - deploy: - <<: *defaults - steps: - - checkout - - run: | - test -z "${DOCKER_USER}" || ( - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && - (test "${DOCKER_ORGANIZATION:-$DOCKER_USER}" = "weaveworks" || ( - for IMAGE in $IMAGES; do - docker tag weaveworks/$IMAGE:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:latest && - docker tag weaveworks/$IMAGE:$(./tools/image-tag) ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:$(./tools/image-tag) - done - )) && + deploy: + <<: *defaults + machine: true + steps: + - checkout + - run: | + test -z "${DOCKER_USER}" || ( + docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && + (test "${DOCKER_ORGANIZATION:-$DOCKER_USER}" = "weaveworks" || ( for IMAGE in $IMAGES; do - docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE && - docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:$(./tools/image-tag) + docker tag weaveworks/$IMAGE:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:latest && + docker tag weaveworks/$IMAGE:$(./tools/image-tag) ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:$(./tools/image-tag) done - ) - - run: | - test -z "${QUAY_USER}" || ( - docker login -e '.' -u "$QUAY_USER" -p "$QUAY_PASSWORD" quay.io && - docker tag weaveworks/scope:$(./tools/image-tag) "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)" && - docker push "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)" - ) - - run: test -z "${UI_BUCKET_KEY_ID}" || (cd $SRCDIR && make ui-upload && make ui-pkg-upload) + )) && + for IMAGE in $IMAGES; do + docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE && + docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:$(./tools/image-tag) + done + ) + - run: | + test -z "${QUAY_USER}" || ( + docker login -e '.' -u "$QUAY_USER" -p "$QUAY_PASSWORD" quay.io && + docker tag weaveworks/scope:$(./tools/image-tag) "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)" && + docker push "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)" + ) + - run: test -z "${UI_BUCKET_KEY_ID}" || (cd $SRCDIR && make ui-upload && make ui-pkg-upload) - deploy-dev: - <<: *defaults - steps: - - checkout - - run: > - test -z "${DEPLOY_BRANCH}" || test -z "${DOCKER_USER}" || ( - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && - docker tag weaveworks/scope:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope:${CIRCLE_BRANCH//\//-} && - docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope:${CIRCLE_BRANCH//\//-} - ) + deploy-dev: + <<: *defaults + machine: true + steps: + - checkout + - run: > + test -z "${DEPLOY_BRANCH}" || test -z "${DOCKER_USER}" || ( + docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && + docker tag weaveworks/scope:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope:${CIRCLE_BRANCH//\//-} && + docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope:${CIRCLE_BRANCH//\//-} + ) workflows: version: 2 test_and_deploy: From 9efcea41b9ab49fc879a131543aaea6ee56c83c7 Mon Sep 17 00:00:00 2001 From: guyfedwards Date: Thu, 16 Aug 2018 10:09:04 +0100 Subject: [PATCH 03/22] add docker build image; --- .circleci/config.yml | 72 ++++++++++++++++++++---------------------- build-image/Dockerfile | 14 ++++++++ 2 files changed, 49 insertions(+), 37 deletions(-) create mode 100644 build-image/Dockerfile diff --git a/.circleci/config.yml b/.circleci/config.yml index 7cd83312c..26baff449 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,19 +4,42 @@ defaults: &defaults environment: GOPATH: /home/ubuntu SRCDIR: /home/ubuntu/src/github.com/weaveworks/scope - PATH: $PATH:$HOME/.local/bin CLOUDSDK_CORE_DISABLE_PROMPTS: 1 SCOPE_UI_BUILD: $HOME/docker/scope_ui_build.tar IMAGES: scope cloud-agent + docker: + image: weaveworks/scope-build-image:latest + +workflows: + version: 2 + test_and_deploy: + jobs: + - build: + filters: + branches: + ignore: gh-pages + - deploy: + filters: + branches: + only: master + requires: + - build + - deploy-dev: + filters: + branches: + ignore: master + requires: + - build jobs: build: <<: *defaults - machine: true steps: - checkout - - - run: curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0 + # PATH must be set here as circle currently does not support string interpolation + # for environment + # https://discuss.circleci.com/t/bin-sh-mkdir-command-not-found/8710 + - run: export PATH="$PATH:$HOME/.local/bin" - run: pip install --upgrade requests - run: | sudo apt-get update && @@ -59,6 +82,10 @@ jobs: machine: true steps: - checkout + # PATH must be set here as circle currently does not support string interpolation + # for environment + # https://discuss.circleci.com/t/bin-sh-mkdir-command-not-found/8710 + - run: export PATH="$PATH:$HOME/.local/bin" - run: | test -z "${DOCKER_USER}" || ( docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && @@ -86,42 +113,13 @@ jobs: machine: true steps: - checkout + # PATH must be set here as circle currently does not support string interpolation + # for environment + # https://discuss.circleci.com/t/bin-sh-mkdir-command-not-found/8710 + - run: export PATH="$PATH:$HOME/.local/bin" - run: > test -z "${DEPLOY_BRANCH}" || test -z "${DOCKER_USER}" || ( docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && docker tag weaveworks/scope:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope:${CIRCLE_BRANCH//\//-} && docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope:${CIRCLE_BRANCH//\//-} ) -workflows: - version: 2 - test_and_deploy: - jobs: - - build: - filters: - branches: - ignore: gh-pages - - deploy: - filters: - branches: - only: master - requires: - - build - - deploy-dev: - filters: - branches: - ignore: master - requires: - - build - - # release: - # branch: /release-[0-9]+\.[0-9]+/ - # owner: weaveworks - # commands: - # - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get - # - cd $SRCDIR; PRODUCT=scope tools/publish-site "$WP_LIVE_URL" "$WP_LIVE_USER" "$WP_LIVE_PASSWORD" - # issues: - # branch: /.*/ - # owner: weaveworks - # commands: - # - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get - # - cd $SRCDIR; PRODUCT=scope tools/publish-site "$WP_DEV_URL" "$WP_DEV_USER" "$WP_DEV_PASSWORD" diff --git a/build-image/Dockerfile b/build-image/Dockerfile new file mode 100644 index 000000000..3958cd8be --- /dev/null +++ b/build-image/Dockerfile @@ -0,0 +1,14 @@ +FROM golang:1.10.1-stretch + +RUN apt-get update && apt-get install jq pv \ + && chmod a+wr --recursive /usr/local/go/pkg \ + && (curl https://sdk.cloud.google.com | bash) \ + && pip install --upgrade requests awscli + + +ARG revision +LABEL maintainer="Weaveworks " \ + org.opencontainers.image.title="build-image" \ + org.opencontainers.image.source="https://github.com/weaveworks/scope/tree/master/build-image" \ + org.opencontainers.image.revision="${revision}" \ + org.opencontainers.image.vendor="Weaveworks" From 191b6c8c08a58c759abe886893c67a235ae7376e Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sat, 25 Aug 2018 10:08:49 +0000 Subject: [PATCH 04/22] Don't need a third build image --- build-image/Dockerfile | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 build-image/Dockerfile diff --git a/build-image/Dockerfile b/build-image/Dockerfile deleted file mode 100644 index 3958cd8be..000000000 --- a/build-image/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM golang:1.10.1-stretch - -RUN apt-get update && apt-get install jq pv \ - && chmod a+wr --recursive /usr/local/go/pkg \ - && (curl https://sdk.cloud.google.com | bash) \ - && pip install --upgrade requests awscli - - -ARG revision -LABEL maintainer="Weaveworks " \ - org.opencontainers.image.title="build-image" \ - org.opencontainers.image.source="https://github.com/weaveworks/scope/tree/master/build-image" \ - org.opencontainers.image.revision="${revision}" \ - org.opencontainers.image.vendor="Weaveworks" From 89d7ab1677d8bf3796d390f497f665d832cf96d8 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 26 Aug 2018 16:09:24 +0000 Subject: [PATCH 05/22] Remove unnecessary file --- client/build-external/.gitignore | 1 - 1 file changed, 1 deletion(-) delete mode 100644 client/build-external/.gitignore diff --git a/client/build-external/.gitignore b/client/build-external/.gitignore deleted file mode 100644 index a4b63e4d7..000000000 --- a/client/build-external/.gitignore +++ /dev/null @@ -1 +0,0 @@ -# this file is only present to force git to create this directory in the working tree From ec49a38b831d863da642df349d3120dcd560258e Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sat, 25 Aug 2018 15:28:02 +0000 Subject: [PATCH 06/22] Tag build images with git hash --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index b08fc43e6..e654ab914 100644 --- a/Makefile +++ b/Makefile @@ -195,10 +195,12 @@ endif $(SCOPE_UI_BUILD_UPTODATE): client/Dockerfile client/package.json client/webpack.local.config.js client/webpack.production.config.js client/server.js client/.eslintrc $(SUDO) docker build -t $(SCOPE_UI_BUILD_IMAGE) client + $(SUDO) docker tag $(SCOPE_UI_BUILD_IMAGE) $(SCOPE_UI_BUILD_IMAGE):$(IMAGE_TAG) touch $@ $(SCOPE_BACKEND_BUILD_UPTODATE): backend/* $(SUDO) docker build -t $(SCOPE_BACKEND_BUILD_IMAGE) backend + $(SUDO) docker tag $(SCOPE_BACKEND_BUILD_IMAGE) $(SCOPE_BACKEND_BUILD_IMAGE):$(IMAGE_TAG) touch $@ ui-upload: client/build-external/index.html From c1b93c3161571f77edf348ffc6f4e7516c87bc79 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sat, 25 Aug 2018 11:09:10 +0000 Subject: [PATCH 07/22] Break into steps, use current build images --- .circleci/config.yml | 171 +++++++++++++++++++++++++++++++------------ 1 file changed, 123 insertions(+), 48 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 26baff449..3bde7ad0f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,29 +1,43 @@ version: 2 defaults: &defaults - environment: - GOPATH: /home/ubuntu - SRCDIR: /home/ubuntu/src/github.com/weaveworks/scope - CLOUDSDK_CORE_DISABLE_PROMPTS: 1 - SCOPE_UI_BUILD: $HOME/docker/scope_ui_build.tar - IMAGES: scope cloud-agent + working_directory: /go/src/github.com/weaveworks/scope docker: - image: weaveworks/scope-build-image:latest + - image: weaveworks/scope-backend-build:circle-2.0-2885645 + +client-defaults: &client-defaults + working_directory: /home/weave + docker: + - image: weaveworks/scope-ui-build:circle-2.0-2885645 workflows: version: 2 test_and_deploy: jobs: - - build: - filters: - branches: - ignore: gh-pages + - lint + - unit-test + - client-lint + - client-test + - arm-build + - darwin-build + - build + - integration-tests: + requires: + - lint + - unit-test + - build + - gen-coverage: + requires: + - unit-test + - integration-tests - deploy: filters: branches: only: master requires: - - build + - client-lint + - client-test + - integration-tests - deploy-dev: filters: branches: @@ -32,61 +46,120 @@ workflows: - build jobs: + lint: + <<: *defaults + steps: + - checkout + - run: make BUILD_IN_CONTAINER=false lint + + unit-test: + <<: *defaults + parallelism: 1 + steps: + - checkout + - run: COVERDIR=./coverage make BUILD_IN_CONTAINER=false CODECGEN_UID=23 tests + - persist_to_workspace: + root: . + paths: + - coverage + + client-lint: + <<: *client-defaults + steps: + - checkout + - run: make BUILD_IN_CONTAINER=false client-lint + + client-test: + <<: *client-defaults + steps: + - checkout + - run: make BUILD_IN_CONTAINER=false client-test + + arm-build: + <<: *defaults + steps: + - checkout + - run: GOARCH=arm make BUILD_IN_CONTAINER=false GO_BUILD_INSTALL_DEPS= prog/scope + + darwin-build: + <<: *defaults + steps: + - checkout + - run: GOOS=darwin make BUILD_IN_CONTAINER=false GO_BUILD_INSTALL_DEPS= prog/scope + build: <<: *defaults steps: - checkout - # PATH must be set here as circle currently does not support string interpolation - # for environment - # https://discuss.circleci.com/t/bin-sh-mkdir-command-not-found/8710 - - run: export PATH="$PATH:$HOME/.local/bin" - - run: pip install --upgrade requests - - run: | - sudo apt-get update && - sudo apt-get install jq pv && - sudo chmod a+wr --recursive /usr/local/go/pkg && - (curl https://sdk.cloud.google.com | bash) && - (test -z "$SECRET_PASSWORD" || bin/setup-circleci-secrets "$SECRET_PASSWORD") && - make deps && - mkdir -p $(dirname $SRCDIR) && - cp -r $(pwd)/ $SRCDIR - - run: "cd $SRCDIR/client; ../tools/rebuild-image weaveworks/scope-ui-build . Dockerfile package.json webpack.production.config.js .eslintrc .babelrc && touch $SRCDIR/.scope_ui_build.uptodate" - - run: "cd $SRCDIR/backend; ../tools/rebuild-image weaveworks/scope-backend-build . Dockerfile build.sh && touch $SRCDIR/.scope_backend_build.uptodate" - - run: sudo apt-get update && sudo apt-get install python-pip && sudo pip install awscli + - setup_remote_docker + - run: make BUILD_IN_CONTAINER=false SUDO= all + - run: cd extras; make BUILD_IN_CONTAINER=false + - persist_to_workspace: + root: . + paths: + - scope.tar + - cloud-agent.tar + integration-tests: + machine: + image: circleci/classic:201709-01 + working_directory: /home/circleci/src/github.com/weaveworks/scope + environment: + SRCDIR: /home/circleci/src/github.com/weaveworks/scope + CIRCLE_ARTIFACTS: /tmp/artifacts + CLOUDSDK_CORE_DISABLE_PROMPTS: 1 + parallelism: 2 + steps: + - checkout + - attach_workspace: + at: . + # kick off creation of test VMs + - run: test -z "$SECRET_PASSWORD" || bin/setup-circleci-secrets "$SECRET_PASSWORD" - run: test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh make_template) - - run: cd $SRCDIR; make RM= lint - - run: cd $SRCDIR; COVERDIR=./coverage make RM= CODECGEN_UID=23 tests - - run: cd $SRCDIR; make RM= client-test static - - run: cd $SRCDIR; make RM= client-lint static - - run: cd $SRCDIR; rm -f prog/scope; if [ "$CIRCLE_NODE_INDEX" = "0" ]; then GOARCH=arm make GO_BUILD_INSTALL_DEPS= RM= prog/scope; else GOOS=darwin make GO_BUILD_INSTALL_DEPS= RM= prog/scope; fi - - run: cd $SRCDIR; rm -f prog/scope; make RM= - - run: cd $SRCDIR/extras; ./build_on_circle.sh - - run: "test -z \"$SECRET_PASSWORD\" || (cd $SRCDIR/integration; ./gce.sh setup && eval $(./gce.sh hosts); ./setup.sh)" + - run: test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh setup && eval $(./gce.sh hosts); ./setup.sh) - run: command: test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; eval $(./gce.sh hosts); ./run_all.sh) no_output_timeout: 5m - - run: test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh destroy) - - run: test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR; ./tools/cover/gather_coverage.sh ./coverage $SRCDIR/coverage) - - run: test "$CIRCLE_NODE_INDEX" != "0" || (goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/profile.cov -service=circleci || true) - - run: test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR; cp */*.codecgen.go $CIRCLE_ARTIFACTS) + # Destroy testing VMs: + - run: + command: test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh destroy) + background: true + - persist_to_workspace: + root: . + paths: + - coverage - # machine: - # pre: - # - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0 - # services: - # - docker + gen-coverage: + <<: *defaults + environment: + CIRCLE_ARTIFACTS: /tmp/artifacts + steps: + - checkout + - run: mkdir $CIRCLE_ARTIFACTS + - attach_workspace: + at: . + - run: ./tools/cover/gather_coverage.sh ./coverage $SRCDIR/coverage + - run: goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=profile.cov -service=circleci + - run: cp coverage.* */*.codecgen.go $CIRCLE_ARTIFACTS + - store_artifacts: + path: /tmp/artifacts deploy: <<: *defaults - machine: true + environment: + IMAGES: scope cloud-agent steps: - checkout + - setup_remote_docker + - attach_workspace: + at: . # PATH must be set here as circle currently does not support string interpolation # for environment # https://discuss.circleci.com/t/bin-sh-mkdir-command-not-found/8710 - run: export PATH="$PATH:$HOME/.local/bin" - run: | + docker load -i scope.tar + docker load -i cloud-agent.tar test -z "${DOCKER_USER}" || ( docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && (test "${DOCKER_ORGANIZATION:-$DOCKER_USER}" = "weaveworks" || ( @@ -110,9 +183,11 @@ jobs: deploy-dev: <<: *defaults - machine: true steps: - checkout + - setup_remote_docker + - attach_workspace: + at: . # PATH must be set here as circle currently does not support string interpolation # for environment # https://discuss.circleci.com/t/bin-sh-mkdir-command-not-found/8710 From c07b6b5ca8e797d4a5e668fff212a42b59dc1aa1 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sat, 25 Aug 2018 16:51:08 +0000 Subject: [PATCH 08/22] Sleep to stop TestRegistryDelete() failing --- probe/docker/registry_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/probe/docker/registry_test.go b/probe/docker/registry_test.go index 0e8e4bdcd..3868371d5 100644 --- a/probe/docker/registry_test.go +++ b/probe/docker/registry_test.go @@ -470,7 +470,7 @@ func TestRegistryDelete(t *testing.T) { setupStubs(mdc, func() { registry := testRegistry() defer registry.Stop() - runtime.Gosched() + time.Sleep(time.Millisecond * 100) // Allow for goroutines to get started // Collect all the events. mtx := sync.Mutex{} From fd08129e304a52a745c74179876ea380932aaf56 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sat, 25 Aug 2018 11:11:28 +0000 Subject: [PATCH 09/22] Remove deploy-dev job; it isn't used --- .circleci/config.yml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3bde7ad0f..8c7873b60 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,12 +38,6 @@ workflows: - client-lint - client-test - integration-tests - - deploy-dev: - filters: - branches: - ignore: master - requires: - - build jobs: lint: @@ -180,21 +174,3 @@ jobs: docker push "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)" ) - run: test -z "${UI_BUCKET_KEY_ID}" || (cd $SRCDIR && make ui-upload && make ui-pkg-upload) - - deploy-dev: - <<: *defaults - steps: - - checkout - - setup_remote_docker - - attach_workspace: - at: . - # PATH must be set here as circle currently does not support string interpolation - # for environment - # https://discuss.circleci.com/t/bin-sh-mkdir-command-not-found/8710 - - run: export PATH="$PATH:$HOME/.local/bin" - - run: > - test -z "${DEPLOY_BRANCH}" || test -z "${DOCKER_USER}" || ( - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && - docker tag weaveworks/scope:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope:${CIRCLE_BRANCH//\//-} && - docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope:${CIRCLE_BRANCH//\//-} - ) From d7b40846a4ddde95b84a83645da42c2f7a12c38e Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sat, 25 Aug 2018 15:30:08 +0000 Subject: [PATCH 10/22] Run arm and darwin builds after main build Try to balance the parallelism more --- .circleci/config.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8c7873b60..f48083d85 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,8 +18,12 @@ workflows: - unit-test - client-lint - client-test - - arm-build - - darwin-build + - arm-build: + requires: + - build + - darwin-build: + requires: + - build - build - integration-tests: requires: From ebfed1bc832ac7d5c683e0bf96b1c1960e2d027a Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sat, 25 Aug 2018 11:19:15 +0000 Subject: [PATCH 11/22] Don't insist the build container is up-to-date when BUILD_IN_CONTAINER=false Why do you care? You're either inside it or not using it. --- Makefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index e654ab914..f42be4121 100644 --- a/Makefile +++ b/Makefile @@ -99,7 +99,7 @@ $(SCOPE_EXE) $(RUNSVINIT) lint tests shell prog/staticui/staticui.go prog/extern else -$(SCOPE_EXE): $(SCOPE_BACKEND_BUILD_UPTODATE) +$(SCOPE_EXE): time $(GO) build $(GO_BUILD_FLAGS) -o $@ ./$(@D) @strings $@ | grep cgo_stub\\\.go >/dev/null || { \ rm $@; \ @@ -118,24 +118,24 @@ $(CODECGEN_EXE): $(CODECGEN_DIR)/*.go mkdir -p $(@D) $(GO_HOST) build $(GO_BUILD_FLAGS) -o $@ ./$(CODECGEN_DIR) -$(RUNSVINIT): $(SCOPE_BACKEND_BUILD_UPTODATE) +$(RUNSVINIT): time $(GO) build $(GO_BUILD_FLAGS) -o $@ ./$(@D) -shell: $(SCOPE_BACKEND_BUILD_UPTODATE) +shell: /bin/bash -tests: $(SCOPE_BACKEND_BUILD_UPTODATE) $(CODECGEN_TARGETS) prog/staticui/staticui.go prog/externalui/externalui.go +tests: $(CODECGEN_TARGETS) prog/staticui/staticui.go prog/externalui/externalui.go ./tools/test -no-go-get -tags $(GO_BUILD_TAGS) -lint: $(SCOPE_BACKEND_BUILD_UPTODATE) +lint: ./tools/lint ./tools/shell-lint tools -prog/staticui/staticui.go: $(SCOPE_BACKEND_BUILD_UPTODATE) +prog/staticui/staticui.go: mkdir -p prog/staticui esc -o $@ -pkg staticui -prefix client/build client/build -prog/externalui/externalui.go: $(SCOPE_BACKEND_BUILD_UPTODATE) +prog/externalui/externalui.go: mkdir -p prog/externalui esc -o $@ -pkg externalui -prefix client/build-external -include '\.html$$' client/build-external From 5d029d74f190d07894c05e8404d5017630a96970 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sat, 25 Aug 2018 11:31:30 +0000 Subject: [PATCH 12/22] Install docker client in build container --- backend/Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/backend/Dockerfile b/backend/Dockerfile index 5bddcf8e3..9b0605482 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -35,6 +35,13 @@ RUN go get -tags netgo \ chmod a+wr --recursive /usr/local/go && \ rm -rf /go/pkg/ /go/src/ RUN pip install yapf==0.16.2 flake8==3.3.0 requests==2.19.1 + +# Install Docker (client only) +ENV DOCKERVERSION=17.03.2-ce +RUN curl -fsSLO https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKERVERSION}.tgz && \ + tar xzvf docker-${DOCKERVERSION}.tgz --strip 1 -C /usr/local/bin docker/docker && \ + rm docker-${DOCKERVERSION}.tgz + COPY build.sh / ENTRYPOINT ["/build.sh"] From 8b9d49d9d66a6e2a1ec5b9461b3bbba4126d87e0 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sat, 25 Aug 2018 14:58:09 +0000 Subject: [PATCH 13/22] Run client static build and run client lint and test in the same job, after client-build, to reduce the number of jobs running in parallel. --- .circleci/config.yml | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f48083d85..53d5bdea9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ defaults: &defaults - image: weaveworks/scope-backend-build:circle-2.0-2885645 client-defaults: &client-defaults - working_directory: /home/weave + working_directory: /home/weave/scope docker: - image: weaveworks/scope-ui-build:circle-2.0-2885645 @@ -16,15 +16,19 @@ workflows: jobs: - lint - unit-test - - client-lint - - client-test + - client-build + - client-test: + requires: + - client-build - arm-build: requires: - build - darwin-build: requires: - build - - build + - build: + requires: + - client-build - integration-tests: requires: - lint @@ -39,7 +43,6 @@ workflows: branches: only: master requires: - - client-lint - client-test - integration-tests @@ -61,17 +64,30 @@ jobs: paths: - coverage - client-lint: + # Create client/build/index.html + client-build: <<: *client-defaults steps: - checkout - - run: make BUILD_IN_CONTAINER=false client-lint + # Convoluted set of steps here to mimic Makefile actions of bind-mounting different dirs into container + - run: | + mv client/app /home/weave/ + cd /home/weave; mkdir build ; yarn run build ; mv build scope/client + cd /home/weave; mkdir build-external; yarn run build-external; mv build-external scope/client + - persist_to_workspace: + root: /home/weave/scope + paths: + - client/build/ + - client/build-external/ client-test: <<: *client-defaults steps: - checkout - - run: make BUILD_IN_CONTAINER=false client-test + - run: | + mv client/app client/test /home/weave/ + cd /home/weave; yarn run lint + cd /home/weave; yarn test arm-build: <<: *defaults @@ -90,7 +106,9 @@ jobs: steps: - checkout - setup_remote_docker - - run: make BUILD_IN_CONTAINER=false SUDO= all + - attach_workspace: + at: . + - run: make BUILD_IN_CONTAINER=false SUDO= static all - run: cd extras; make BUILD_IN_CONTAINER=false - persist_to_workspace: root: . From 1d2a5c65448e49bcf8a4772754f57bcd1631a7c6 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sat, 25 Aug 2018 17:04:36 +0000 Subject: [PATCH 14/22] Set up dependencies for integration test job * Build 'runner' program in build job and save to workspace; * Save build outputs - externalui, staticui, codecgen output; * Install required tools in test VM; * Call 'make deps' in integration-tests-job and touch 'runner' from workspace so it doesn't rebuild. --- .circleci/config.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 53d5bdea9..081bf5a04 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -110,11 +110,17 @@ jobs: at: . - run: make BUILD_IN_CONTAINER=false SUDO= static all - run: cd extras; make BUILD_IN_CONTAINER=false + - run: make -C tools/runner - persist_to_workspace: root: . paths: - scope.tar - cloud-agent.tar + - tools/runner/runner + - prog/externalui/ + - prog/staticui/ + - report/report.codecgen.go + - render/detailed/detailed.codecgen.go integration-tests: machine: @@ -129,10 +135,15 @@ jobs: - checkout - attach_workspace: at: . + - run: | + sudo apt-get update + sudo apt-get install python-pip jq pv + sudo pip install awscli # kick off creation of test VMs - run: test -z "$SECRET_PASSWORD" || bin/setup-circleci-secrets "$SECRET_PASSWORD" - run: test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh make_template) - run: test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh setup && eval $(./gce.sh hosts); ./setup.sh) + - run: make deps; touch tools/runner/runner - run: command: test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; eval $(./gce.sh hosts); ./run_all.sh) no_output_timeout: 5m From e1552de65aea1d50b3c9a967ddea50b64d8c7e98 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 26 Aug 2018 14:25:07 +0000 Subject: [PATCH 15/22] 'requests' is installed via pip; we don't need to install via apt-get too --- backend/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 9b0605482..6deceb351 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -4,9 +4,9 @@ RUN set -eux; \ export arch_val="$(dpkg --print-architecture)"; \ apt-get update && \ if [ "$arch_val" = "amd64" ]; then \ - apt-get install -y libpcap-dev python-requests time file shellcheck git gcc-arm-linux-gnueabihf curl build-essential python-pip; \ + apt-get install -y libpcap-dev time file shellcheck git gcc-arm-linux-gnueabihf curl build-essential python-pip; \ else \ - apt-get install -y libpcap-dev python-requests time file shellcheck git curl build-essential python-pip; \ + apt-get install -y libpcap-dev time file shellcheck git curl build-essential python-pip; \ fi; \ \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* From 29e6409fab87efb348d8af5e9ff768bf6d7de198 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 26 Aug 2018 14:27:21 +0000 Subject: [PATCH 16/22] Move all go installs to same Dockerfile step Reduces the size of the resulting image --- backend/Dockerfile | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 6deceb351..48d932d43 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -16,16 +16,8 @@ RUN go clean -i net && \ export arch_val="$(dpkg --print-architecture)"; \ if [ "$arch_val" != "ppc64el" ]; then \ go install -race -tags netgo std; \ - fi; -RUN export arch_val="$(dpkg --print-architecture)"; \ - if [ "$arch_val" = "amd64" ]; then \ - curl -fsSL -o shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \ - chmod +x shfmt && \ - mv shfmt /usr/bin; \ - fi; - # Skipped installing shfmt, as the version v1.3.0 isn't supported for ppc64le - # and the later version of shfmt doesn't work with the application well -RUN go get -tags netgo \ + fi; \ + go get -tags netgo \ github.com/fzipp/gocyclo \ github.com/golang/lint/golint \ github.com/kisielk/errcheck \ @@ -34,6 +26,16 @@ RUN go get -tags netgo \ github.com/client9/misspell/cmd/misspell && \ chmod a+wr --recursive /usr/local/go && \ rm -rf /go/pkg/ /go/src/ + + # Only install shfmt on amd64, as the version v1.3.0 isn't supported for ppc64le + # and the later version of shfmt doesn't work with the application well +RUN export arch_val="$(dpkg --print-architecture)"; \ + if [ "$arch_val" = "amd64" ]; then \ + curl -fsSL -o shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \ + chmod +x shfmt && \ + mv shfmt /usr/bin; \ + fi; + RUN pip install yapf==0.16.2 flake8==3.3.0 requests==2.19.1 # Install Docker (client only) From cf0d85b7f4fe9c195230552808a26a03f77edee1 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 26 Aug 2018 16:10:07 +0000 Subject: [PATCH 17/22] Run coverage as part of integration test job This lets it run at the same time as GCE VMs are deleted, and saves us having to run up another job just for coverage. --- .circleci/config.yml | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 081bf5a04..acb0ddef6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,10 +34,6 @@ workflows: - lint - unit-test - build - - gen-coverage: - requires: - - unit-test - - integration-tests - deploy: filters: branches: @@ -130,6 +126,7 @@ jobs: SRCDIR: /home/circleci/src/github.com/weaveworks/scope CIRCLE_ARTIFACTS: /tmp/artifacts CLOUDSDK_CORE_DISABLE_PROMPTS: 1 + GOPATH: /home/circleci/ parallelism: 2 steps: - checkout @@ -139,6 +136,7 @@ jobs: sudo apt-get update sudo apt-get install python-pip jq pv sudo pip install awscli + - run: mkdir $CIRCLE_ARTIFACTS # kick off creation of test VMs - run: test -z "$SECRET_PASSWORD" || bin/setup-circleci-secrets "$SECRET_PASSWORD" - run: test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh make_template) @@ -151,20 +149,7 @@ jobs: - run: command: test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh destroy) background: true - - persist_to_workspace: - root: . - paths: - - coverage - - gen-coverage: - <<: *defaults - environment: - CIRCLE_ARTIFACTS: /tmp/artifacts - steps: - - checkout - - run: mkdir $CIRCLE_ARTIFACTS - - attach_workspace: - at: . + # Code coverage - run: ./tools/cover/gather_coverage.sh ./coverage $SRCDIR/coverage - run: goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=profile.cov -service=circleci - run: cp coverage.* */*.codecgen.go $CIRCLE_ARTIFACTS From 6a51cd81858ddfa1e23116712baa0f5b00ff1d81 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 26 Aug 2018 16:32:14 +0000 Subject: [PATCH 18/22] Tidy up --- .circleci/config.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index acb0ddef6..3ac99fe48 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -123,7 +123,6 @@ jobs: image: circleci/classic:201709-01 working_directory: /home/circleci/src/github.com/weaveworks/scope environment: - SRCDIR: /home/circleci/src/github.com/weaveworks/scope CIRCLE_ARTIFACTS: /tmp/artifacts CLOUDSDK_CORE_DISABLE_PROMPTS: 1 GOPATH: /home/circleci/ @@ -139,18 +138,19 @@ jobs: - run: mkdir $CIRCLE_ARTIFACTS # kick off creation of test VMs - run: test -z "$SECRET_PASSWORD" || bin/setup-circleci-secrets "$SECRET_PASSWORD" - - run: test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh make_template) - - run: test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh setup && eval $(./gce.sh hosts); ./setup.sh) + - run: test -z "$SECRET_PASSWORD" || (cd integration; ./gce.sh make_template) + - run: test -z "$SECRET_PASSWORD" || (cd integration; ./gce.sh setup && eval $(./gce.sh hosts); ./setup.sh) - run: make deps; touch tools/runner/runner + # Run all integration tests - run: - command: test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; eval $(./gce.sh hosts); ./run_all.sh) + command: test -z "$SECRET_PASSWORD" || (cd integration; eval $(./gce.sh hosts); ./run_all.sh) no_output_timeout: 5m # Destroy testing VMs: - run: - command: test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh destroy) + command: test -z "$SECRET_PASSWORD" || (cd integration; ./gce.sh destroy) background: true # Code coverage - - run: ./tools/cover/gather_coverage.sh ./coverage $SRCDIR/coverage + - run: ./tools/cover/gather_coverage.sh ./coverage - run: goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=profile.cov -service=circleci - run: cp coverage.* */*.codecgen.go $CIRCLE_ARTIFACTS - store_artifacts: @@ -191,4 +191,4 @@ jobs: docker tag weaveworks/scope:$(./tools/image-tag) "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)" && docker push "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)" ) - - run: test -z "${UI_BUCKET_KEY_ID}" || (cd $SRCDIR && make ui-upload && make ui-pkg-upload) + - run: test -z "${UI_BUCKET_KEY_ID}" || (make ui-upload && make ui-pkg-upload) From 8a0c42299e244ead2cec0dd6ba045ac78160ba56 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 26 Aug 2018 14:28:43 +0000 Subject: [PATCH 19/22] Use newer build image --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3ac99fe48..c1c212ab8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,7 +3,7 @@ version: 2 defaults: &defaults working_directory: /go/src/github.com/weaveworks/scope docker: - - image: weaveworks/scope-backend-build:circle-2.0-2885645 + - image: weaveworks/scope-backend-build:circle-2.0-4ad2f14 client-defaults: &client-defaults working_directory: /home/weave/scope From 7b9a15efbf8a982e0cda95fafdc42f3c12ca729a Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 27 Aug 2018 21:23:00 +0000 Subject: [PATCH 20/22] Combine arm and darwin build jobs saves spinning up another container --- .circleci/config.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c1c212ab8..af47b602e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,10 +20,7 @@ workflows: - client-test: requires: - client-build - - arm-build: - requires: - - build - - darwin-build: + - xplatform-build: requires: - build - build: @@ -85,16 +82,11 @@ jobs: cd /home/weave; yarn run lint cd /home/weave; yarn test - arm-build: - <<: *defaults - steps: - - checkout - - run: GOARCH=arm make BUILD_IN_CONTAINER=false GO_BUILD_INSTALL_DEPS= prog/scope - - darwin-build: + xplatform-build: <<: *defaults steps: - checkout + - run: GOARCH=arm make BUILD_IN_CONTAINER=false GO_BUILD_INSTALL_DEPS= prog/scope - run: GOOS=darwin make BUILD_IN_CONTAINER=false GO_BUILD_INSTALL_DEPS= prog/scope build: From 1734def7eefd905b2767194b101c73aceb9c4d0e Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 30 Aug 2018 08:24:35 +0000 Subject: [PATCH 21/22] Get deploy step to work Simplify deploy shell scripts; Run yarn bundle as part of client-build step; Install awscli as part of deploy step; Don't run ui-upload in another container. --- .circleci/config.yml | 48 ++++++++++++++++++-------------------------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index af47b602e..cfd003085 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,15 +63,16 @@ jobs: steps: - checkout # Convoluted set of steps here to mimic Makefile actions of bind-mounting different dirs into container - - run: | - mv client/app /home/weave/ - cd /home/weave; mkdir build ; yarn run build ; mv build scope/client - cd /home/weave; mkdir build-external; yarn run build-external; mv build-external scope/client + - run: mv client/app /home/weave/ + - run: cd /home/weave; mkdir build ; yarn run build ; mv build scope/client + - run: cd /home/weave; mkdir build-external; yarn run build-external; mv build-external scope/client + - run: cd /home/weave; mkdir tmp ; yarn run bundle ; mv tmp scope - persist_to_workspace: root: /home/weave/scope paths: - client/build/ - client/build-external/ + - tmp/weave-scope.tgz client-test: <<: *client-defaults @@ -126,7 +127,6 @@ jobs: - run: | sudo apt-get update sudo apt-get install python-pip jq pv - sudo pip install awscli - run: mkdir $CIRCLE_ARTIFACTS # kick off creation of test VMs - run: test -z "$SECRET_PASSWORD" || bin/setup-circleci-secrets "$SECRET_PASSWORD" @@ -157,30 +157,22 @@ jobs: - setup_remote_docker - attach_workspace: at: . - # PATH must be set here as circle currently does not support string interpolation - # for environment - # https://discuss.circleci.com/t/bin-sh-mkdir-command-not-found/8710 - - run: export PATH="$PATH:$HOME/.local/bin" - run: | + pip install awscli docker load -i scope.tar docker load -i cloud-agent.tar - test -z "${DOCKER_USER}" || ( - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && - (test "${DOCKER_ORGANIZATION:-$DOCKER_USER}" = "weaveworks" || ( - for IMAGE in $IMAGES; do - docker tag weaveworks/$IMAGE:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:latest && - docker tag weaveworks/$IMAGE:$(./tools/image-tag) ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:$(./tools/image-tag) - done - )) && - for IMAGE in $IMAGES; do - docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE && - docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:$(./tools/image-tag) - done - ) - run: | - test -z "${QUAY_USER}" || ( - docker login -e '.' -u "$QUAY_USER" -p "$QUAY_PASSWORD" quay.io && - docker tag weaveworks/scope:$(./tools/image-tag) "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)" && - docker push "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)" - ) - - run: test -z "${UI_BUCKET_KEY_ID}" || (make ui-upload && make ui-pkg-upload) + test -z "${DOCKER_USER}" && exit 0 + docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS + for IMAGE in $IMAGES; do + test "${DOCKER_ORGANIZATION:-$DOCKER_USER}" = "weaveworks" || docker tag weaveworks/$IMAGE:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:latest + docker tag weaveworks/$IMAGE:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:$(./tools/image-tag) + docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:latest + docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:$(./tools/image-tag) + done + - run: | + test -z "${QUAY_USER}" && exit 0 + docker login -e '.' -u "$QUAY_USER" -p "$QUAY_PASSWORD" quay.io + docker tag weaveworks/scope:$(./tools/image-tag) "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)" + docker push "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)" + - run: test -z "${UI_BUCKET_KEY_ID}" || (make BUILD_IN_CONTAINER=false ui-upload ui-pkg-upload) From d9e88c0e4385e86e9e7e21f50359fc6d21bc79f3 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 30 Aug 2018 11:57:40 +0000 Subject: [PATCH 22/22] Use newer ui-build image --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cfd003085..afe65009e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ defaults: &defaults client-defaults: &client-defaults working_directory: /home/weave/scope docker: - - image: weaveworks/scope-ui-build:circle-2.0-2885645 + - image: weaveworks/scope-ui-build:circle-2.0-1734def7 workflows: version: 2