Merge pull request #3333 from weaveworks/circle-2.0

Move to CircleCI 2.0
2026-07-16 03:49:52 +00:00 · 2018-09-01 13:48:06 +01:00
parent 65bdb145f9 d9e88c0e43
commit fda40b8338
5 changed files with 208 additions and 137 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -0,0 +1,178 @@
+version: 2
+
+defaults: &defaults
+  working_directory: /go/src/github.com/weaveworks/scope
+  docker:
+    - image: weaveworks/scope-backend-build:circle-2.0-4ad2f14
+
+client-defaults: &client-defaults
+  working_directory: /home/weave/scope
+  docker:
+    - image: weaveworks/scope-ui-build:circle-2.0-1734def7
+
+workflows:
+  version: 2
+  test_and_deploy:
+    jobs:
+      - lint
+      - unit-test
+      - client-build
+      - client-test:
+          requires:
+            - client-build
+      - xplatform-build:
+          requires:
+            - build
+      - build:
+          requires:
+            - client-build
+      - integration-tests:
+          requires:
+            - lint
+            - unit-test
+            - build
+      - deploy:
+          filters:
+            branches:
+              only: master
+          requires:
+            - client-test
+            - integration-tests
+
+jobs:
+  lint:
+    <<: *defaults
+    steps:
+      - checkout
+      - run: make BUILD_IN_CONTAINER=false lint
+
+  unit-test:
+    <<: *defaults
+    parallelism: 1
+    steps:
+      - checkout
+      - run: COVERDIR=./coverage make BUILD_IN_CONTAINER=false CODECGEN_UID=23 tests
+      - persist_to_workspace:
+          root: .
+          paths:
+          - coverage
+
+  # Create client/build/index.html
+  client-build:
+    <<: *client-defaults
+    steps:
+      - checkout
+      # Convoluted set of steps here to mimic Makefile actions of bind-mounting different dirs into container
+      - run: mv client/app /home/weave/
+      - run: cd /home/weave; mkdir build         ; yarn run build         ; mv build          scope/client
+      - run: cd /home/weave; mkdir build-external; yarn run build-external; mv build-external scope/client
+      - run: cd /home/weave; mkdir tmp           ; yarn run bundle        ; mv tmp            scope
+      - persist_to_workspace:
+          root: /home/weave/scope
+          paths:
+          - client/build/
+          - client/build-external/
+          - tmp/weave-scope.tgz
+
+  client-test:
+    <<: *client-defaults
+    steps:
+      - checkout
+      - run: |
+          mv client/app client/test /home/weave/
+          cd /home/weave; yarn run lint
+          cd /home/weave; yarn test
+
+  xplatform-build:
+    <<: *defaults
+    steps:
+      - checkout
+      - run: GOARCH=arm  make BUILD_IN_CONTAINER=false GO_BUILD_INSTALL_DEPS= prog/scope
+      - run: GOOS=darwin make BUILD_IN_CONTAINER=false GO_BUILD_INSTALL_DEPS= prog/scope
+
+  build:
+    <<: *defaults
+    steps:
+      - checkout
+      - setup_remote_docker
+      - attach_workspace:
+          at: .
+      - run: make BUILD_IN_CONTAINER=false SUDO= static all
+      - run: cd extras; make BUILD_IN_CONTAINER=false
+      - run: make -C tools/runner
+      - persist_to_workspace:
+          root: .
+          paths:
+          - scope.tar
+          - cloud-agent.tar
+          - tools/runner/runner
+          - prog/externalui/
+          - prog/staticui/
+          - report/report.codecgen.go
+          - render/detailed/detailed.codecgen.go
+
+  integration-tests:
+    machine:
+      image: circleci/classic:201709-01
+    working_directory: /home/circleci/src/github.com/weaveworks/scope
+    environment:
+      CIRCLE_ARTIFACTS: /tmp/artifacts
+      CLOUDSDK_CORE_DISABLE_PROMPTS: 1
+      GOPATH: /home/circleci/
+    parallelism: 2
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      - run: |
+          sudo apt-get update
+          sudo apt-get install python-pip jq pv
+      - run: mkdir $CIRCLE_ARTIFACTS
+      # kick off creation of test VMs
+      - run: test -z "$SECRET_PASSWORD" || bin/setup-circleci-secrets "$SECRET_PASSWORD"
+      - run: test -z "$SECRET_PASSWORD" || (cd integration; ./gce.sh make_template)
+      - run: test -z "$SECRET_PASSWORD" || (cd integration; ./gce.sh setup && eval $(./gce.sh hosts); ./setup.sh)
+      - run: make deps; touch tools/runner/runner
+      # Run all integration tests
+      - run:
+          command: test -z "$SECRET_PASSWORD" || (cd integration; eval $(./gce.sh hosts); ./run_all.sh)
+          no_output_timeout: 5m
+      # Destroy testing VMs:
+      - run:
+          command: test -z "$SECRET_PASSWORD" || (cd integration; ./gce.sh destroy)
+          background: true
+      # Code coverage
+      - run: ./tools/cover/gather_coverage.sh ./coverage
+      - run: goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=profile.cov -service=circleci
+      - run: cp coverage.* */*.codecgen.go $CIRCLE_ARTIFACTS
+      - store_artifacts:
+          path: /tmp/artifacts
+
+  deploy:
+    <<: *defaults
+    environment:
+      IMAGES: scope cloud-agent
+    steps:
+      - checkout
+      - setup_remote_docker
+      - attach_workspace:
+          at: .
+      - run: |
+          pip install awscli
+          docker load -i scope.tar
+          docker load -i cloud-agent.tar
+      - run: |
+          test -z "${DOCKER_USER}" && exit 0
+          docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
+          for IMAGE in $IMAGES; do
+              test "${DOCKER_ORGANIZATION:-$DOCKER_USER}" = "weaveworks" || docker tag weaveworks/$IMAGE:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:latest
+              docker tag weaveworks/$IMAGE:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:$(./tools/image-tag)
+              docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:latest
+              docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:$(./tools/image-tag)
+          done
+      - run: |
+          test -z "${QUAY_USER}" && exit 0
+          docker login -e '.' -u "$QUAY_USER" -p "$QUAY_PASSWORD" quay.io
+          docker tag weaveworks/scope:$(./tools/image-tag) "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)"
+          docker push "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)"
+      - run: test -z "${UI_BUCKET_KEY_ID}" || (make BUILD_IN_CONTAINER=false ui-upload ui-pkg-upload)
--- a/16
+++ b/16
@@ -99,7 +99,7 @@ $(SCOPE_EXE) $(RUNSVINIT) lint tests shell prog/staticui/staticui.go prog/extern

 else

-$(SCOPE_EXE): $(SCOPE_BACKEND_BUILD_UPTODATE)
+$(SCOPE_EXE):
 	time $(GO) build $(GO_BUILD_FLAGS) -o $@ ./$(@D)
 	@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
 	        rm $@; \
@@ -118,24 +118,24 @@ $(CODECGEN_EXE): $(CODECGEN_DIR)/*.go
 	mkdir -p $(@D)
 	$(GO_HOST) build $(GO_BUILD_FLAGS) -o $@ ./$(CODECGEN_DIR)

-$(RUNSVINIT): $(SCOPE_BACKEND_BUILD_UPTODATE)
+$(RUNSVINIT):
 	time $(GO) build $(GO_BUILD_FLAGS) -o $@ ./$(@D)

-shell: $(SCOPE_BACKEND_BUILD_UPTODATE)
+shell:
 	/bin/bash

-tests: $(SCOPE_BACKEND_BUILD_UPTODATE) $(CODECGEN_TARGETS) prog/staticui/staticui.go prog/externalui/externalui.go
+tests: $(CODECGEN_TARGETS) prog/staticui/staticui.go prog/externalui/externalui.go
 	./tools/test -no-go-get -tags $(GO_BUILD_TAGS)

-lint: $(SCOPE_BACKEND_BUILD_UPTODATE)
+lint:
 	./tools/lint
 	./tools/shell-lint tools

-prog/staticui/staticui.go: $(SCOPE_BACKEND_BUILD_UPTODATE)
+prog/staticui/staticui.go:
 	mkdir -p prog/staticui
 	esc -o $@ -pkg staticui -prefix client/build client/build

-prog/externalui/externalui.go: $(SCOPE_BACKEND_BUILD_UPTODATE)
+prog/externalui/externalui.go:
 	mkdir -p prog/externalui
 	esc -o $@ -pkg externalui -prefix client/build-external -include '\.html$$' client/build-external

@@ -195,10 +195,12 @@ endif

 $(SCOPE_UI_BUILD_UPTODATE): client/Dockerfile client/package.json client/webpack.local.config.js client/webpack.production.config.js client/server.js client/.eslintrc
 	$(SUDO) docker build -t $(SCOPE_UI_BUILD_IMAGE) client
+	$(SUDO) docker tag $(SCOPE_UI_BUILD_IMAGE) $(SCOPE_UI_BUILD_IMAGE):$(IMAGE_TAG)
 	touch $@

 $(SCOPE_BACKEND_BUILD_UPTODATE): backend/*
 	$(SUDO) docker build -t $(SCOPE_BACKEND_BUILD_IMAGE) backend
+	$(SUDO) docker tag $(SCOPE_BACKEND_BUILD_IMAGE) $(SCOPE_BACKEND_BUILD_IMAGE):$(IMAGE_TAG)
 	touch $@

 ui-upload: client/build-external/index.html
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -4,9 +4,9 @@ RUN set -eux; \
   export arch_val="$(dpkg --print-architecture)"; \
   apt-get update && \
   if [ "$arch_val" = "amd64" ]; then \
-     apt-get install -y libpcap-dev python-requests time file shellcheck git gcc-arm-linux-gnueabihf curl build-essential python-pip; \
+     apt-get install -y libpcap-dev time file shellcheck git gcc-arm-linux-gnueabihf curl build-essential python-pip; \
   else \
-     apt-get install -y libpcap-dev python-requests time file shellcheck git curl build-essential python-pip; \
+     apt-get install -y libpcap-dev time file shellcheck git curl build-essential python-pip; \
   fi; \
   \
   rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
@@ -16,16 +16,8 @@ RUN go clean -i net && \
   export arch_val="$(dpkg --print-architecture)"; \
   if [ "$arch_val" != "ppc64el" ]; then \
 	go install -race -tags netgo std; \
-   fi;
-RUN export arch_val="$(dpkg --print-architecture)"; \
-   if [ "$arch_val" = "amd64" ]; then \
-     curl -fsSL -o shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
-     chmod +x shfmt && \
-     mv shfmt /usr/bin; \
-   fi;
-   # Skipped installing shfmt, as the version v1.3.0 isn't supported for ppc64le
-   # and the later version of shfmt doesn't work with the application well
-RUN go get -tags netgo \
+   fi; \
+    go get -tags netgo \
 		github.com/fzipp/gocyclo \
 		github.com/golang/lint/golint \
 		github.com/kisielk/errcheck \
@@ -34,7 +26,24 @@ RUN go get -tags netgo \
 		github.com/client9/misspell/cmd/misspell && \
 	chmod a+wr --recursive /usr/local/go && \
 	rm -rf /go/pkg/ /go/src/
+
+   # Only install shfmt on amd64, as the version v1.3.0 isn't supported for ppc64le
+   # and the later version of shfmt doesn't work with the application well
+RUN export arch_val="$(dpkg --print-architecture)"; \
+   if [ "$arch_val" = "amd64" ]; then \
+     curl -fsSL -o shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
+     chmod +x shfmt && \
+     mv shfmt /usr/bin; \
+   fi;
+
 RUN pip install yapf==0.16.2 flake8==3.3.0 requests==2.19.1
+
+# Install Docker (client only)
+ENV DOCKERVERSION=17.03.2-ce
+RUN curl -fsSLO https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKERVERSION}.tgz && \
+    tar xzvf docker-${DOCKERVERSION}.tgz --strip 1 -C /usr/local/bin docker/docker && \
+    rm docker-${DOCKERVERSION}.tgz
+
 COPY build.sh /
 ENTRYPOINT ["/build.sh"]

--- a/circle.yml
+++ b/circle.yml
@@ -1,117 +0,0 @@
-general:
-  branches:
-    ignore:
-      - gh-pages
-
-machine:
-  pre:
-    - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0
-  services:
-    - docker
-  environment:
-    GOPATH: /home/ubuntu
-    SRCDIR: /home/ubuntu/src/github.com/weaveworks/scope
-    PATH: $PATH:$HOME/.local/bin
-    CLOUDSDK_CORE_DISABLE_PROMPTS: 1
-    SCOPE_UI_BUILD: $HOME/docker/scope_ui_build.tar
-    IMAGES: scope cloud-agent
-
-dependencies:
-  pre:
-      - pip install --upgrade requests
-  cache_directories:
-    - "~/docker"
-  override:
-    - |
-      sudo apt-get update &&
-      sudo apt-get install jq pv &&
-      sudo chmod a+wr --recursive /usr/local/go/pkg &&
-      sudo chown ubuntu:ubuntu "$HOME/.bashrc.backup"
-      (curl https://sdk.cloud.google.com | bash) &&
-      (test -z "$SECRET_PASSWORD" || bin/setup-circleci-secrets "$SECRET_PASSWORD") &&
-      make deps &&
-      mkdir -p $(dirname $SRCDIR) &&
-      cp -r $(pwd)/ $SRCDIR
-    - "cd $SRCDIR/client; ../tools/rebuild-image weaveworks/scope-ui-build . Dockerfile package.json webpack.production.config.js .eslintrc .babelrc && touch $SRCDIR/.scope_ui_build.uptodate"
-    - "cd $SRCDIR/backend; ../tools/rebuild-image weaveworks/scope-backend-build . Dockerfile build.sh && touch $SRCDIR/.scope_backend_build.uptodate"
-    - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh make_template):
-        parallel: false
-    - sudo apt-get update && sudo apt-get install python-pip && sudo pip install awscli
-
-test:
-  override:
-    - cd $SRCDIR; make RM= lint:
-        parallel: true
-    - cd $SRCDIR; COVERDIR=./coverage make RM= CODECGEN_UID=23 tests:
-        parallel: true
-    - cd $SRCDIR; make RM= client-test static:
-        parallel: true
-    - cd $SRCDIR; make RM= client-lint static:
-        parallel: true
-    - cd $SRCDIR; rm -f prog/scope; if [ "$CIRCLE_NODE_INDEX" = "0" ]; then GOARCH=arm make GO_BUILD_INSTALL_DEPS= RM= prog/scope; else GOOS=darwin make GO_BUILD_INSTALL_DEPS= RM= prog/scope; fi:
-        parallel: true
-    - cd $SRCDIR; rm -f prog/scope; make RM=:
-        parallel: true
-    - cd $SRCDIR/extras; ./build_on_circle.sh:
-        parallel: true
-    - "test -z \"$SECRET_PASSWORD\" || (cd $SRCDIR/integration; ./gce.sh setup && eval $(./gce.sh hosts); ./setup.sh)":
-        parallel: true
-    - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; eval $(./gce.sh hosts); ./run_all.sh):
-        parallel: true
-        timeout: 300
-  post:
-    - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh destroy):
-        parallel: true
-    - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR; ./tools/cover/gather_coverage.sh ./coverage $SRCDIR/coverage):
-        parallel: true
-    - test "$CIRCLE_NODE_INDEX" != "0" || (goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/profile.cov -service=circleci || true):
-        parallel: true
-    - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR; cp */*.codecgen.go $CIRCLE_ARTIFACTS):
-        parallel: true
-
-deployment:
-  hub:
-    branch: master
-    commands:
-      - |
-        test -z "${DOCKER_USER}" || (
-          docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS &&
-          (test "${DOCKER_ORGANIZATION:-$DOCKER_USER}" = "weaveworks" || (
-            for IMAGE in $IMAGES; do
-              docker tag weaveworks/$IMAGE:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:latest &&
-              docker tag weaveworks/$IMAGE:$(./tools/image-tag) ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:$(./tools/image-tag)
-            done
-          )) &&
-          for IMAGE in $IMAGES; do
-            docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE &&
-            docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:$(./tools/image-tag)
-          done
-        )
-      - |
-        test -z "${QUAY_USER}" || (
-          docker login -e '.' -u "$QUAY_USER" -p "$QUAY_PASSWORD" quay.io &&
-          docker tag weaveworks/scope:$(./tools/image-tag) "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)" &&
-          docker push "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)"
-        )
-      - test -z "${UI_BUCKET_KEY_ID}" || (cd $SRCDIR && make ui-upload && make ui-pkg-upload)
-  hub-dev:
-    branch: /^((?!master).)*$/  # not the master branch
-    commands:
-      - >
-        test -z "${DEPLOY_BRANCH}" || test -z "${DOCKER_USER}" || (
-          docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS &&
-          docker tag weaveworks/scope:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope:${CIRCLE_BRANCH//\//-} &&
-          docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope:${CIRCLE_BRANCH//\//-}
-        )
-#  release:
-#    branch: /release-[0-9]+\.[0-9]+/
-#    owner: weaveworks
-#    commands:
-#      - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get
-#      - cd $SRCDIR; PRODUCT=scope tools/publish-site "$WP_LIVE_URL" "$WP_LIVE_USER" "$WP_LIVE_PASSWORD"
-#  issues:
-#    branch: /.*/
-#    owner: weaveworks
-#    commands:
-#      - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get
-#      - cd $SRCDIR; PRODUCT=scope tools/publish-site "$WP_DEV_URL" "$WP_DEV_USER" "$WP_DEV_PASSWORD"
--- a/client/build-external/.gitignore
+++ b/client/build-external/.gitignore
@@ -1 +0,0 @@
-# this file is only present to force git to create this directory in the working tree
				`@@ -1 +0,0 @@`
				`# this file is only present to force git to create this directory in the working tree`