diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..afe65009e --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,178 @@ +version: 2 + +defaults: &defaults + working_directory: /go/src/github.com/weaveworks/scope + docker: + - image: weaveworks/scope-backend-build:circle-2.0-4ad2f14 + +client-defaults: &client-defaults + working_directory: /home/weave/scope + docker: + - image: weaveworks/scope-ui-build:circle-2.0-1734def7 + +workflows: + version: 2 + test_and_deploy: + jobs: + - lint + - unit-test + - client-build + - client-test: + requires: + - client-build + - xplatform-build: + requires: + - build + - build: + requires: + - client-build + - integration-tests: + requires: + - lint + - unit-test + - build + - deploy: + filters: + branches: + only: master + requires: + - client-test + - integration-tests + +jobs: + lint: + <<: *defaults + steps: + - checkout + - run: make BUILD_IN_CONTAINER=false lint + + unit-test: + <<: *defaults + parallelism: 1 + steps: + - checkout + - run: COVERDIR=./coverage make BUILD_IN_CONTAINER=false CODECGEN_UID=23 tests + - persist_to_workspace: + root: . + paths: + - coverage + + # Create client/build/index.html + client-build: + <<: *client-defaults + steps: + - checkout + # Convoluted set of steps here to mimic Makefile actions of bind-mounting different dirs into container + - run: mv client/app /home/weave/ + - run: cd /home/weave; mkdir build ; yarn run build ; mv build scope/client + - run: cd /home/weave; mkdir build-external; yarn run build-external; mv build-external scope/client + - run: cd /home/weave; mkdir tmp ; yarn run bundle ; mv tmp scope + - persist_to_workspace: + root: /home/weave/scope + paths: + - client/build/ + - client/build-external/ + - tmp/weave-scope.tgz + + client-test: + <<: *client-defaults + steps: + - checkout + - run: | + mv client/app client/test /home/weave/ + cd /home/weave; yarn run lint + cd /home/weave; yarn test + + xplatform-build: + <<: *defaults + steps: + - checkout + - run: GOARCH=arm make BUILD_IN_CONTAINER=false GO_BUILD_INSTALL_DEPS= prog/scope + - run: GOOS=darwin make BUILD_IN_CONTAINER=false GO_BUILD_INSTALL_DEPS= prog/scope + + build: + <<: *defaults + steps: + - checkout + - setup_remote_docker + - attach_workspace: + at: . + - run: make BUILD_IN_CONTAINER=false SUDO= static all + - run: cd extras; make BUILD_IN_CONTAINER=false + - run: make -C tools/runner + - persist_to_workspace: + root: . + paths: + - scope.tar + - cloud-agent.tar + - tools/runner/runner + - prog/externalui/ + - prog/staticui/ + - report/report.codecgen.go + - render/detailed/detailed.codecgen.go + + integration-tests: + machine: + image: circleci/classic:201709-01 + working_directory: /home/circleci/src/github.com/weaveworks/scope + environment: + CIRCLE_ARTIFACTS: /tmp/artifacts + CLOUDSDK_CORE_DISABLE_PROMPTS: 1 + GOPATH: /home/circleci/ + parallelism: 2 + steps: + - checkout + - attach_workspace: + at: . + - run: | + sudo apt-get update + sudo apt-get install python-pip jq pv + - run: mkdir $CIRCLE_ARTIFACTS + # kick off creation of test VMs + - run: test -z "$SECRET_PASSWORD" || bin/setup-circleci-secrets "$SECRET_PASSWORD" + - run: test -z "$SECRET_PASSWORD" || (cd integration; ./gce.sh make_template) + - run: test -z "$SECRET_PASSWORD" || (cd integration; ./gce.sh setup && eval $(./gce.sh hosts); ./setup.sh) + - run: make deps; touch tools/runner/runner + # Run all integration tests + - run: + command: test -z "$SECRET_PASSWORD" || (cd integration; eval $(./gce.sh hosts); ./run_all.sh) + no_output_timeout: 5m + # Destroy testing VMs: + - run: + command: test -z "$SECRET_PASSWORD" || (cd integration; ./gce.sh destroy) + background: true + # Code coverage + - run: ./tools/cover/gather_coverage.sh ./coverage + - run: goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=profile.cov -service=circleci + - run: cp coverage.* */*.codecgen.go $CIRCLE_ARTIFACTS + - store_artifacts: + path: /tmp/artifacts + + deploy: + <<: *defaults + environment: + IMAGES: scope cloud-agent + steps: + - checkout + - setup_remote_docker + - attach_workspace: + at: . + - run: | + pip install awscli + docker load -i scope.tar + docker load -i cloud-agent.tar + - run: | + test -z "${DOCKER_USER}" && exit 0 + docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS + for IMAGE in $IMAGES; do + test "${DOCKER_ORGANIZATION:-$DOCKER_USER}" = "weaveworks" || docker tag weaveworks/$IMAGE:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:latest + docker tag weaveworks/$IMAGE:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:$(./tools/image-tag) + docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:latest + docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:$(./tools/image-tag) + done + - run: | + test -z "${QUAY_USER}" && exit 0 + docker login -e '.' -u "$QUAY_USER" -p "$QUAY_PASSWORD" quay.io + docker tag weaveworks/scope:$(./tools/image-tag) "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)" + docker push "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)" + - run: test -z "${UI_BUCKET_KEY_ID}" || (make BUILD_IN_CONTAINER=false ui-upload ui-pkg-upload) diff --git a/Makefile b/Makefile index b08fc43e6..f42be4121 100644 --- a/Makefile +++ b/Makefile @@ -99,7 +99,7 @@ $(SCOPE_EXE) $(RUNSVINIT) lint tests shell prog/staticui/staticui.go prog/extern else -$(SCOPE_EXE): $(SCOPE_BACKEND_BUILD_UPTODATE) +$(SCOPE_EXE): time $(GO) build $(GO_BUILD_FLAGS) -o $@ ./$(@D) @strings $@ | grep cgo_stub\\\.go >/dev/null || { \ rm $@; \ @@ -118,24 +118,24 @@ $(CODECGEN_EXE): $(CODECGEN_DIR)/*.go mkdir -p $(@D) $(GO_HOST) build $(GO_BUILD_FLAGS) -o $@ ./$(CODECGEN_DIR) -$(RUNSVINIT): $(SCOPE_BACKEND_BUILD_UPTODATE) +$(RUNSVINIT): time $(GO) build $(GO_BUILD_FLAGS) -o $@ ./$(@D) -shell: $(SCOPE_BACKEND_BUILD_UPTODATE) +shell: /bin/bash -tests: $(SCOPE_BACKEND_BUILD_UPTODATE) $(CODECGEN_TARGETS) prog/staticui/staticui.go prog/externalui/externalui.go +tests: $(CODECGEN_TARGETS) prog/staticui/staticui.go prog/externalui/externalui.go ./tools/test -no-go-get -tags $(GO_BUILD_TAGS) -lint: $(SCOPE_BACKEND_BUILD_UPTODATE) +lint: ./tools/lint ./tools/shell-lint tools -prog/staticui/staticui.go: $(SCOPE_BACKEND_BUILD_UPTODATE) +prog/staticui/staticui.go: mkdir -p prog/staticui esc -o $@ -pkg staticui -prefix client/build client/build -prog/externalui/externalui.go: $(SCOPE_BACKEND_BUILD_UPTODATE) +prog/externalui/externalui.go: mkdir -p prog/externalui esc -o $@ -pkg externalui -prefix client/build-external -include '\.html$$' client/build-external @@ -195,10 +195,12 @@ endif $(SCOPE_UI_BUILD_UPTODATE): client/Dockerfile client/package.json client/webpack.local.config.js client/webpack.production.config.js client/server.js client/.eslintrc $(SUDO) docker build -t $(SCOPE_UI_BUILD_IMAGE) client + $(SUDO) docker tag $(SCOPE_UI_BUILD_IMAGE) $(SCOPE_UI_BUILD_IMAGE):$(IMAGE_TAG) touch $@ $(SCOPE_BACKEND_BUILD_UPTODATE): backend/* $(SUDO) docker build -t $(SCOPE_BACKEND_BUILD_IMAGE) backend + $(SUDO) docker tag $(SCOPE_BACKEND_BUILD_IMAGE) $(SCOPE_BACKEND_BUILD_IMAGE):$(IMAGE_TAG) touch $@ ui-upload: client/build-external/index.html diff --git a/backend/Dockerfile b/backend/Dockerfile index 5bddcf8e3..48d932d43 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -4,9 +4,9 @@ RUN set -eux; \ export arch_val="$(dpkg --print-architecture)"; \ apt-get update && \ if [ "$arch_val" = "amd64" ]; then \ - apt-get install -y libpcap-dev python-requests time file shellcheck git gcc-arm-linux-gnueabihf curl build-essential python-pip; \ + apt-get install -y libpcap-dev time file shellcheck git gcc-arm-linux-gnueabihf curl build-essential python-pip; \ else \ - apt-get install -y libpcap-dev python-requests time file shellcheck git curl build-essential python-pip; \ + apt-get install -y libpcap-dev time file shellcheck git curl build-essential python-pip; \ fi; \ \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -16,16 +16,8 @@ RUN go clean -i net && \ export arch_val="$(dpkg --print-architecture)"; \ if [ "$arch_val" != "ppc64el" ]; then \ go install -race -tags netgo std; \ - fi; -RUN export arch_val="$(dpkg --print-architecture)"; \ - if [ "$arch_val" = "amd64" ]; then \ - curl -fsSL -o shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \ - chmod +x shfmt && \ - mv shfmt /usr/bin; \ - fi; - # Skipped installing shfmt, as the version v1.3.0 isn't supported for ppc64le - # and the later version of shfmt doesn't work with the application well -RUN go get -tags netgo \ + fi; \ + go get -tags netgo \ github.com/fzipp/gocyclo \ github.com/golang/lint/golint \ github.com/kisielk/errcheck \ @@ -34,7 +26,24 @@ RUN go get -tags netgo \ github.com/client9/misspell/cmd/misspell && \ chmod a+wr --recursive /usr/local/go && \ rm -rf /go/pkg/ /go/src/ + + # Only install shfmt on amd64, as the version v1.3.0 isn't supported for ppc64le + # and the later version of shfmt doesn't work with the application well +RUN export arch_val="$(dpkg --print-architecture)"; \ + if [ "$arch_val" = "amd64" ]; then \ + curl -fsSL -o shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \ + chmod +x shfmt && \ + mv shfmt /usr/bin; \ + fi; + RUN pip install yapf==0.16.2 flake8==3.3.0 requests==2.19.1 + +# Install Docker (client only) +ENV DOCKERVERSION=17.03.2-ce +RUN curl -fsSLO https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKERVERSION}.tgz && \ + tar xzvf docker-${DOCKERVERSION}.tgz --strip 1 -C /usr/local/bin docker/docker && \ + rm docker-${DOCKERVERSION}.tgz + COPY build.sh / ENTRYPOINT ["/build.sh"] diff --git a/circle.yml b/circle.yml deleted file mode 100644 index b334fa972..000000000 --- a/circle.yml +++ /dev/null @@ -1,117 +0,0 @@ -general: - branches: - ignore: - - gh-pages - -machine: - pre: - - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0 - services: - - docker - environment: - GOPATH: /home/ubuntu - SRCDIR: /home/ubuntu/src/github.com/weaveworks/scope - PATH: $PATH:$HOME/.local/bin - CLOUDSDK_CORE_DISABLE_PROMPTS: 1 - SCOPE_UI_BUILD: $HOME/docker/scope_ui_build.tar - IMAGES: scope cloud-agent - -dependencies: - pre: - - pip install --upgrade requests - cache_directories: - - "~/docker" - override: - - | - sudo apt-get update && - sudo apt-get install jq pv && - sudo chmod a+wr --recursive /usr/local/go/pkg && - sudo chown ubuntu:ubuntu "$HOME/.bashrc.backup" - (curl https://sdk.cloud.google.com | bash) && - (test -z "$SECRET_PASSWORD" || bin/setup-circleci-secrets "$SECRET_PASSWORD") && - make deps && - mkdir -p $(dirname $SRCDIR) && - cp -r $(pwd)/ $SRCDIR - - "cd $SRCDIR/client; ../tools/rebuild-image weaveworks/scope-ui-build . Dockerfile package.json webpack.production.config.js .eslintrc .babelrc && touch $SRCDIR/.scope_ui_build.uptodate" - - "cd $SRCDIR/backend; ../tools/rebuild-image weaveworks/scope-backend-build . Dockerfile build.sh && touch $SRCDIR/.scope_backend_build.uptodate" - - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh make_template): - parallel: false - - sudo apt-get update && sudo apt-get install python-pip && sudo pip install awscli - -test: - override: - - cd $SRCDIR; make RM= lint: - parallel: true - - cd $SRCDIR; COVERDIR=./coverage make RM= CODECGEN_UID=23 tests: - parallel: true - - cd $SRCDIR; make RM= client-test static: - parallel: true - - cd $SRCDIR; make RM= client-lint static: - parallel: true - - cd $SRCDIR; rm -f prog/scope; if [ "$CIRCLE_NODE_INDEX" = "0" ]; then GOARCH=arm make GO_BUILD_INSTALL_DEPS= RM= prog/scope; else GOOS=darwin make GO_BUILD_INSTALL_DEPS= RM= prog/scope; fi: - parallel: true - - cd $SRCDIR; rm -f prog/scope; make RM=: - parallel: true - - cd $SRCDIR/extras; ./build_on_circle.sh: - parallel: true - - "test -z \"$SECRET_PASSWORD\" || (cd $SRCDIR/integration; ./gce.sh setup && eval $(./gce.sh hosts); ./setup.sh)": - parallel: true - - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; eval $(./gce.sh hosts); ./run_all.sh): - parallel: true - timeout: 300 - post: - - test -z "$SECRET_PASSWORD" || (cd $SRCDIR/integration; ./gce.sh destroy): - parallel: true - - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR; ./tools/cover/gather_coverage.sh ./coverage $SRCDIR/coverage): - parallel: true - - test "$CIRCLE_NODE_INDEX" != "0" || (goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/profile.cov -service=circleci || true): - parallel: true - - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR; cp */*.codecgen.go $CIRCLE_ARTIFACTS): - parallel: true - -deployment: - hub: - branch: master - commands: - - | - test -z "${DOCKER_USER}" || ( - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && - (test "${DOCKER_ORGANIZATION:-$DOCKER_USER}" = "weaveworks" || ( - for IMAGE in $IMAGES; do - docker tag weaveworks/$IMAGE:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:latest && - docker tag weaveworks/$IMAGE:$(./tools/image-tag) ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:$(./tools/image-tag) - done - )) && - for IMAGE in $IMAGES; do - docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE && - docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/$IMAGE:$(./tools/image-tag) - done - ) - - | - test -z "${QUAY_USER}" || ( - docker login -e '.' -u "$QUAY_USER" -p "$QUAY_PASSWORD" quay.io && - docker tag weaveworks/scope:$(./tools/image-tag) "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)" && - docker push "quay.io/${QUAY_ORGANIZATION}/scope:$(./tools/image-tag)" - ) - - test -z "${UI_BUCKET_KEY_ID}" || (cd $SRCDIR && make ui-upload && make ui-pkg-upload) - hub-dev: - branch: /^((?!master).)*$/ # not the master branch - commands: - - > - test -z "${DEPLOY_BRANCH}" || test -z "${DOCKER_USER}" || ( - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS && - docker tag weaveworks/scope:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope:${CIRCLE_BRANCH//\//-} && - docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope:${CIRCLE_BRANCH//\//-} - ) -# release: -# branch: /release-[0-9]+\.[0-9]+/ -# owner: weaveworks -# commands: -# - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get -# - cd $SRCDIR; PRODUCT=scope tools/publish-site "$WP_LIVE_URL" "$WP_LIVE_USER" "$WP_LIVE_PASSWORD" -# issues: -# branch: /.*/ -# owner: weaveworks -# commands: -# - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get -# - cd $SRCDIR; PRODUCT=scope tools/publish-site "$WP_DEV_URL" "$WP_DEV_USER" "$WP_DEV_PASSWORD" diff --git a/client/build-external/.gitignore b/client/build-external/.gitignore deleted file mode 100644 index a4b63e4d7..000000000 --- a/client/build-external/.gitignore +++ /dev/null @@ -1 +0,0 @@ -# this file is only present to force git to create this directory in the working tree