diff --git a/tools/.circleci/config.yml b/tools/.circleci/config.yml index 2e1a47005..27546e129 100644 --- a/tools/.circleci/config.yml +++ b/tools/.circleci/config.yml @@ -10,11 +10,11 @@ jobs: steps: - checkout - run: cd build; make - - run: docker run --rm -v "$PWD:$PWD" -w "$PWD" --entrypoint sh weaveworks/build-golang -c ./lint . - - run: docker run --rm -v "$PWD:$PWD" -w "$PWD" --entrypoint sh weaveworks/build-golang -c ./shell-lint . - - run: docker run --rm -v "$PWD/cover:/go/src/cover" -w "/go/src/cover" --entrypoint sh weaveworks/build-golang -c make - - run: docker run --rm -v "$PWD/socks:/go/src/socks" -w "/go/src/socks" --entrypoint sh weaveworks/build-golang -c "make proxy" - - run: docker run --rm -v "$PWD/runner:/go/src/runner" -w "/go/src/runner" --entrypoint sh weaveworks/build-golang -c make + - run: docker run --rm -v "$PWD:$PWD" -w "$PWD" --entrypoint sh weaveworks/build-golang -c "./shell-lint ." + - run: docker run --rm -v "$PWD:/go/src" -w "/go/src/cover" --entrypoint sh weaveworks/build-golang -c make + - run: docker run --rm -v "$PWD:/go/src" -w "/go/src/socks" --entrypoint sh weaveworks/build-golang -c "make proxy" + - run: docker run --rm -v "$PWD:/go/src" -w "/go/src/runner" --entrypoint sh weaveworks/build-golang -c make + - run: docker run --rm -v "$PWD:/go/src" -w "/go/src" --entrypoint sh weaveworks/build-golang -c "./lint ./build ./config_management ./cover ./dependencies ./integration ./provisioning ./runner ./scheduler ./socks" - deploy: command: | diff --git a/tools/build/golang/Dockerfile b/tools/build/golang/Dockerfile index df97b2c83..f56c156d7 100644 --- a/tools/build/golang/Dockerfile +++ b/tools/build/golang/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13.1-stretch +FROM golang:1.14.4-stretch RUN apt-get update && \ apt-get install -y \ curl \ @@ -8,13 +8,16 @@ RUN apt-get update && \ libprotobuf-dev \ make \ protobuf-compiler \ - python-pip \ + python3 \ + python3-pip \ python-requests \ python-yaml \ + libssl-dev \ + python-openssl \ shellcheck \ unzip && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -RUN pip install attrs pyhcl yapf==0.16.2 flake8==3.3.0 +RUN pip3 install attrs==19.2.0 pyhcl yapf==0.16.2 flake8==3.3.0 RUN curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \ echo "b1925c2c405458811f0c227266402cf1868b4de529f114722c2e3a5af4ac7bb2 shfmt" | sha256sum -c && \ chmod +x shfmt && \ @@ -34,7 +37,7 @@ RUN go get -tags netgo \ github.com/golang/protobuf/protoc-gen-go \ github.com/kisielk/errcheck \ github.com/mjibson/esc \ - github.com/prometheus/prometheus/cmd/promtool && \ + && \ rm -rf /go/pkg /go/src RUN mkdir protoc && \ cd protoc && \ diff --git a/tools/dependencies/cross_versions.py b/tools/dependencies/cross_versions.py index dd920f0ef..a27892842 100755 --- a/tools/dependencies/cross_versions.py +++ b/tools/dependencies/cross_versions.py @@ -80,8 +80,9 @@ def cross_versions(config): def main(argv): try: config = _validate_input(argv) - print(linesep.join('\t'.join(triple) - for triple in cross_versions(config))) + print( + linesep.join('\t'.join(triple) + for triple in cross_versions(config))) except Exception as e: print(str(e)) exit(_ERROR_RUNTIME) diff --git a/tools/lint b/tools/lint index 72621acbb..3f2d72e9c 100755 --- a/tools/lint +++ b/tools/lint @@ -219,12 +219,16 @@ matches_any() { return 1 } -filter_out() { +read_patterns() { local patterns_file="$1" if [ -n "$patterns_file" ] && [ -r "$patterns_file" ]; then - local patterns - patterns=$(sed '/^#.*$/d ; /^\s*$/d' "$patterns_file") # Remove blank lines and comments before we start iterating. - [ -n "$DEBUG" ] && echo >&2 "> Filters:" && echo >&2 "$patterns" + sed '/^#.*$/d ; /^\s*$/d' "$patterns_file" # Remove blank lines and comments. + fi +} + +filter_out() { + local patterns="$1" + if [ -n "$patterns" ]; then local filtered_out=() while read -r filename; do matches_any "$filename" "$patterns" && filtered_out+=("$filename") || echo "$filename" @@ -238,11 +242,13 @@ filter_out() { lint_directory() { local dirname="$1" local lint_result=0 + matches_any "$dirname" "$PATTERNS" && return 0 + [ -n "$DEBUG" ] && echo >&2 "> Linting directory: $dirname" # This test is just checking if there are any Go files in the directory if compgen -G "$dirname/*.go" >/dev/null; then lint_go "${dirname}" || lint_result=1 fi - find . -maxdepth 1 "$dirname" | filter_out "$LINT_IGNORE_FILE" | lint_files + find "$dirname" -maxdepth 1 | filter_out "$PATTERNS" | lint_files || lint_result=1 return $lint_result } @@ -255,11 +261,12 @@ lint_directories() { } list_directories() { - if [ $# -gt 0 ]; then - find "$@" \( -name vendor -o -name .git -o -name .cache -o -name .pkg \) -prune -o -type d - fi + find "$@" \( -name vendor -o -name .git -o -name .cache -o -name .pkg \) -prune -o -type d } +PATTERNS=$(read_patterns "$LINT_IGNORE_FILE") +[ -n "$DEBUG" ] && echo >&2 "> Filters:" && echo >&2 "$PATTERNS" + if [ $# = 1 ] && [ -f "$1" ]; then lint "$1" else diff --git a/tools/provisioning/gcp/main.tf b/tools/provisioning/gcp/main.tf index af5a22ebc..1f9b8bacb 100755 --- a/tools/provisioning/gcp/main.tf +++ b/tools/provisioning/gcp/main.tf @@ -18,8 +18,10 @@ resource "google_compute_instance" "tf_test_vm" { zone = "${var.gcp_zone}" count = "${var.num_hosts}" - disk { - image = "${var.gcp_image}" + boot_disk { + initialize_params { + image = "${var.gcp_image}" + } } tags = [ diff --git a/tools/provisioning/gcp/outputs.tf b/tools/provisioning/gcp/outputs.tf index 210398ba5..e09197937 100755 --- a/tools/provisioning/gcp/outputs.tf +++ b/tools/provisioning/gcp/outputs.tf @@ -3,11 +3,11 @@ output "username" { } output "public_ips" { - value = ["${google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.assigned_nat_ip}"] + value = ["${google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.nat_ip}"] } output "private_ips" { - value = ["${google_compute_instance.tf_test_vm.*.network_interface.0.address}"] + value = ["${google_compute_instance.tf_test_vm.*.network_interface.0.network_ip}"] } output "hostnames" { @@ -24,7 +24,7 @@ output "hostnames" { output "private_etc_hosts" { value = "${join("\n", "${formatlist("%v %v.%v.%v", - google_compute_instance.tf_test_vm.*.network_interface.0.address, + google_compute_instance.tf_test_vm.*.network_interface.0.network_ip, google_compute_instance.tf_test_vm.*.name, google_compute_instance.tf_test_vm.*.zone, var.app @@ -36,7 +36,7 @@ output "private_etc_hosts" { output "public_etc_hosts" { value = "${join("\n", "${formatlist("%v %v.%v.%v", - google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.assigned_nat_ip, + google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.nat_ip, google_compute_instance.tf_test_vm.*.name, google_compute_instance.tf_test_vm.*.zone, var.app @@ -47,8 +47,8 @@ output "public_etc_hosts" { output "ansible_inventory" { value = "${format("[all]\n%s", join("\n", "${formatlist("%v private_ip=%v", - google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.assigned_nat_ip, - google_compute_instance.tf_test_vm.*.network_interface.0.address + google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.nat_ip, + google_compute_instance.tf_test_vm.*.network_interface.0.network_ip )}" ))}" } diff --git a/tools/scheduler/README.md b/tools/scheduler/README.md index d9c4aa41e..5c406911c 100644 --- a/tools/scheduler/README.md +++ b/tools/scheduler/README.md @@ -23,41 +23,33 @@ $ pip install -r requirements.txt -t lib - Run: ```console - $ appcfg.py --version $(date '+%Y%m%dt%H%M%S') update . - XX:XX PM Application: positive-cocoa-90213; version: 1 - XX:XX PM Host: appengine.google.com - XX:XX PM Starting update of app: positive-cocoa-90213, version: 1 - XX:XX PM Getting current resource limits. - Your browser has been opened to visit: + $ gcloud app deploy --version $(date '+%Y%m%dt%H%M%S') --project positive-cocoa-90213 + Services to deploy: - https://accounts.google.com/o/oauth2/auth?scope=... + descriptor: [/Users/simon/weave/build-tools/scheduler/app.yaml] + source: [/Users/simon/weave/build-tools/scheduler] + target project: [positive-cocoa-90213] + target service: [default] + target version: [20200512t154238] + target url: [https://positive-cocoa-90213.appspot.com] - If your browser is on a different machine then exit and re-run this - application with the command-line parameter - --noauth_local_webserver + Do you want to continue (Y/n)? - Authentication successful. - XX:XX PM Scanning files on local disk. - XX:XX PM Scanned 500 files. - XX:XX PM Scanned 1000 files. - XX:XX PM Cloning 1220 application files. - XX:XX PM Uploading 28 files and blobs. - XX:XX PM Uploaded 28 files and blobs. - XX:XX PM Compilation starting. - XX:XX PM Compilation completed. - XX:XX PM Starting deployment. - XX:XX PM Checking if deployment succeeded. - XX:XX PM Will check again in 1 seconds. - XX:XX PM Checking if deployment succeeded. - XX:XX PM Will check again in 2 seconds. - XX:XX PM Checking if deployment succeeded. - XX:XX PM Will check again in 4 seconds. - XX:XX PM Checking if deployment succeeded. - XX:XX PM Deployment successful. - XX:XX PM Checking if updated app version is serving. - XX:XX PM Completed update of app: positive-cocoa-90213, version: 1 - XX:XX PM Uploading cron entries. + Beginning deployment of service [default]... + ╔════════════════════════════════════════════════════════════╗ + ╠═ Uploading 433 files to Google Cloud Storage ═╣ + ╚════════════════════════════════════════════════════════════╝ + File upload done. + Updating service [default]...done. + Setting traffic split for service [default]...done. + Deployed service [default] to [https://positive-cocoa-90213.appspot.com] + + You can stream logs from the command line by running: + $ gcloud app logs tail -s default + + To view your application in the web browser run: + $ gcloud app browse --project=positive-cocoa-90213 ``` - Go to [console.cloud.google.com](https://console.cloud.google.com) > Weave Integration Tests (`positive-cocoa-90213`) > AppEngine > Versions and ensure traffic is being directed to the newly deployed version. diff --git a/tools/scheduler/app.yaml b/tools/scheduler/app.yaml index 21f5f0527..e21848c80 100644 --- a/tools/scheduler/app.yaml +++ b/tools/scheduler/app.yaml @@ -1,5 +1,3 @@ -application: positive-cocoa-90213 -version: 1 runtime: python27 api_version: 1 threadsafe: true diff --git a/tools/scheduler/main.py b/tools/scheduler/main.py index 733de8077..4dd1e3424 100644 --- a/tools/scheduler/main.py +++ b/tools/scheduler/main.py @@ -82,14 +82,14 @@ def schedule(test_run, shard_count, shard): test_times_dict = dict(test_times) test_times.sort(key=operator.itemgetter(1)) - shards = {i: [] for i in xrange(shard_count)} + shards = {i: [] for i in range(shard_count)} while test_times: test_name, time = test_times.pop() # find shortest shard and put it in that s, _ = min( ((i, sum(test_times_dict[t] for t in shards[i])) - for i in xrange(shard_count)), + for i in range(shard_count)), key=operator.itemgetter(1)) shards[s].append(test_name) @@ -123,9 +123,6 @@ def _matches_any_regex(name, regexes): return matches -# See also: https://circleci.com/account/api -CIRCLE_CI_API_TOKEN = 'cffb83afd920cfa109cbd3e9eecb7511a2d18bb9' - # N.B.: When adding a project below, please ensure: # - its CircleCI project is either public, or is followed by the user attached # to the above API token @@ -137,8 +134,7 @@ PROJECTS = [ ('weaveworks/weave', 'positive-cocoa-90213', 'us-central1-a', True, None), ('weaveworks/scope', 'scope-integration-tests', 'us-central1-a', False, None), - ('weaveworks/wks', 'wks-tests', 'us-central1-a', True, - CIRCLE_CI_API_TOKEN), + ('weaveworks/wksctl', 'wks-tests', 'us-central1-a', True, None), ] diff --git a/tools/scheduler/requirements.txt b/tools/scheduler/requirements.txt index 17e9d6714..0bebea87b 100644 --- a/tools/scheduler/requirements.txt +++ b/tools/scheduler/requirements.txt @@ -1,2 +1,3 @@ flask==1.0.0 google-api-python-client==1.6.7 +werkzeug<1.0