Squashed 'tools/' changes from f041a74ff..604e133c2

604e133c2 Merge pull request #167 from weaveworks/update-scheduler-instructions bdd647e92 Merge pull request #169 from weaveworks/go-1.14.4 2c2792e7d Upgrade to Go 1.14.4 e6dd5e8f0 Merge pull request #168 from weaveworks/downgrade-werkzeug 144aa64c0 Restruct scheduler to use compatible version of werkzeug library cc807901d Update instructions on how to deploy gc/scheduler 51a217660 Merge pull request #166 from weaveworks/165-rename-circleci-to-wksctl bfd7a2bbe Renames circleci project wks -> wksctl afte code move a5f7be3cc Note that the API token was revoked 15e4cfb94 Merge pull request #164 from weaveworks/improve-lint dc8722589 Fix lint errors in Python code 8cb6a62a2 Set failure state if lint_files fails c635ce412 Simplify the no-arguments case 167b7ed57 Skip entire directories that match filter patterns 1caa455a6 Refactor: read filter patterns once at start 4cc7911e5 Merge pull request #163 from weaveworks/fix-lint 393808dab Merge pull request #162 from weaveworks/golang-python3 3c5fcd16e Make lint run on CircleCI f0936a1e9 lint all directories if not given a list 68f62a393 Fix typo in lint script 9ef44b057 Stop installing promtool 882df5ec5 Install python 3 instead of obsolete v2.7 988002377 Update Go to latest 1.13.3 2a2df5278 Merge pull request #161 from murali-reddy/terraform-update bf8ed23b9 update google_compute_instance arguments as per the latest documentation git-subtree-dir: tools git-subtree-split: 604e133c2b7eb805766eab7be28723ef1ccd32aa
2026-02-14 18:09:59 +00:00 · 2020-09-03 10:15:13 +02:00
parent a0d60e4de9
commit 37ea4937a4
10 changed files with 67 additions and 67 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -10,11 +10,11 @@ jobs:
    steps:
    - checkout
    - run: cd build; make
-    - run: docker run --rm -v "$PWD:$PWD" -w "$PWD" --entrypoint sh weaveworks/build-golang -c ./lint .
-    - run: docker run --rm -v "$PWD:$PWD" -w "$PWD" --entrypoint sh weaveworks/build-golang -c ./shell-lint .
-    - run: docker run --rm -v "$PWD/cover:/go/src/cover" -w "/go/src/cover" --entrypoint sh weaveworks/build-golang -c make
-    - run: docker run --rm -v "$PWD/socks:/go/src/socks" -w "/go/src/socks" --entrypoint sh weaveworks/build-golang -c "make proxy"
-    - run: docker run --rm -v "$PWD/runner:/go/src/runner" -w "/go/src/runner" --entrypoint sh weaveworks/build-golang -c make
+    - run: docker run --rm -v "$PWD:$PWD" -w "$PWD" --entrypoint sh weaveworks/build-golang -c "./shell-lint ."
+    - run: docker run --rm -v "$PWD:/go/src" -w "/go/src/cover" --entrypoint sh weaveworks/build-golang -c make
+    - run: docker run --rm -v "$PWD:/go/src" -w "/go/src/socks" --entrypoint sh weaveworks/build-golang -c "make proxy"
+    - run: docker run --rm -v "$PWD:/go/src" -w "/go/src/runner" --entrypoint sh weaveworks/build-golang -c make
+    - run: docker run --rm -v "$PWD:/go/src" -w "/go/src" --entrypoint sh weaveworks/build-golang -c "./lint ./build ./config_management ./cover ./dependencies ./integration ./provisioning ./runner ./scheduler ./socks"

    - deploy:
        command: |
--- a/build/golang/Dockerfile
+++ b/build/golang/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.13.1-stretch
+FROM golang:1.14.4-stretch
 RUN apt-get update && \
    apt-get install -y \
      curl \
@@ -8,13 +8,16 @@ RUN apt-get update && \
      libprotobuf-dev \
      make \
      protobuf-compiler \
-      python-pip \
+      python3 \
+      python3-pip \
      python-requests \
      python-yaml \
+      libssl-dev \
+      python-openssl \
      shellcheck \
      unzip && \
    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-RUN pip install attrs pyhcl yapf==0.16.2 flake8==3.3.0
+RUN pip3 install attrs==19.2.0 pyhcl yapf==0.16.2 flake8==3.3.0
 RUN curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
 	echo "b1925c2c405458811f0c227266402cf1868b4de529f114722c2e3a5af4ac7bb2  shfmt" | sha256sum -c && \
 	chmod +x shfmt && \
@@ -34,7 +37,7 @@ RUN go get -tags netgo \
 		github.com/golang/protobuf/protoc-gen-go \
 		github.com/kisielk/errcheck \
 		github.com/mjibson/esc \
-		github.com/prometheus/prometheus/cmd/promtool && \
+		&& \
 		rm -rf /go/pkg /go/src
 RUN mkdir protoc && \
 	cd protoc && \
--- a/dependencies/cross_versions.py
+++ b/dependencies/cross_versions.py
@@ -80,8 +80,9 @@ def cross_versions(config):
 def main(argv):
    try:
        config = _validate_input(argv)
-        print(linesep.join('\t'.join(triple)
-                           for triple in cross_versions(config)))
+        print(
+            linesep.join('\t'.join(triple)
+                         for triple in cross_versions(config)))
    except Exception as e:
        print(str(e))
        exit(_ERROR_RUNTIME)
--- a/23
+++ b/23
@@ -219,12 +219,16 @@ matches_any() {
    return 1
 }

-filter_out() {
+read_patterns() {
    local patterns_file="$1"
    if [ -n "$patterns_file" ] && [ -r "$patterns_file" ]; then
-        local patterns
-        patterns=$(sed '/^#.*$/d ; /^\s*$/d' "$patterns_file") # Remove blank lines and comments before we start iterating.
-        [ -n "$DEBUG" ] && echo >&2 "> Filters:" && echo >&2 "$patterns"
+        sed '/^#.*$/d ; /^\s*$/d' "$patterns_file" # Remove blank lines and comments.
+    fi
+}
+
+filter_out() {
+    local patterns="$1"
+    if [ -n "$patterns" ]; then
        local filtered_out=()
        while read -r filename; do
            matches_any "$filename" "$patterns" && filtered_out+=("$filename") || echo "$filename"
@@ -238,11 +242,13 @@ filter_out() {
 lint_directory() {
    local dirname="$1"
    local lint_result=0
+    matches_any "$dirname" "$PATTERNS" && return 0
+    [ -n "$DEBUG" ] && echo >&2 "> Linting directory: $dirname"
    # This test is just checking if there are any Go files in the directory
    if compgen -G "$dirname/*.go" >/dev/null; then
        lint_go "${dirname}" || lint_result=1
    fi
-    find . -maxdepth 1 "$dirname" | filter_out "$LINT_IGNORE_FILE" | lint_files
+    find "$dirname" -maxdepth 1 | filter_out "$PATTERNS" | lint_files || lint_result=1
    return $lint_result
 }

@@ -255,11 +261,12 @@ lint_directories() {
 }

 list_directories() {
-    if [ $# -gt 0 ]; then
-        find "$@" \( -name vendor -o -name .git -o -name .cache -o -name .pkg \) -prune -o -type d
-    fi
+    find "$@" \( -name vendor -o -name .git -o -name .cache -o -name .pkg \) -prune -o -type d
 }

+PATTERNS=$(read_patterns "$LINT_IGNORE_FILE")
+[ -n "$DEBUG" ] && echo >&2 "> Filters:" && echo >&2 "$PATTERNS"
+
 if [ $# = 1 ] && [ -f "$1" ]; then
    lint "$1"
 else
--- a/provisioning/gcp/main.tf
+++ b/provisioning/gcp/main.tf
@@ -18,8 +18,10 @@ resource "google_compute_instance" "tf_test_vm" {
  zone         = "${var.gcp_zone}"
  count        = "${var.num_hosts}"

-  disk {
-    image = "${var.gcp_image}"
+  boot_disk {
+    initialize_params {
+      image = "${var.gcp_image}"
+    }
  }

  tags = [
--- a/provisioning/gcp/outputs.tf
+++ b/provisioning/gcp/outputs.tf
@@ -3,11 +3,11 @@ output "username" {
 }

 output "public_ips" {
-  value = ["${google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.assigned_nat_ip}"]
+  value = ["${google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.nat_ip}"]
 }

 output "private_ips" {
-  value = ["${google_compute_instance.tf_test_vm.*.network_interface.0.address}"]
+  value = ["${google_compute_instance.tf_test_vm.*.network_interface.0.network_ip}"]
 }

 output "hostnames" {
@@ -24,7 +24,7 @@ output "hostnames" {
 output "private_etc_hosts" {
  value = "${join("\n", 
    "${formatlist("%v %v.%v.%v", 
-      google_compute_instance.tf_test_vm.*.network_interface.0.address, 
+      google_compute_instance.tf_test_vm.*.network_interface.0.network_ip,
      google_compute_instance.tf_test_vm.*.name, 
      google_compute_instance.tf_test_vm.*.zone, 
      var.app
@@ -36,7 +36,7 @@ output "private_etc_hosts" {
 output "public_etc_hosts" {
  value = "${join("\n", 
    "${formatlist("%v %v.%v.%v", 
-      google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.assigned_nat_ip, 
+      google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.nat_ip,
      google_compute_instance.tf_test_vm.*.name, 
      google_compute_instance.tf_test_vm.*.zone, 
      var.app
@@ -47,8 +47,8 @@ output "public_etc_hosts" {
 output "ansible_inventory" {
  value = "${format("[all]\n%s", join("\n",
    "${formatlist("%v private_ip=%v",
-      google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.assigned_nat_ip,
-      google_compute_instance.tf_test_vm.*.network_interface.0.address
+      google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.nat_ip,
+      google_compute_instance.tf_test_vm.*.network_interface.0.network_ip
    )}"
  ))}"
 }
--- a/scheduler/README.md
+++ b/scheduler/README.md
@@ -23,41 +23,33 @@ $ pip install -r requirements.txt -t lib

 - Run:
  ```console
-  $ appcfg.py --version $(date '+%Y%m%dt%H%M%S') update .
-  XX:XX PM Application: positive-cocoa-90213; version: 1
-  XX:XX PM Host: appengine.google.com
-  XX:XX PM Starting update of app: positive-cocoa-90213, version: 1
-  XX:XX PM Getting current resource limits.
-  Your browser has been opened to visit:
+  $ gcloud app deploy --version $(date '+%Y%m%dt%H%M%S') --project positive-cocoa-90213
+  Services to deploy:

-      https://accounts.google.com/o/oauth2/auth?scope=...
+  descriptor:      [/Users/simon/weave/build-tools/scheduler/app.yaml]
+  source:          [/Users/simon/weave/build-tools/scheduler]
+  target project:  [positive-cocoa-90213]
+  target service:  [default]
+  target version:  [20200512t154238]
+  target url:      [https://positive-cocoa-90213.appspot.com]

-  If your browser is on a different machine then exit and re-run this
-  application with the command-line parameter

-    --noauth_local_webserver
+  Do you want to continue (Y/n)?

-  Authentication successful.
-  XX:XX PM Scanning files on local disk.
-  XX:XX PM Scanned 500 files.
-  XX:XX PM Scanned 1000 files.
-  XX:XX PM Cloning 1220 application files.
-  XX:XX PM Uploading 28 files and blobs.
-  XX:XX PM Uploaded 28 files and blobs.
-  XX:XX PM Compilation starting.
-  XX:XX PM Compilation completed.
-  XX:XX PM Starting deployment.
-  XX:XX PM Checking if deployment succeeded.
-  XX:XX PM Will check again in 1 seconds.
-  XX:XX PM Checking if deployment succeeded.
-  XX:XX PM Will check again in 2 seconds.
-  XX:XX PM Checking if deployment succeeded.
-  XX:XX PM Will check again in 4 seconds.
-  XX:XX PM Checking if deployment succeeded.
-  XX:XX PM Deployment successful.
-  XX:XX PM Checking if updated app version is serving.
-  XX:XX PM Completed update of app: positive-cocoa-90213, version: 1
-  XX:XX PM Uploading cron entries.
+  Beginning deployment of service [default]...
+  ╔════════════════════════════════════════════════════════════╗
+  ╠═ Uploading 433 files to Google Cloud Storage              ═╣
+  ╚════════════════════════════════════════════════════════════╝
+  File upload done.
+  Updating service [default]...done.
+  Setting traffic split for service [default]...done.
+  Deployed service [default] to [https://positive-cocoa-90213.appspot.com]
+
+  You can stream logs from the command line by running:
+    $ gcloud app logs tail -s default
+
+  To view your application in the web browser run:
+    $ gcloud app browse --project=positive-cocoa-90213
  ```

 - Go to [console.cloud.google.com](https://console.cloud.google.com) > Weave Integration Tests (`positive-cocoa-90213`) > AppEngine > Versions and ensure traffic is being directed to the newly deployed version.
--- a/scheduler/app.yaml
+++ b/scheduler/app.yaml
@@ -1,5 +1,3 @@
-application: positive-cocoa-90213
-version: 1
 runtime: python27
 api_version: 1
 threadsafe: true
--- a/scheduler/main.py
+++ b/scheduler/main.py
@@ -82,14 +82,14 @@ def schedule(test_run, shard_count, shard):
    test_times_dict = dict(test_times)
    test_times.sort(key=operator.itemgetter(1))

-    shards = {i: [] for i in xrange(shard_count)}
+    shards = {i: [] for i in range(shard_count)}
    while test_times:
        test_name, time = test_times.pop()

        # find shortest shard and put it in that
        s, _ = min(
            ((i, sum(test_times_dict[t] for t in shards[i]))
-             for i in xrange(shard_count)),
+             for i in range(shard_count)),
            key=operator.itemgetter(1))

        shards[s].append(test_name)
@@ -123,9 +123,6 @@ def _matches_any_regex(name, regexes):
            return matches


-# See also: https://circleci.com/account/api
-CIRCLE_CI_API_TOKEN = 'cffb83afd920cfa109cbd3e9eecb7511a2d18bb9'
-
 # N.B.: When adding a project below, please ensure:
 # - its CircleCI project is either public, or is followed by the user attached
 #   to the above API token
@@ -137,8 +134,7 @@ PROJECTS = [
    ('weaveworks/weave', 'positive-cocoa-90213', 'us-central1-a', True, None),
    ('weaveworks/scope', 'scope-integration-tests', 'us-central1-a', False,
     None),
-    ('weaveworks/wks', 'wks-tests', 'us-central1-a', True,
-     CIRCLE_CI_API_TOKEN),
+    ('weaveworks/wksctl', 'wks-tests', 'us-central1-a', True, None),
 ]


--- a/scheduler/requirements.txt
+++ b/scheduler/requirements.txt
@@ -1,2 +1,3 @@
 flask==0.12.4
 google-api-python-client==1.6.7
+werkzeug<1.0