Merge commit '5e3c8ab35850163674a069eb8f02cb0e7509b163' into drop-email2

tools/README.md

@@ -60,9 +60,8 @@ To update the code in build-tools.git, the process is therefore:
 
 If you have any questions about, feedback for or problems with `build-tools`:
 
-- Invite yourself to the <a href="https://weaveworks.github.io/community-slack/" target="_blank"> #weave-community </a> slack channel.
-- Ask a question on the <a href="https://weave-community.slack.com/messages/general/"> #weave-community</a> slack channel.
-- Send an email to <a href="mailto:weave-users@weave.works">weave-users@weave.works</a>
-- <a href="https://github.com/weaveworks/build-tools/issues/new">File an issue.</a>
+- Invite yourself to the <a href="https://slack.weave.works/" target="_blank">Weave Users Slack</a>.
+- Ask a question on the [#general](https://weave-community.slack.com/messages/general/) slack channel.
+- [File an issue](https://github.com/weaveworks/build-tools/issues/new).
 
 Your feedback is always welcome!

tools/build/golang/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.10.0-stretch
+FROM golang:1.11.1-stretch
 RUN apt-get update && \
     apt-get install -y \
       curl \
@@ -30,7 +30,7 @@ RUN go get -tags netgo \
 		github.com/gogo/protobuf/gogoproto \
 		github.com/gogo/protobuf/protoc-gen-gogoslick \
 		github.com/golang/dep/... \
-		github.com/golang/lint/golint \
+		golang.org/x/lint/golint \
 		github.com/golang/protobuf/protoc-gen-go \
 		github.com/kisielk/errcheck \
 		github.com/mjibson/esc \

tools/scheduler/README.md

@@ -1,6 +1,64 @@
-To upload newer version:
+# scheduler
 
+## Development
+
+### Dependencies
+
+Download and install:
+
+- the [original App Engine SDK for Python](https://cloud.google.com/appengine/docs/standard/python/download) ([Linux](https://storage.googleapis.com/appengine-sdks/featured/google_appengine_1.9.77.zip), [macOS](https://storage.googleapis.com/appengine-sdks/featured/GoogleAppEngineLauncher-1.9.77.dmg)) -- this should add `appcfg.py` to your `PATH`
+- `python` 2.7.x
+- `pip`
+
+### Setup
+
+```console
+$ pip install -U virtualenv
+$ virtualenv --python=$(which python2.7) $TMPDIR/scheduler
+$ source $TMPDIR/scheduler/bin/activate
+$ pip install -r requirements.txt -t lib
 ```
-pip install -r requirements.txt -t lib
-appcfg.py update .
-```
+
+## Deployment
+
+- Run:
+  ```console
+  $ appcfg.py --version $(date '+%Y%m%dt%H%M%S') update .
+  XX:XX PM Application: positive-cocoa-90213; version: 1
+  XX:XX PM Host: appengine.google.com
+  XX:XX PM Starting update of app: positive-cocoa-90213, version: 1
+  XX:XX PM Getting current resource limits.
+  Your browser has been opened to visit:
+
+      https://accounts.google.com/o/oauth2/auth?scope=...
+
+  If your browser is on a different machine then exit and re-run this
+  application with the command-line parameter
+
+    --noauth_local_webserver
+
+  Authentication successful.
+  XX:XX PM Scanning files on local disk.
+  XX:XX PM Scanned 500 files.
+  XX:XX PM Scanned 1000 files.
+  XX:XX PM Cloning 1220 application files.
+  XX:XX PM Uploading 28 files and blobs.
+  XX:XX PM Uploaded 28 files and blobs.
+  XX:XX PM Compilation starting.
+  XX:XX PM Compilation completed.
+  XX:XX PM Starting deployment.
+  XX:XX PM Checking if deployment succeeded.
+  XX:XX PM Will check again in 1 seconds.
+  XX:XX PM Checking if deployment succeeded.
+  XX:XX PM Will check again in 2 seconds.
+  XX:XX PM Checking if deployment succeeded.
+  XX:XX PM Will check again in 4 seconds.
+  XX:XX PM Checking if deployment succeeded.
+  XX:XX PM Deployment successful.
+  XX:XX PM Checking if updated app version is serving.
+  XX:XX PM Completed update of app: positive-cocoa-90213, version: 1
+  XX:XX PM Uploading cron entries.
+  ```
+
+- Go to [console.cloud.google.com](https://console.cloud.google.com) > Weave Integration Tests (`positive-cocoa-90213`) > AppEngine > Versions and ensure traffic is being directed to the newly deployed version.
+- Click on Tools > Logs, and ensure the application is behaving well.

tools/scheduler/main.py

@@ -123,10 +123,22 @@ def _matches_any_regex(name, regexes):
             return matches
 
 
+# See also: https://circleci.com/account/api
+CIRCLE_CI_API_TOKEN = 'cffb83afd920cfa109cbd3e9eecb7511a2d18bb9'
+
+# N.B.: When adding a project below, please ensure:
+# - its CircleCI project is either public, or is followed by the user attached
+#   to the above API token
+# - user positive-cocoa-90213@appspot.gserviceaccount.com has "Compute Admin"
+#   access to its GCP project (or any other role including
+#   compute.instances.list/delete and compute.firewalls.list/delete)
 PROJECTS = [
-    ('weaveworks/weave', 'weave-net-tests', 'us-central1-a', True),
-    ('weaveworks/weave', 'positive-cocoa-90213', 'us-central1-a', True),
-    ('weaveworks/scope', 'scope-integration-tests', 'us-central1-a', False),
+    ('weaveworks/weave', 'weave-net-tests', 'us-central1-a', True, None),
+    ('weaveworks/weave', 'positive-cocoa-90213', 'us-central1-a', True, None),
+    ('weaveworks/scope', 'scope-integration-tests', 'us-central1-a', False,
+     None),
+    ('weaveworks/wks', 'wks-tests', 'us-central1-a', True,
+     CIRCLE_CI_API_TOKEN),
 ]
 
 
@@ -134,18 +146,19 @@ PROJECTS = [
 def gc():
     # Get list of running VMs, pick build id out of VM name
     credentials = GoogleCredentials.get_application_default()
-    compute = discovery.build('compute', 'v1', credentials=credentials)
+    compute = discovery.build(
+        'compute', 'v1', credentials=credentials, cache_discovery=False)
 
-    for repo, project, zone, gc_fw in PROJECTS:
-        gc_project(compute, repo, project, zone, gc_fw)
+    for repo, project, zone, gc_fw, circleci_api_token in PROJECTS:
+        gc_project(compute, repo, project, zone, gc_fw, circleci_api_token)
 
     return "Done"
 
 
-def gc_project(compute, repo, project, zone, gc_fw):
+def gc_project(compute, repo, project, zone, gc_fw, circleci_api_token):
     logging.info("GCing %s, %s, %s", repo, project, zone)
     # Get list of builds, filter down to running builds:
-    running = _get_running_builds(repo)
+    running = _get_running_builds(repo, circleci_api_token)
     # Stop VMs for builds that aren't running:
     _gc_compute_engine_instances(compute, project, zone, running)
     # Remove firewall rules for builds that aren't running:
@@ -153,11 +166,17 @@ def gc_project(compute, repo, project, zone, gc_fw):
         _gc_firewall_rules(compute, project, running)
 
 
-def _get_running_builds(repo):
-    result = urlfetch.fetch(
-        'https://circleci.com/api/v1/project/%s' % repo,
-        headers={'Accept': 'application/json'})
-    assert result.status_code == 200
+def _get_running_builds(repo, circleci_api_token):
+    if circleci_api_token:
+        url = 'https://circleci.com/api/v1/project/%s?circle-token=%s' % (
+            repo, circleci_api_token)
+    else:
+        url = 'https://circleci.com/api/v1/project/%s' % repo
+    result = urlfetch.fetch(url, headers={'Accept': 'application/json'})
+    if result.status_code != 200:
+        raise RuntimeError(
+            'Failed to get builds for "%s". URL: %s, Status: %s. Response: %s'
+            % (repo, url, result.status_code, result.content))
     builds = json.loads(result.content)
     running = {
         build['build_num']

tools/scheduler/requirements.txt

@@ -1,2 +1,2 @@
-flask
-google-api-python-client
+flask==0.12.4
+google-api-python-client==1.6.7

tools/socks/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"net/http"
@@ -11,7 +12,6 @@ import (
 	socks5 "github.com/armon/go-socks5"
 	"github.com/weaveworks/common/mflag"
 	"github.com/weaveworks/common/mflagext"
-	"golang.org/x/net/context"
 )
 
 type pacFileParameters struct {