Merge pull request #1517 from weaveworks/docs-automation

Circle integration for auto docs publishing.

circle.yml

@@ -69,3 +69,15 @@ deployment:
           (test "${DOCKER_ORGANIZATION:-$DOCKER_USER}" == "weaveworks" || docker tag weaveworks/scope:latest ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope:latest) &&
           docker push ${DOCKER_ORGANIZATION:-$DOCKER_USER}/scope
         )
+  release:
+    branch: /release-[0-9]+\.[0-9]+/
+    owner: weaveworks
+    commands:
+      - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get
+      - cd $SRCDIR; PRODUCT=scope tools/publish-site "$WP_LIVE_URL" "$WP_LIVE_USER" "$WP_LIVE_PASSWORD"
+  issues:
+    branch: /.*/
+    owner: weaveworks
+    commands:
+      - go get github.com/weaveworks/wordepress && cd /home/ubuntu/src/github.com/weaveworks/wordepress && git checkout v1.0.0 && cd cmd/wordepress && go get
+      - cd $SRCDIR; PRODUCT=scope tools/publish-site "$WP_DEV_URL" "$WP_DEV_USER" "$WP_DEV_PASSWORD"

tools/publish-site

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -e
+set -o pipefail
+
+: ${PRODUCT:=}
+
+fatal() {
+    echo "$@" >&2
+    exit 1
+}
+
+if [ ! -d .git ] ; then
+    fatal "Current directory is not a git clone"
+fi
+
+if [ -z "${PRODUCT}" ]; then
+	fatal "Must specify PRODUCT"
+fi
+
+if ! BRANCH=$(git symbolic-ref --short HEAD) || [ -z "$BRANCH" ] ; then
+    fatal "Could not determine branch"
+fi
+
+case "$BRANCH" in
+    issues/*)
+        VERSION="${BRANCH#issues/}"
+        TAGS="$VERSION"
+        ;;
+    *)
+        if echo "$BRANCH" | grep -qE '^[0-9]+\.[0-9]+' ; then
+            DESCRIBE=$(git describe --match 'v*')
+            if ! VERSION=$(echo "$DESCRIBE" | grep -oP '(?<=^v)[0-9]+\.[0-9]+\.[0-9]+') ; then
+                fatal "Could not infer latest $BRANCH version from $DESCRIBE"
+            fi
+            TAGS="$VERSION latest"
+        else
+            VERSION="$BRANCH"
+            TAGS="$VERSION"
+        fi
+        ;;
+esac
+
+for TAG in $TAGS ; do
+    echo ">>> Publishing $PRODUCT $VERSION to $1/docs/$PRODUCT/$TAG"
+    wordepress \
+        --url "$1" --user "$2" --password "$3" \
+        --product "$PRODUCT" --version "$VERSION" --tag "$TAG" \
+        publish site
+done

tools/scheduler/app.yaml

@@ -11,3 +11,5 @@ handlers:
 libraries:
 - name: webapp2
   version: latest
+- name: ssl
+  version: latest

tools/scheduler/main.py

@@ -15,13 +15,10 @@ app = flask.Flask('scheduler')
 app.debug = True
 
 # We use exponential moving average to record
-# test run times.  Higher alpha discounts historic
+# test run times.  Higher alpha discounts historic
 # observations faster.
 alpha = 0.3
 
-PROJECT = 'positive-cocoa-90213'
-ZONE = 'us-central1-a'
-
 class Test(ndb.Model):
   total_run_time = ndb.FloatProperty(default=0.) # Not total, but a EWMA
   total_runs = ndb.IntegerProperty(default=0)
@@ -77,12 +74,28 @@ def schedule(test_run, shard_count, shard):
 
 NAME_RE = re.compile(r'^host(?P<index>\d+)-(?P<build>\d+)-(?P<shard>\d+)$')
 
+PROJECTS = [
+  ('weaveworks/weave', 'positive-cocoa-90213', 'us-central1-a'),
+  ('weaveworks/scope', 'scope-integration-tests', 'us-central1-a'),
+]
+
 @app.route('/tasks/gc')
 def gc():
   # Get list of running VMs, pick build id out of VM name
   credentials = GoogleCredentials.get_application_default()
   compute = discovery.build('compute', 'v1', credentials=credentials)
-  instances = compute.instances().list(project=PROJECT, zone=ZONE).execute()
+
+  for repo, project, zone in PROJECTS:
+    gc_project(compute, repo, project, zone)
+
+  return "Done"
+
+def gc_project(compute, repo, project, zone):
+  logging.info("GCing %s, %s, %s", repo, project, zone)
+  instances = compute.instances().list(project=project, zone=zone).execute()
+  if 'items' not in instances:
+    return
+
   host_by_build = collections.defaultdict(list)
   for instance in instances['items']:
     matches = NAME_RE.match(instance['name'])
@@ -92,7 +105,7 @@ def gc():
   logging.info("Running VMs by build: %r", host_by_build)
 
   # Get list of builds, filter down to runnning builds
-  result = urlfetch.fetch('https://circleci.com/api/v1/project/weaveworks/weave',
+  result = urlfetch.fetch('https://circleci.com/api/v1/project/%s' % repo,
     headers={'Accept': 'application/json'})
   assert result.status_code == 200
   builds = json.loads(result.content)
@@ -107,6 +120,6 @@ def gc():
     for name in names:
       stopped.append(name)
       logging.info("Stopping VM %s", name)
-      compute.instances().delete(project=PROJECT, zone=ZONE, instance=name).execute()
+      compute.instances().delete(project=project, zone=zone, instance=name).execute()
 
-  return (flask.json.jsonify(running=list(running), stopped=stopped), 200)
+  return