Squashed 'tools/' changes from e9e7e6b..b990f48

b990f48 Merge pull request #42 from kinvolk/lorenzo/fix-git-diff
224a145 Check if SHA1 exists before calling `git diff`
1c3000d Add auto_apply config for wcloud
0ebf5c0 Fix wcloud -serivice
4fe078a Merge pull request #39 from weaveworks/fix-wrong-subtree-use
3f4934d Remove generate_latest_map
48beb60 Sync changes done directly in scope/tools
45dcdd5 Merge pull request #37 from weaveworks/fix-mflag-missing
b895344 Use mflag package from weaveworks fork until we find a better solution
e030008 Merge pull request #36 from weaveworks/wcloud-service-flags
9cbab40 Add wcloud Makefile
ef55901 Review feedback, and build wcloud in circle.
3fe92f5 Add wcloud deploy --service flag
3527b56 Merge pull request #34 from weaveworks/repo-branch
92cd0b8 [wcloud] Add support for repo_branch option
9f760ab Allow wcloud users to override username
38037f8 Merge pull request #33 from weaveworks/wcloud-templates
7acfbd7 Propagate the local username
e6876d1 Add template fields to wcloud config.
f1bb537 Merge pull request #30 from weaveworks/mike/shell-lint/dont-error-if-empty
e60f5df Merge pull request #31 from weaveworks/mike/fix-shell-lint-errors
e8e2b69 integrations: Fix a shellcheck linter error
a781575 shell-lint: Don't fail if no shell scripts found
db5efc0 Merge pull request #28 from weaveworks/mike/add-image-tag
5312c40 Import image-tag script into build tools so it can be shared
7e850f8 Fix logs path
dda9785 Update deploy api
f2f4e5b Fix the wcloud client
3925eb6 Merge pull request #27 from weaveworks/wcloud-events
77355b9 Lint
d9a1c6c Add wcloud events, update flags and error nicely when there is no config

git-subtree-dir: tools
git-subtree-split: b990f488bdc7909b62d9245bc4b4caf58f5ae7ea

.gitignore

@@ -2,5 +2,6 @@ cover/cover
 socks/proxy
 socks/image.tar
 runner/runner
+cmd/wcloud/wcloud
 *.pyc
 *~

circle.yml

@@ -20,4 +20,5 @@ test:
     - cd $SRCDIR/cover; make
     - cd $SRCDIR/socks; make
     - cd $SRCDIR/runner; make
+    - cd $SRCDIR/cmd/wcloud; make
 

cmd/wcloud/Makefile

@@ -0,0 +1,11 @@
+.PHONY: all clean
+
+all: wcloud
+
+wcloud: *.go
+	go get ./$(@D)
+	go build -o $@ ./$(@D)
+
+clean:
+	rm -rf wcloud
+	go clean ./...

cmd/wcloud/cli.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"os/user"
 	"path/filepath"
 	"strings"
 	"time"
@@ -15,6 +16,19 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
+// ArrayFlags allows you to collect repeated flags
+type ArrayFlags []string
+
+func (a *ArrayFlags) String() string {
+	return strings.Join(*a, ",")
+}
+
+// Set implements flags.Value
+func (a *ArrayFlags) Set(value string) error {
+	*a = append(*a, value)
+	return nil
+}
+
 func env(key, def string) string {
 	if val, ok := os.LookupEnv(key); ok {
 		return val
@@ -52,6 +66,8 @@ func main() {
 		config(c, os.Args[2:])
 	case "logs":
 		logs(c, os.Args[2:])
+	case "events":
+		events(c, os.Args[2:])
 	case "help":
 		usage()
 	default:
@@ -60,6 +76,17 @@ func main() {
 }
 
 func deploy(c Client, args []string) {
+	var (
+		flags    = flag.NewFlagSet("", flag.ContinueOnError)
+		username = flags.String("u", "", "Username to report to deploy service (default with be current user)")
+		services ArrayFlags
+	)
+	flags.Var(&services, "service", "Service to update (can be repeated)")
+	if err := flags.Parse(args); err != nil {
+		usage()
+		return
+	}
+	args = flags.Args()
 	if len(args) != 1 {
 		usage()
 		return
@@ -69,9 +96,19 @@ func deploy(c Client, args []string) {
 		usage()
 		return
 	}
+	if *username == "" {
+		user, err := user.Current()
+		if err != nil {
+			fmt.Println(err.Error())
+			os.Exit(1)
+		}
+		*username = user.Username
+	}
 	deployment := Deployment{
-		ImageName: parts[0],
-		Version:   parts[1],
+		ImageName:        parts[0],
+		Version:          parts[1],
+		TriggeringUser:   *username,
+		IntendedServices: services,
 	}
 	if err := c.Deploy(deployment); err != nil {
 		fmt.Println(err.Error())
@@ -80,14 +117,17 @@ func deploy(c Client, args []string) {
 }
 
 func list(c Client, args []string) {
-	flags := flag.NewFlagSet("list", flag.ContinueOnError)
-	page := flags.Int("page", 0, "Zero based index of page to list.")
-	pagesize := flags.Int("page-size", 10, "Number of results per page")
+	var (
+		flags = flag.NewFlagSet("", flag.ContinueOnError)
+		since = flags.Duration("since", 7*24*time.Hour, "How far back to fetch results")
+	)
 	if err := flags.Parse(args); err != nil {
 		usage()
 		return
 	}
-	deployments, err := c.GetDeployments(*page, *pagesize)
+	through := time.Now()
+	from := through.Add(-*since)
+	deployments, err := c.GetDeployments(from.Unix(), through.Unix())
 	if err != nil {
 		fmt.Println(err.Error())
 		os.Exit(1)
@@ -109,6 +149,26 @@ func list(c Client, args []string) {
 	table.Render()
 }
 
+func events(c Client, args []string) {
+	var (
+		flags = flag.NewFlagSet("", flag.ContinueOnError)
+		since = flags.Duration("since", 7*24*time.Hour, "How far back to fetch results")
+	)
+	if err := flags.Parse(args); err != nil {
+		usage()
+		return
+	}
+	through := time.Now()
+	from := through.Add(-*since)
+	events, err := c.GetEvents(from.Unix(), through.Unix())
+	if err != nil {
+		fmt.Println(err.Error())
+		os.Exit(1)
+	}
+
+	fmt.Println("events: ", string(events))
+}
+
 func loadConfig(filename string) (*Config, error) {
 	extension := filepath.Ext(filename)
 	var config Config

cmd/wcloud/client.go

@@ -38,7 +38,7 @@ func (c Client) Deploy(deployment Deployment) error {
 	if err := json.NewEncoder(&buf).Encode(deployment); err != nil {
 		return err
 	}
-	req, err := c.newRequest("POST", "/api/deploy", &buf)
+	req, err := c.newRequest("POST", "/api/deploy/deploy", &buf)
 	if err != nil {
 		return err
 	}
@@ -53,8 +53,8 @@ func (c Client) Deploy(deployment Deployment) error {
 }
 
 // GetDeployments returns a list of deployments
-func (c Client) GetDeployments(page, pagesize int) ([]Deployment, error) {
-	req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy?page=%d&pagesize=%d", page, pagesize), nil)
+func (c Client) GetDeployments(from, through int64) ([]Deployment, error) {
+	req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/deploy?from=%d&through=%d", from, through), nil)
 	if err != nil {
 		return nil, err
 	}
@@ -74,6 +74,22 @@ func (c Client) GetDeployments(page, pagesize int) ([]Deployment, error) {
 	return response.Deployments, nil
 }
 
+// GetEvents returns the raw events.
+func (c Client) GetEvents(from, through int64) ([]byte, error) {
+	req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/event?from=%d&through=%d", from, through), nil)
+	if err != nil {
+		return nil, err
+	}
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("Error making request: %s", res.Status)
+	}
+	return ioutil.ReadAll(res.Body)
+}
+
 // GetConfig returns the current Config
 func (c Client) GetConfig() (*Config, error) {
 	req, err := c.newRequest("GET", "/api/config/deploy", nil)
@@ -84,6 +100,9 @@ func (c Client) GetConfig() (*Config, error) {
 	if err != nil {
 		return nil, err
 	}
+	if res.StatusCode == 404 {
+		return nil, fmt.Errorf("No configuration uploaded yet.")
+	}
 	if res.StatusCode != 200 {
 		return nil, fmt.Errorf("Error making request: %s", res.Status)
 	}
@@ -116,7 +135,7 @@ func (c Client) SetConfig(config *Config) error {
 
 // GetLogs returns the logs for a given deployment.
 func (c Client) GetLogs(deployID string) ([]byte, error) {
-	req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/%s/log", deployID), nil)
+	req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/deploy/%s/log", deployID), nil)
 	if err != nil {
 		return nil, err
 	}

cmd/wcloud/types.go

@@ -12,24 +12,32 @@ type Deployment struct {
 	Version   string    `json:"version"`
 	Priority  int       `json:"priority"`
 	State     string    `json:"status"`
-	LogKey    string    `json:"-"`
+
+	TriggeringUser   string   `json:"triggering_user"`
+	IntendedServices []string `json:"intended_services"`
 }
 
 // Config for the deployment system for a user.
 type Config struct {
 	RepoURL        string `json:"repo_url" yaml:"repo_url"`
+	RepoBranch     string `json:"repo_branch" yaml:"repo_branch"`
 	RepoPath       string `json:"repo_path" yaml:"repo_path"`
 	RepoKey        string `json:"repo_key" yaml:"repo_key"`
 	KubeconfigPath string `json:"kubeconfig_path" yaml:"kubeconfig_path"`
+	AutoApply      bool   `json:"auto_apply" yaml:"auto_apply"`
 
 	Notifications []NotificationConfig `json:"notifications" yaml:"notifications"`
 
 	// Globs of files not to change, relative to the route of the repo
 	ConfigFileBlackList []string `json:"config_file_black_list" yaml:"config_file_black_list"`
+
+	CommitMessageTemplate string `json:"commit_message_template" yaml:"commit_message_template"` // See https://golang.org/pkg/text/template/
 }
 
 // NotificationConfig describes how to send notifications
 type NotificationConfig struct {
-	SlackWebhookURL string `json:"slack_webhook_url" yaml:"slack_webhook_url"`
-	SlackUsername   string `json:"slack_username" yaml:"slack_username"`
+	SlackWebhookURL      string `json:"slack_webhook_url" yaml:"slack_webhook_url"`
+	SlackUsername        string `json:"slack_username" yaml:"slack_username"`
+	MessageTemplate      string `json:"message_template" yaml:"message_template"`
+	ApplyMessageTemplate string `json:"apply_message_template" yaml:"apply_message_template"`
 }

image-tag

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+WORKING_SUFFIX=$(if ! git diff --exit-code --quiet HEAD >&2; \
+                 then echo "-WIP"; \
+                 else echo ""; \
+                 fi)
+BRANCH_PREFIX=$(git rev-parse --abbrev-ref HEAD)
+echo "${BRANCH_PREFIX//\//-}-$(git rev-parse --short HEAD)$WORKING_SUFFIX"

integration/gce.sh

@@ -39,6 +39,7 @@ function vm_names {
 
 # Delete all vms in this account
 function destroy {
+	local names
 	names="$(vm_names)"
 	if [ "$(gcloud compute instances list --zone "$ZONE" -q "$names" | wc -l)" -le 1 ] ; then
 		return 0

lint

@@ -40,11 +40,11 @@ function spell_check {
 	filename="$1"
 	local lint_result=0
 
-	# we don't want to spell check tar balls or binaries
+	# we don't want to spell check tar balls, binaries, Makefile and json files
 	if file "$filename" | grep executable >/dev/null 2>&1; then
 		return $lint_result
 	fi
-	if [[ $filename == *".tar" || $filename == *".gz" ]]; then
+	if [[ $filename == *".tar" || $filename == *".gz" || $filename == *".json" || $(basename "$filename") == "Makefile" ]]; then
 		return $lint_result
 	fi
 

rebuild-image

@@ -40,6 +40,13 @@ commit_timestamp() {
 	git show -s --format=%ct "$rev"
 }
 
+# Is the SHA1 actually present in the repo?
+# It could be it isn't, e.g. after a force push
+is_valid_commit() {
+	local rev=$1
+	git rev-parse --quiet --verify "$rev^{commit}" > /dev/null
+}
+
 cached_revision=$(cached_image_rev)
 if [ -z "$cached_revision" ]; then
 	echo ">>> No cached image found; rebuilding"
@@ -47,6 +54,12 @@ if [ -z "$cached_revision" ]; then
 	exit 0
 fi
 
+if ! is_valid_commit "$cached_revision"; then
+	echo ">>> Git commit of cached image not found in repo; rebuilding"
+	rebuild
+	exit 0
+fi
+
 echo ">>> Found cached image rev $cached_revision"
 if has_changes "$cached_revision" "$CIRCLE_SHA1" ; then
 	echo ">>> Found changes, rebuilding"

runner/runner.go

@@ -14,8 +14,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/docker/docker/pkg/mflag"
 	"github.com/mgutz/ansi"
+	"github.com/weaveworks/docker/pkg/mflag"
 )
 
 const (

shell-lint

@@ -10,4 +10,4 @@
 # - files-with-type
 # - file >= 5.22
 
-"$(dirname "${BASH_SOURCE[0]}")/files-with-type" text/x-shellscript "$@" | xargs shellcheck
+"$(dirname "${BASH_SOURCE[0]}")/files-with-type" text/x-shellscript "$@" | xargs --no-run-if-empty shellcheck

socks/main.go

@@ -9,7 +9,7 @@ import (
 	"text/template"
 
 	socks5 "github.com/armon/go-socks5"
-	"github.com/docker/docker/pkg/mflag"
+	"github.com/weaveworks/docker/pkg/mflag"
 	"github.com/weaveworks/weave/common/mflagext"
 	"golang.org/x/net/context"
 )

test

@@ -47,8 +47,15 @@ fi
 
 fail=0
 
-# NB: Relies on paths being prefixed with './'.
-TESTDIRS=( $(git ls-files -- '*_test.go' | grep -vE '^(vendor|prog|experimental)/' | xargs -n1 dirname  | sort -u | sed -e 's|^|./|') )
+if [ -z "$TESTDIRS" ]; then
+    # NB: Relies on paths being prefixed with './'.
+    TESTDIRS=( $(git ls-files -- '*_test.go' | grep -vE '^(vendor|prog|experimental)/' | xargs -n1 dirname  | sort -u | sed -e 's|^|./|') )
+else
+    # TESTDIRS on the right side is not really an array variable, it
+    # is just a string with spaces, but it is written like that to
+    # shut up the shellcheck tool.
+    TESTDIRS=( $(for d in ${TESTDIRS[*]}; do echo "$d"; done) )
+fi
 
 # If running on circle, use the scheduler to work out what tests to run on what shard
 if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ] && [ -x "$DIR/sched" ]; then