From 7eb07eebff9bc1297c87a04ee4d703b491bc8ca2 Mon Sep 17 00:00:00 2001 From: Lorenzo Manacorda Date: Wed, 19 Oct 2016 12:36:22 +0200 Subject: [PATCH] Squashed 'tools/' changes from e9e7e6b..b990f48 b990f48 Merge pull request #42 from kinvolk/lorenzo/fix-git-diff 224a145 Check if SHA1 exists before calling `git diff` 1c3000d Add auto_apply config for wcloud 0ebf5c0 Fix wcloud -serivice 4fe078a Merge pull request #39 from weaveworks/fix-wrong-subtree-use 3f4934d Remove generate_latest_map 48beb60 Sync changes done directly in scope/tools 45dcdd5 Merge pull request #37 from weaveworks/fix-mflag-missing b895344 Use mflag package from weaveworks fork until we find a better solution e030008 Merge pull request #36 from weaveworks/wcloud-service-flags 9cbab40 Add wcloud Makefile ef55901 Review feedback, and build wcloud in circle. 3fe92f5 Add wcloud deploy --service flag 3527b56 Merge pull request #34 from weaveworks/repo-branch 92cd0b8 [wcloud] Add support for repo_branch option 9f760ab Allow wcloud users to override username 38037f8 Merge pull request #33 from weaveworks/wcloud-templates 7acfbd7 Propagate the local username e6876d1 Add template fields to wcloud config. f1bb537 Merge pull request #30 from weaveworks/mike/shell-lint/dont-error-if-empty e60f5df Merge pull request #31 from weaveworks/mike/fix-shell-lint-errors e8e2b69 integrations: Fix a shellcheck linter error a781575 shell-lint: Don't fail if no shell scripts found db5efc0 Merge pull request #28 from weaveworks/mike/add-image-tag 5312c40 Import image-tag script into build tools so it can be shared 7e850f8 Fix logs path dda9785 Update deploy api f2f4e5b Fix the wcloud client 3925eb6 Merge pull request #27 from weaveworks/wcloud-events 77355b9 Lint d9a1c6c Add wcloud events, update flags and error nicely when there is no config git-subtree-dir: tools git-subtree-split: b990f488bdc7909b62d9245bc4b4caf58f5ae7ea --- .gitignore | 1 + circle.yml | 1 + cmd/wcloud/Makefile | 11 +++++++ cmd/wcloud/cli.go | 72 ++++++++++++++++++++++++++++++++++++++++---- cmd/wcloud/client.go | 27 ++++++++++++++--- cmd/wcloud/types.go | 14 +++++++-- image-tag | 12 ++++++++ integration/gce.sh | 1 + lint | 4 +-- rebuild-image | 13 ++++++++ runner/runner.go | 2 +- shell-lint | 2 +- socks/main.go | 2 +- test | 11 +++++-- 14 files changed, 153 insertions(+), 20 deletions(-) create mode 100644 cmd/wcloud/Makefile create mode 100755 image-tag diff --git a/.gitignore b/.gitignore index b6ea60f8f..635dff1f1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,6 @@ cover/cover socks/proxy socks/image.tar runner/runner +cmd/wcloud/wcloud *.pyc *~ diff --git a/circle.yml b/circle.yml index 0c1ebddb1..74cf3d40a 100644 --- a/circle.yml +++ b/circle.yml @@ -20,4 +20,5 @@ test: - cd $SRCDIR/cover; make - cd $SRCDIR/socks; make - cd $SRCDIR/runner; make + - cd $SRCDIR/cmd/wcloud; make diff --git a/cmd/wcloud/Makefile b/cmd/wcloud/Makefile new file mode 100644 index 000000000..86b0df380 --- /dev/null +++ b/cmd/wcloud/Makefile @@ -0,0 +1,11 @@ +.PHONY: all clean + +all: wcloud + +wcloud: *.go + go get ./$(@D) + go build -o $@ ./$(@D) + +clean: + rm -rf wcloud + go clean ./... diff --git a/cmd/wcloud/cli.go b/cmd/wcloud/cli.go index e9dd1aa39..ba3c355fd 100644 --- a/cmd/wcloud/cli.go +++ b/cmd/wcloud/cli.go @@ -7,6 +7,7 @@ import ( "fmt" "io/ioutil" "os" + "os/user" "path/filepath" "strings" "time" @@ -15,6 +16,19 @@ import ( "gopkg.in/yaml.v2" ) +// ArrayFlags allows you to collect repeated flags +type ArrayFlags []string + +func (a *ArrayFlags) String() string { + return strings.Join(*a, ",") +} + +// Set implements flags.Value +func (a *ArrayFlags) Set(value string) error { + *a = append(*a, value) + return nil +} + func env(key, def string) string { if val, ok := os.LookupEnv(key); ok { return val @@ -52,6 +66,8 @@ func main() { config(c, os.Args[2:]) case "logs": logs(c, os.Args[2:]) + case "events": + events(c, os.Args[2:]) case "help": usage() default: @@ -60,6 +76,17 @@ func main() { } func deploy(c Client, args []string) { + var ( + flags = flag.NewFlagSet("", flag.ContinueOnError) + username = flags.String("u", "", "Username to report to deploy service (default with be current user)") + services ArrayFlags + ) + flags.Var(&services, "service", "Service to update (can be repeated)") + if err := flags.Parse(args); err != nil { + usage() + return + } + args = flags.Args() if len(args) != 1 { usage() return @@ -69,9 +96,19 @@ func deploy(c Client, args []string) { usage() return } + if *username == "" { + user, err := user.Current() + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + *username = user.Username + } deployment := Deployment{ - ImageName: parts[0], - Version: parts[1], + ImageName: parts[0], + Version: parts[1], + TriggeringUser: *username, + IntendedServices: services, } if err := c.Deploy(deployment); err != nil { fmt.Println(err.Error()) @@ -80,14 +117,17 @@ func deploy(c Client, args []string) { } func list(c Client, args []string) { - flags := flag.NewFlagSet("list", flag.ContinueOnError) - page := flags.Int("page", 0, "Zero based index of page to list.") - pagesize := flags.Int("page-size", 10, "Number of results per page") + var ( + flags = flag.NewFlagSet("", flag.ContinueOnError) + since = flags.Duration("since", 7*24*time.Hour, "How far back to fetch results") + ) if err := flags.Parse(args); err != nil { usage() return } - deployments, err := c.GetDeployments(*page, *pagesize) + through := time.Now() + from := through.Add(-*since) + deployments, err := c.GetDeployments(from.Unix(), through.Unix()) if err != nil { fmt.Println(err.Error()) os.Exit(1) @@ -109,6 +149,26 @@ func list(c Client, args []string) { table.Render() } +func events(c Client, args []string) { + var ( + flags = flag.NewFlagSet("", flag.ContinueOnError) + since = flags.Duration("since", 7*24*time.Hour, "How far back to fetch results") + ) + if err := flags.Parse(args); err != nil { + usage() + return + } + through := time.Now() + from := through.Add(-*since) + events, err := c.GetEvents(from.Unix(), through.Unix()) + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + fmt.Println("events: ", string(events)) +} + func loadConfig(filename string) (*Config, error) { extension := filepath.Ext(filename) var config Config diff --git a/cmd/wcloud/client.go b/cmd/wcloud/client.go index 7f21b817e..57836cd9a 100644 --- a/cmd/wcloud/client.go +++ b/cmd/wcloud/client.go @@ -38,7 +38,7 @@ func (c Client) Deploy(deployment Deployment) error { if err := json.NewEncoder(&buf).Encode(deployment); err != nil { return err } - req, err := c.newRequest("POST", "/api/deploy", &buf) + req, err := c.newRequest("POST", "/api/deploy/deploy", &buf) if err != nil { return err } @@ -53,8 +53,8 @@ func (c Client) Deploy(deployment Deployment) error { } // GetDeployments returns a list of deployments -func (c Client) GetDeployments(page, pagesize int) ([]Deployment, error) { - req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy?page=%d&pagesize=%d", page, pagesize), nil) +func (c Client) GetDeployments(from, through int64) ([]Deployment, error) { + req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/deploy?from=%d&through=%d", from, through), nil) if err != nil { return nil, err } @@ -74,6 +74,22 @@ func (c Client) GetDeployments(page, pagesize int) ([]Deployment, error) { return response.Deployments, nil } +// GetEvents returns the raw events. +func (c Client) GetEvents(from, through int64) ([]byte, error) { + req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/event?from=%d&through=%d", from, through), nil) + if err != nil { + return nil, err + } + res, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + if res.StatusCode != 200 { + return nil, fmt.Errorf("Error making request: %s", res.Status) + } + return ioutil.ReadAll(res.Body) +} + // GetConfig returns the current Config func (c Client) GetConfig() (*Config, error) { req, err := c.newRequest("GET", "/api/config/deploy", nil) @@ -84,6 +100,9 @@ func (c Client) GetConfig() (*Config, error) { if err != nil { return nil, err } + if res.StatusCode == 404 { + return nil, fmt.Errorf("No configuration uploaded yet.") + } if res.StatusCode != 200 { return nil, fmt.Errorf("Error making request: %s", res.Status) } @@ -116,7 +135,7 @@ func (c Client) SetConfig(config *Config) error { // GetLogs returns the logs for a given deployment. func (c Client) GetLogs(deployID string) ([]byte, error) { - req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/%s/log", deployID), nil) + req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/deploy/%s/log", deployID), nil) if err != nil { return nil, err } diff --git a/cmd/wcloud/types.go b/cmd/wcloud/types.go index b02ee4c14..a068163c3 100644 --- a/cmd/wcloud/types.go +++ b/cmd/wcloud/types.go @@ -12,24 +12,32 @@ type Deployment struct { Version string `json:"version"` Priority int `json:"priority"` State string `json:"status"` - LogKey string `json:"-"` + + TriggeringUser string `json:"triggering_user"` + IntendedServices []string `json:"intended_services"` } // Config for the deployment system for a user. type Config struct { RepoURL string `json:"repo_url" yaml:"repo_url"` + RepoBranch string `json:"repo_branch" yaml:"repo_branch"` RepoPath string `json:"repo_path" yaml:"repo_path"` RepoKey string `json:"repo_key" yaml:"repo_key"` KubeconfigPath string `json:"kubeconfig_path" yaml:"kubeconfig_path"` + AutoApply bool `json:"auto_apply" yaml:"auto_apply"` Notifications []NotificationConfig `json:"notifications" yaml:"notifications"` // Globs of files not to change, relative to the route of the repo ConfigFileBlackList []string `json:"config_file_black_list" yaml:"config_file_black_list"` + + CommitMessageTemplate string `json:"commit_message_template" yaml:"commit_message_template"` // See https://golang.org/pkg/text/template/ } // NotificationConfig describes how to send notifications type NotificationConfig struct { - SlackWebhookURL string `json:"slack_webhook_url" yaml:"slack_webhook_url"` - SlackUsername string `json:"slack_username" yaml:"slack_username"` + SlackWebhookURL string `json:"slack_webhook_url" yaml:"slack_webhook_url"` + SlackUsername string `json:"slack_username" yaml:"slack_username"` + MessageTemplate string `json:"message_template" yaml:"message_template"` + ApplyMessageTemplate string `json:"apply_message_template" yaml:"apply_message_template"` } diff --git a/image-tag b/image-tag new file mode 100755 index 000000000..edee35930 --- /dev/null +++ b/image-tag @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +WORKING_SUFFIX=$(if ! git diff --exit-code --quiet HEAD >&2; \ + then echo "-WIP"; \ + else echo ""; \ + fi) +BRANCH_PREFIX=$(git rev-parse --abbrev-ref HEAD) +echo "${BRANCH_PREFIX//\//-}-$(git rev-parse --short HEAD)$WORKING_SUFFIX" diff --git a/integration/gce.sh b/integration/gce.sh index 1f1019fce..dd0e92770 100755 --- a/integration/gce.sh +++ b/integration/gce.sh @@ -39,6 +39,7 @@ function vm_names { # Delete all vms in this account function destroy { + local names names="$(vm_names)" if [ "$(gcloud compute instances list --zone "$ZONE" -q "$names" | wc -l)" -le 1 ] ; then return 0 diff --git a/lint b/lint index 994d4c321..30e8f904e 100755 --- a/lint +++ b/lint @@ -40,11 +40,11 @@ function spell_check { filename="$1" local lint_result=0 - # we don't want to spell check tar balls or binaries + # we don't want to spell check tar balls, binaries, Makefile and json files if file "$filename" | grep executable >/dev/null 2>&1; then return $lint_result fi - if [[ $filename == *".tar" || $filename == *".gz" ]]; then + if [[ $filename == *".tar" || $filename == *".gz" || $filename == *".json" || $(basename "$filename") == "Makefile" ]]; then return $lint_result fi diff --git a/rebuild-image b/rebuild-image index a579a47de..bc1360bee 100755 --- a/rebuild-image +++ b/rebuild-image @@ -40,6 +40,13 @@ commit_timestamp() { git show -s --format=%ct "$rev" } +# Is the SHA1 actually present in the repo? +# It could be it isn't, e.g. after a force push +is_valid_commit() { + local rev=$1 + git rev-parse --quiet --verify "$rev^{commit}" > /dev/null +} + cached_revision=$(cached_image_rev) if [ -z "$cached_revision" ]; then echo ">>> No cached image found; rebuilding" @@ -47,6 +54,12 @@ if [ -z "$cached_revision" ]; then exit 0 fi +if ! is_valid_commit "$cached_revision"; then + echo ">>> Git commit of cached image not found in repo; rebuilding" + rebuild + exit 0 +fi + echo ">>> Found cached image rev $cached_revision" if has_changes "$cached_revision" "$CIRCLE_SHA1" ; then echo ">>> Found changes, rebuilding" diff --git a/runner/runner.go b/runner/runner.go index 707369d81..c92ac6b5b 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -14,8 +14,8 @@ import ( "sync" "time" - "github.com/docker/docker/pkg/mflag" "github.com/mgutz/ansi" + "github.com/weaveworks/docker/pkg/mflag" ) const ( diff --git a/shell-lint b/shell-lint index 5cc77cb2a..fffbb8b11 100755 --- a/shell-lint +++ b/shell-lint @@ -10,4 +10,4 @@ # - files-with-type # - file >= 5.22 -"$(dirname "${BASH_SOURCE[0]}")/files-with-type" text/x-shellscript "$@" | xargs shellcheck +"$(dirname "${BASH_SOURCE[0]}")/files-with-type" text/x-shellscript "$@" | xargs --no-run-if-empty shellcheck diff --git a/socks/main.go b/socks/main.go index 520df27d9..83a214980 100644 --- a/socks/main.go +++ b/socks/main.go @@ -9,7 +9,7 @@ import ( "text/template" socks5 "github.com/armon/go-socks5" - "github.com/docker/docker/pkg/mflag" + "github.com/weaveworks/docker/pkg/mflag" "github.com/weaveworks/weave/common/mflagext" "golang.org/x/net/context" ) diff --git a/test b/test index f8d76c56b..4ee4ca1a4 100755 --- a/test +++ b/test @@ -47,8 +47,15 @@ fi fail=0 -# NB: Relies on paths being prefixed with './'. -TESTDIRS=( $(git ls-files -- '*_test.go' | grep -vE '^(vendor|prog|experimental)/' | xargs -n1 dirname | sort -u | sed -e 's|^|./|') ) +if [ -z "$TESTDIRS" ]; then + # NB: Relies on paths being prefixed with './'. + TESTDIRS=( $(git ls-files -- '*_test.go' | grep -vE '^(vendor|prog|experimental)/' | xargs -n1 dirname | sort -u | sed -e 's|^|./|') ) +else + # TESTDIRS on the right side is not really an array variable, it + # is just a string with spaces, but it is written like that to + # shut up the shellcheck tool. + TESTDIRS=( $(for d in ${TESTDIRS[*]}; do echo "$d"; done) ) +fi # If running on circle, use the scheduler to work out what tests to run on what shard if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ] && [ -x "$DIR/sched" ]; then