From 7a35f21ea33d03d1d14c58b8b3dc780566908b3c Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Tue, 12 Jul 2016 14:21:55 +0100 Subject: [PATCH 1/3] Squashed 'tools/' changes from 7a66090..8c6170d 8c6170d Fix lint in all the build-tools scripts 239935c shell-lint tool d9ab133 Script for finding files with a given type 1b64e46 Add Weave Cloud client f2e40b4 Time out commands after three minutes 2da55ce Don't spell-check compressed files e9749a5 Make scheduler aware of test parallelisation git-subtree-dir: tools git-subtree-split: 8c6170d292d34923d23d5eef6b6ac5e4af6b7188 --- .gitignore | 2 + README.md | 4 + cmd/wcloud/cli.go | 178 ++++++++++++++++++++++++++++++++++++ cmd/wcloud/client.go | 131 ++++++++++++++++++++++++++ cmd/wcloud/types.go | 24 +++++ cover/gather_coverage.sh | 11 +-- files-with-type | 13 +++ integration/assert.sh | 20 ++-- integration/gce.sh | 88 +++++++++--------- integration/run_all.sh | 20 ++-- integration/sanity_check.sh | 14 +-- lint | 14 +-- publish-site | 2 +- rebuild-image | 26 +++--- runner/runner.go | 21 ++++- sched | 26 +++--- scheduler/main.py | 15 ++- shell-lint | 13 +++ socks/connect.sh | 11 ++- test | 49 +++++----- 20 files changed, 543 insertions(+), 139 deletions(-) create mode 100644 cmd/wcloud/cli.go create mode 100644 cmd/wcloud/client.go create mode 100644 cmd/wcloud/types.go create mode 100755 files-with-type create mode 100755 shell-lint diff --git a/.gitignore b/.gitignore index f5c83426c..b6ea60f8f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ cover/cover socks/proxy socks/image.tar runner/runner +*.pyc +*~ diff --git a/README.md b/README.md index 32ddb57b1..e570ef717 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,15 @@ Included in this repo are tools shared by weave.git and scope.git. They include - ```cover```: a tool which merges overlapping coverage reports generated by go test +- ```files-with-type```: a tool to search directories for files of a given + MIME type - ```lint```: a script to lint Go project; runs various tools like golint, go vet, errcheck etc - ```rebuild-image```: a script to rebuild docker images when their input files change; useful when you using docker images to build your software, but you don't want to build the image every time. +- ```shell-lint```: a script to lint multiple shell files with + [shellcheck](http://www.shellcheck.net/) - ```socks```: a simple, dockerised SOCKS proxy for getting your laptop onto the Weave network - ```test```: a script to run all go unit tests in subdirectories, gather the diff --git a/cmd/wcloud/cli.go b/cmd/wcloud/cli.go new file mode 100644 index 000000000..e9dd1aa39 --- /dev/null +++ b/cmd/wcloud/cli.go @@ -0,0 +1,178 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" + + "github.com/olekukonko/tablewriter" + "gopkg.in/yaml.v2" +) + +func env(key, def string) string { + if val, ok := os.LookupEnv(key); ok { + return val + } + return def +} + +var ( + token = env("SERVICE_TOKEN", "") + baseURL = env("BASE_URL", "https://cloud.weave.works") +) + +func usage() { + fmt.Println(`Usage: + deploy : Deploy image to your configured env + list List recent deployments + config () Get (or set) the configured env + logs Show lots for the given deployment`) +} + +func main() { + if len(os.Args) <= 1 { + usage() + os.Exit(1) + } + + c := NewClient(token, baseURL) + + switch os.Args[1] { + case "deploy": + deploy(c, os.Args[2:]) + case "list": + list(c, os.Args[2:]) + case "config": + config(c, os.Args[2:]) + case "logs": + logs(c, os.Args[2:]) + case "help": + usage() + default: + usage() + } +} + +func deploy(c Client, args []string) { + if len(args) != 1 { + usage() + return + } + parts := strings.SplitN(args[0], ":", 2) + if len(parts) < 2 { + usage() + return + } + deployment := Deployment{ + ImageName: parts[0], + Version: parts[1], + } + if err := c.Deploy(deployment); err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } +} + +func list(c Client, args []string) { + flags := flag.NewFlagSet("list", flag.ContinueOnError) + page := flags.Int("page", 0, "Zero based index of page to list.") + pagesize := flags.Int("page-size", 10, "Number of results per page") + if err := flags.Parse(args); err != nil { + usage() + return + } + deployments, err := c.GetDeployments(*page, *pagesize) + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + table := tablewriter.NewWriter(os.Stdout) + table.SetHeader([]string{"Created", "ID", "Image", "Version", "State"}) + table.SetBorder(false) + table.SetColumnSeparator(" ") + for _, deployment := range deployments { + table.Append([]string{ + deployment.CreatedAt.Format(time.RFC822), + deployment.ID, + deployment.ImageName, + deployment.Version, + deployment.State, + }) + } + table.Render() +} + +func loadConfig(filename string) (*Config, error) { + extension := filepath.Ext(filename) + var config Config + buf, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + if extension == ".yaml" || extension == ".yml" { + if err := yaml.Unmarshal(buf, &config); err != nil { + return nil, err + } + } else { + if err := json.NewDecoder(bytes.NewReader(buf)).Decode(&config); err != nil { + return nil, err + } + } + return &config, nil +} + +func config(c Client, args []string) { + if len(args) > 1 { + usage() + return + } + + if len(args) == 1 { + config, err := loadConfig(args[0]) + if err != nil { + fmt.Println("Error reading config:", err) + os.Exit(1) + } + + if err := c.SetConfig(config); err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + } else { + config, err := c.GetConfig() + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + buf, err := yaml.Marshal(config) + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + fmt.Println(string(buf)) + } +} + +func logs(c Client, args []string) { + if len(args) != 1 { + usage() + return + } + + output, err := c.GetLogs(args[0]) + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + fmt.Println(string(output)) +} diff --git a/cmd/wcloud/client.go b/cmd/wcloud/client.go new file mode 100644 index 000000000..7f21b817e --- /dev/null +++ b/cmd/wcloud/client.go @@ -0,0 +1,131 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" +) + +// Client for the deployment service +type Client struct { + token string + baseURL string +} + +// NewClient makes a new Client +func NewClient(token, baseURL string) Client { + return Client{ + token: token, + baseURL: baseURL, + } +} + +func (c Client) newRequest(method, path string, body io.Reader) (*http.Request, error) { + req, err := http.NewRequest(method, c.baseURL+path, body) + if err != nil { + return nil, err + } + req.Header.Add("Authorization", fmt.Sprintf("Scope-Probe token=%s", c.token)) + return req, nil +} + +// Deploy notifies the deployment service about a new deployment +func (c Client) Deploy(deployment Deployment) error { + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(deployment); err != nil { + return err + } + req, err := c.newRequest("POST", "/api/deploy", &buf) + if err != nil { + return err + } + res, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + if res.StatusCode != 204 { + return fmt.Errorf("Error making request: %s", res.Status) + } + return nil +} + +// GetDeployments returns a list of deployments +func (c Client) GetDeployments(page, pagesize int) ([]Deployment, error) { + req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy?page=%d&pagesize=%d", page, pagesize), nil) + if err != nil { + return nil, err + } + res, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + if res.StatusCode != 200 { + return nil, fmt.Errorf("Error making request: %s", res.Status) + } + var response struct { + Deployments []Deployment `json:"deployments"` + } + if err := json.NewDecoder(res.Body).Decode(&response); err != nil { + return nil, err + } + return response.Deployments, nil +} + +// GetConfig returns the current Config +func (c Client) GetConfig() (*Config, error) { + req, err := c.newRequest("GET", "/api/config/deploy", nil) + if err != nil { + return nil, err + } + res, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + if res.StatusCode != 200 { + return nil, fmt.Errorf("Error making request: %s", res.Status) + } + var config Config + if err := json.NewDecoder(res.Body).Decode(&config); err != nil { + return nil, err + } + return &config, nil +} + +// SetConfig sets the current Config +func (c Client) SetConfig(config *Config) error { + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(config); err != nil { + return err + } + req, err := c.newRequest("POST", "/api/config/deploy", &buf) + if err != nil { + return err + } + res, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + if res.StatusCode != 204 { + return fmt.Errorf("Error making request: %s", res.Status) + } + return nil +} + +// GetLogs returns the logs for a given deployment. +func (c Client) GetLogs(deployID string) ([]byte, error) { + req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/%s/log", deployID), nil) + if err != nil { + return nil, err + } + res, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + if res.StatusCode != 200 { + return nil, fmt.Errorf("Error making request: %s", res.Status) + } + return ioutil.ReadAll(res.Body) +} diff --git a/cmd/wcloud/types.go b/cmd/wcloud/types.go new file mode 100644 index 000000000..16e6b2622 --- /dev/null +++ b/cmd/wcloud/types.go @@ -0,0 +1,24 @@ +package main + +import ( + "time" +) + +// Deployment describes a deployment +type Deployment struct { + ID string `json:"id"` + CreatedAt time.Time `json:"created_at"` + ImageName string `json:"image_name"` + Version string `json:"version"` + Priority int `json:"priority"` + State string `json:"status"` + LogKey string `json:"-"` +} + +// Config for the deployment system for a user. +type Config struct { + RepoURL string `json:"repo_url" yaml:"repo_url"` + RepoPath string `json:"repo_path" yaml:"repo_path"` + RepoKey string `json:"repo_key" yaml:"repo_key"` + KubeconfigPath string `json:"kubeconfig_path" yaml:"kubeconfig_path"` +} diff --git a/cover/gather_coverage.sh b/cover/gather_coverage.sh index 9026745a1..271ac7d40 100755 --- a/cover/gather_coverage.sh +++ b/cover/gather_coverage.sh @@ -3,19 +3,18 @@ # merges them and produces a complete report. set -ex -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DESTINATION=$1 FROMDIR=$2 -mkdir -p $DESTINATION +mkdir -p "$DESTINATION" if [ -n "$CIRCLECI" ]; then - for i in $(seq 1 $(($CIRCLE_NODE_TOTAL - 1))); do - scp node$i:$FROMDIR/* $DESTINATION || true + for i in $(seq 1 $((CIRCLE_NODE_TOTAL - 1))); do + scp "node$i:$FROMDIR"/* "$DESTINATION" || true done fi go get github.com/weaveworks/build-tools/cover -cover $DESTINATION/* >profile.cov +cover "$DESTINATION"/* >profile.cov go tool cover -html=profile.cov -o coverage.html go tool cover -func=profile.cov -o coverage.txt -tar czf coverage.tar.gz $DESTINATION +tar czf coverage.tar.gz "$DESTINATION" diff --git a/files-with-type b/files-with-type new file mode 100755 index 000000000..d969f4405 --- /dev/null +++ b/files-with-type @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +# Find all files with a given MIME type. +# +# e.g. +# $ files-with-type text/x-shellscript k8s infra +# +# Assumes `find`, `xargs`, and `file` are all installed. + +mime_type=$1 +shift + +find "$@" -print0 -type f |xargs -0 file --mime-type | grep "${mime_type}" | sed -e 's/:.*$//' diff --git a/integration/assert.sh b/integration/assert.sh index 1a2f1f778..da689b3eb 100644 --- a/integration/assert.sh +++ b/integration/assert.sh @@ -24,14 +24,14 @@ export INVARIANT=${INVARIANT:-} export CONTINUE=${CONTINUE:-} args="$(getopt -n "$0" -l \ - verbose,help,stop,discover,invariant,continue vhxdic $*)" \ + verbose,help,stop,discover,invariant,continue vhxdic "$@")" \ || exit -1 for arg in $args; do case "$arg" in -h) echo "$0 [-vxidc]" \ "[--verbose] [--stop] [--invariant] [--discover] [--continue]" - echo "`sed 's/./ /g' <<< "$0"` [-h] [--help]" + echo "$(sed 's/./ /g' <<< "$0") [-h] [--help]" exit 0;; --help) cat < /dev/null 2>&1 || status=$? + (eval "$1" <<< "${3:-}") > /dev/null 2>&1 || status=$? expected=${2:-0} if [[ "$status" -eq "$expected" ]]; then [[ -z "$DEBUG" ]] || echo -n . @@ -143,7 +143,7 @@ _assert_fail() { skip_if() { # skip_if - (eval $@) > /dev/null 2>&1 && status=0 || status=$? + (eval "$@") > /dev/null 2>&1 && status=0 || status=$? [[ "$status" -eq 0 ]] || return skip } diff --git a/integration/gce.sh b/integration/gce.sh index bb6f95efa..1f1019fce 100755 --- a/integration/gce.sh +++ b/integration/gce.sh @@ -7,15 +7,15 @@ set -e -: ${KEY_FILE:=/tmp/gce_private_key.json} -: ${SSH_KEY_FILE:=$HOME/.ssh/gce_ssh_key} -: ${IMAGE:=ubuntu-14-04} -: ${ZONE:=us-central1-a} -: ${PROJECT:=} -: ${TEMPLATE_NAME:=} -: ${NUM_HOSTS:=} +: "${KEY_FILE:=/tmp/gce_private_key.json}" +: "${SSH_KEY_FILE:=$HOME/.ssh/gce_ssh_key}" +: "${IMAGE:=ubuntu-14-04}" +: "${ZONE:=us-central1-a}" +: "${PROJECT:=}" +: "${TEMPLATE_NAME:=}" +: "${NUM_HOSTS:=}" -if [ -z "${PROJECT}" -o -z "${NUM_HOSTS}" -o -z "${TEMPLATE_NAME}" ]; then +if [ -z "${PROJECT}" ] || [ -z "${NUM_HOSTS}" ] || [ -z "${TEMPLATE_NAME}" ]; then echo "Must specify PROJECT, NUM_HOSTS and TEMPLATE_NAME" exit 1 fi @@ -26,21 +26,21 @@ if [ -n "$CIRCLECI" ]; then fi # Setup authentication -gcloud auth activate-service-account --key-file $KEY_FILE 1>/dev/null -gcloud config set project $PROJECT +gcloud auth activate-service-account --key-file "$KEY_FILE" 1>/dev/null +gcloud config set project "$PROJECT" function vm_names { local names= - for i in $(seq 1 $NUM_HOSTS); do - names="host$i$SUFFIX $names" + for i in $(seq 1 "$NUM_HOSTS"); do + names=( "host$i$SUFFIX" "${names[@]}" ) done - echo "$names" + echo "${names[@]}" } # Delete all vms in this account function destroy { names="$(vm_names)" - if [ $(gcloud compute instances list --zone $ZONE -q $names | wc -l) -le 1 ] ; then + if [ "$(gcloud compute instances list --zone "$ZONE" -q "$names" | wc -l)" -le 1 ] ; then return 0 fi for i in {0..10}; do @@ -60,23 +60,23 @@ function destroy { } function internal_ip { - jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].networkIP" $1 + jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].networkIP" "$1" } function external_ip { - jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].accessConfigs[0].natIP" $1 + jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].accessConfigs[0].natIP" "$1" } function try_connect { for i in {0..10}; do - ssh -t $1 true && return + ssh -t "$1" true && return sleep 2 done } function install_docker_on { name=$1 - ssh -t $name sudo bash -x -s <>/etc/hosts\"" + ssh -t "$hostname" "sudo -- sh -c \"cat >>/etc/hosts\"" < "$hosts" } # Create new set of VMs function setup { destroy - names="$(vm_names)" - gcloud compute instances create $names --image $TEMPLATE_NAME --zone $ZONE - gcloud compute config-ssh --ssh-key-file $SSH_KEY_FILE + names=( $(vm_names) ) + gcloud compute instances create "${names[@]}" --image "$TEMPLATE_NAME" --zone "$ZONE" + gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE" sed -i '/UserKnownHostsFile=\/dev\/null/d' ~/.ssh/config # build an /etc/hosts file for these vms hosts=$(mktemp hosts.XXXXXXXXXX) json=$(mktemp json.XXXXXXXXXX) - gcloud compute instances list --format=json >$json - for name in $names; do - echo "$(internal_ip $json $name) $name.$ZONE.$PROJECT" >>$hosts + gcloud compute instances list --format=json > "$json" + for name in "${names[@]}"; do + echo "$(internal_ip "$json" "$name") $name.$ZONE.$PROJECT" >> "$hosts" done - for name in $names; do + for name in "${names[@]}"; do hostname="$name.$ZONE.$PROJECT" # Add the remote ip to the local /etc/hosts sudo sed -i "/$hostname/d" /etc/hosts - sudo sh -c "echo \"$(external_ip $json $name) $hostname\" >>/etc/hosts" - try_connect $hostname + sudo sh -c "echo \"$(external_ip "$json" "$name") $hostname\" >>/etc/hosts" + try_connect "$hostname" - copy_hosts $hostname $hosts & + copy_hosts "$hostname" "$hosts" & done wait - rm $hosts $json + rm "$hosts" "$json" } function make_template { - gcloud compute instances create $TEMPLATE_NAME --image $IMAGE --zone $ZONE - gcloud compute config-ssh --ssh-key-file $SSH_KEY_FILE + gcloud compute instances create "$TEMPLATE_NAME" --image "$IMAGE" --zone "$ZONE" + gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE" name="$TEMPLATE_NAME.$ZONE.$PROJECT" - try_connect $name - install_docker_on $name - gcloud -q compute instances delete $TEMPLATE_NAME --keep-disks boot --zone $ZONE - gcloud compute images create $TEMPLATE_NAME --source-disk $TEMPLATE_NAME --source-disk-zone $ZONE + try_connect "$name" + install_docker_on "$name" + gcloud -q compute instances delete "$TEMPLATE_NAME" --keep-disks boot --zone "$ZONE" + gcloud compute images create "$TEMPLATE_NAME" --source-disk "$TEMPLATE_NAME" --source-disk-zone "$ZONE" } function hosts { hosts= args= json=$(mktemp json.XXXXXXXXXX) - gcloud compute instances list --format=json >$json + gcloud compute instances list --format=json > "$json" for name in $(vm_names); do hostname="$name.$ZONE.$PROJECT" - hosts="$hostname $hosts" - args="--add-host=$hostname:$(internal_ip $json $name) $args" + hosts=( $hostname "${hosts[@]}" ) + args=( "--add-host=$hostname:$(internal_ip "$json" "$name")" "${args[@]}" ) done echo export SSH=\"ssh -l vagrant\" - echo export HOSTS=\"$hosts\" - echo export ADD_HOST_ARGS=\"$args\" - rm $json + echo "export HOSTS=\"${hosts[*]}\"" + echo "export ADD_HOST_ARGS=\"${args[*]}\"" + rm "$json" } case "$1" in @@ -170,7 +170,7 @@ destroy) make_template) # see if template exists - if ! gcloud compute images list | grep $PROJECT | grep $TEMPLATE_NAME; then + if ! gcloud compute images list | grep "$PROJECT" | grep "$TEMPLATE_NAME"; then make_template fi esac diff --git a/integration/run_all.sh b/integration/run_all.sh index 13cb82c41..cb6d93d2c 100755 --- a/integration/run_all.sh +++ b/integration/run_all.sh @@ -1,6 +1,9 @@ #!/bin/bash +set -ex + DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1090 . "$DIR/config.sh" whitely echo Sanity checks @@ -10,18 +13,19 @@ if ! bash "$DIR/sanity_check.sh"; then fi whitely echo ...ok -TESTS="${@:-$(find . -name '*_test.sh')}" -RUNNER_ARGS="" +# shellcheck disable=SC2068 +TESTS=( ${@:-$(find . -name '*_test.sh')} ) +RUNNER_ARGS=( ) # If running on circle, use the scheduler to work out what tests to run -if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" ]; then - RUNNER_ARGS="$RUNNER_ARGS -scheduler" +if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ]; then + RUNNER_ARGS=( "${RUNNER_ARGS[@]}" -scheduler ) fi # If running on circle or PARALLEL is not empty, run tests in parallel -if [ -n "$CIRCLECI" -o -n "$PARALLEL" ]; then - RUNNER_ARGS="$RUNNER_ARGS -parallel" +if [ -n "$CIRCLECI" ] || [ -n "$PARALLEL" ]; then + RUNNER_ARGS=( "${RUNNER_ARGS[@]}" -parallel ) fi -make -C ${DIR}/../runner -HOSTS="$HOSTS" "${DIR}/../runner/runner" $RUNNER_ARGS $TESTS +make -C "${DIR}/../runner" +HOSTS="$HOSTS" "${DIR}/../runner/runner" "${RUNNER_ARGS[@]}" "${TESTS[@]}" diff --git a/integration/sanity_check.sh b/integration/sanity_check.sh index 592a6b77c..c88337fe8 100755 --- a/integration/sanity_check.sh +++ b/integration/sanity_check.sh @@ -1,5 +1,5 @@ #! /bin/bash - +# shellcheck disable=SC1091 . ./config.sh set -e @@ -7,7 +7,7 @@ set -e whitely echo Ping each host from the other for host in $HOSTS; do for other in $HOSTS; do - [ $host = $other ] || run_on $host $PING $other + [ "$host" = "$other" ] || run_on "$host" "$PING" "$other" done done @@ -15,12 +15,12 @@ whitely echo Check we can reach docker for host in $HOSTS; do echo - echo Host Version Info: $host - echo ===================================== + echo "Host Version Info: $host" + echo "=====================================" echo "# docker version" - docker_on $host version + docker_on "$host" version echo "# docker info" - docker_on $host info + docker_on "$host" info echo "# weave version" - weave_on $host version + weave_on "$host" version done diff --git a/lint b/lint index 7d539f5da..e807f8824 100755 --- a/lint +++ b/lint @@ -1,7 +1,7 @@ #!/bin/bash # This scipt lints go files for common errors. # -# Its runs gofmt and go vet, and optionally golint and +# It runs gofmt and go vet, and optionally golint and # gocyclo, if they are installed. # # With no arguments, it lints the current files staged @@ -41,10 +41,10 @@ function spell_check { local lint_result=0 # we don't want to spell check tar balls or binaries - if file $filename | grep executable >/dev/null 2>&1; then + if file "$filename" | grep executable >/dev/null 2>&1; then return $lint_result fi - if [[ $filename == *".tar" ]]; then + if [[ $filename == *".tar" || $filename == *".gz" ]]; then return $lint_result fi @@ -63,11 +63,11 @@ function spell_check { function test_mismatch { filename="$1" - package=$(grep '^package ' $filename | awk '{print $2}') + package=$(grep '^package ' "$filename" | awk '{print $2}') local lint_result=0 if [[ $package == "main" ]]; then - continue # in package main, all bets are off + return # in package main, all bets are off fi if [[ $filename == *"_internal_test.go" ]]; then @@ -115,7 +115,7 @@ function lint_go { # don't have it installed. Also never blocks a commit, # it just warns. if type gocyclo >/dev/null 2>&1; then - gocyclo -over 25 "${filename}" | while read line; do + gocyclo -over 25 "${filename}" | while read -r line; do echo "${filename}": higher than 25 cyclomatic complexity - "${line}" done fi @@ -158,7 +158,7 @@ function lint { function lint_files { local lint_result=0 - while read filename; do + while read -r filename; do lint "${filename}" || lint_result=1 done exit $lint_result diff --git a/publish-site b/publish-site index 4d0984b92..900eb7cb6 100755 --- a/publish-site +++ b/publish-site @@ -3,7 +3,7 @@ set -e set -o pipefail -: ${PRODUCT:=} +: "${PRODUCT:=}" fatal() { echo "$@" >&2 diff --git a/rebuild-image b/rebuild-image index 1e00cbbe0..772ecb934 100755 --- a/rebuild-image +++ b/rebuild-image @@ -5,37 +5,39 @@ set -eux IMAGENAME=$1 -SAVEDNAME=$(echo $IMAGENAME | sed "s/[\/\-]/\./g") +# shellcheck disable=SC2001 +SAVEDNAME=$(echo "$IMAGENAME" | sed "s/[\/\-]/\./g") IMAGEDIR=$2 shift 2 -INPUTFILES=$@ +INPUTFILES=( "$@" ) CACHEDIR=$HOME/docker/ # Rebuild the image rebuild() { - mkdir -p $CACHEDIR - rm $CACHEDIR/$SAVEDNAME* || true - docker build -t $IMAGENAME $IMAGEDIR - docker save $IMAGENAME:latest | gzip - > $CACHEDIR/$SAVEDNAME-$CIRCLE_SHA1.gz + mkdir -p "$CACHEDIR" + rm "$CACHEDIR/$SAVEDNAME"* || true + docker build -t "$IMAGENAME" "$IMAGEDIR" + docker save "$IMAGENAME:latest" | gzip - > "$CACHEDIR/$SAVEDNAME-$CIRCLE_SHA1.gz" } # Get the revision the cached image was build at cached_image_rev() { - find $CACHEDIR -name "$SAVEDNAME-*" -type f | sed -n 's/^[^\-]*\-\([a-z0-9]*\).gz$/\1/p' + find "$CACHEDIR" -name "$SAVEDNAME-*" -type f | sed -n 's/^[^\-]*\-\([a-z0-9]*\).gz$/\1/p' } # Have there been any revision between $1 and $2 has_changes() { local rev1=$1 local rev2=$2 - local changes=$(git diff --oneline $rev1..$rev2 -- $INPUTFILES | wc -l) + local changes + changes=$(git diff --oneline "$rev1..$rev2" -- "${INPUTFILES[@]}" | wc -l) [ "$changes" -gt 0 ] } commit_timestamp() { local rev=$1 - git show -s --format=%ct $rev + git show -s --format=%ct "$rev" } cached_revision=$(cached_image_rev) @@ -46,14 +48,14 @@ if [ -z "$cached_revision" ]; then fi echo ">>> Found cached image rev $cached_revision" -if has_changes $cached_revision $CIRCLE_SHA1 ; then +if has_changes "$cached_revision" "$CIRCLE_SHA1" ; then echo ">>> Found changes, rebuilding" rebuild exit 0 fi IMAGE_TIMEOUT="$(( 3 * 24 * 60 * 60 ))" -if [ "$(commit_timestamp $cached_revision)" -lt "${IMAGE_TIMEOUT}" ]; then +if [ "$(commit_timestamp "$cached_revision")" -lt "${IMAGE_TIMEOUT}" ]; then echo ">>> Image is more the 24hrs old; rebuilding" rebuild exit 0 @@ -61,4 +63,4 @@ fi # we didn't rebuild; import cached version echo ">>> No changes found, importing cached image" -zcat $CACHEDIR/$SAVEDNAME-$cached_revision.gz | docker load +zcat "$CACHEDIR/$SAVEDNAME-$cached_revision.gz" | docker load diff --git a/runner/runner.go b/runner/runner.go index dee4ba638..57769d3d0 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -33,6 +33,7 @@ var ( useScheduler = false runParallel = false verbose = false + timeout = 180 // In seconds. Three minutes ought to be enough for any test consoleLock = sync.Mutex{} ) @@ -96,7 +97,16 @@ func (t test) run(hosts []string) bool { } start := time.Now() - err := cmd.Run() + var err error + + c := make(chan error, 1) + go func() { c <- cmd.Run() }() + select { + case err = <-c: + case <-time.After(time.Duration(timeout) * time.Second): + err = fmt.Errorf("timed out") + } + duration := float64(time.Now().Sub(start)) / float64(time.Second) consoleLock.Lock() @@ -145,15 +155,18 @@ func getSchedule(tests []string) ([]string, error) { shardID = os.Getenv("CIRCLE_NODE_INDEX") requestBody = &bytes.Buffer{} ) + fmt.Printf("getSchedule: %v", tests) if err := json.NewEncoder(requestBody).Encode(schedule{tests}); err != nil { return []string{}, err } url := fmt.Sprintf("http://%s/schedule/%s/%s/%s", schedulerHost, testRun, shardCount, shardID) + fmt.Printf("POSTing to %v: %v", url, requestBody) resp, err := http.Post(url, jsonContentType, requestBody) if err != nil { return []string{}, err } var sched schedule + fmt.Printf("Got response: %v", resp.Body) if err := json.NewDecoder(resp.Body).Decode(&sched); err != nil { return []string{}, err } @@ -245,15 +258,17 @@ func main() { mflag.BoolVar(&runParallel, []string{"parallel"}, false, "Run tests in parallel on hosts where possible") mflag.BoolVar(&verbose, []string{"v"}, false, "Print output from all tests (Also enabled via DEBUG=1)") mflag.StringVar(&schedulerHost, []string{"scheduler-host"}, defaultSchedulerHost, "Hostname of scheduler.") + mflag.IntVar(&timeout, []string{"timeout"}, 180, "Max time to run one test for, in seconds") mflag.Parse() if len(os.Getenv("DEBUG")) > 0 { verbose = true } - tests, err := getTests(mflag.Args()) + testArgs := mflag.Args() + tests, err := getTests(testArgs) if err != nil { - fmt.Printf("Error parsing tests: %v\n", err) + fmt.Printf("Error parsing tests: %v (%v)\n", err, testArgs) os.Exit(1) } diff --git a/sched b/sched index e94e8af8f..cf47773e5 100755 --- a/sched +++ b/sched @@ -1,36 +1,38 @@ #!/usr/bin/python import sys, string, json, urllib import requests +import optparse -BASE_URL="http://positive-cocoa-90213.appspot.com" - -def test_time(test_name, runtime): - r = requests.post(BASE_URL + "/record/%s/%f" % (urllib.quote(test_name, safe=""), runtime)) +def test_time(target, test_name, runtime): + r = requests.post(target + "/record/%s/%f" % (urllib.quote(test_name, safe=""), runtime)) print r.text assert r.status_code == 204 -def test_sched(test_run, shard_count, shard_id): +def test_sched(target, test_run, shard_count, shard_id): tests = json.dumps({'tests': string.split(sys.stdin.read())}) - r = requests.post(BASE_URL + "/schedule/%s/%d/%d" % (test_run, shard_count, shard_id), data=tests) + r = requests.post(target + "/schedule/%s/%d/%d" % (test_run, shard_count, shard_id), data=tests) assert r.status_code == 200 result = r.json() for test in sorted(result['tests']): print test def usage(): - print "%s " % sys.argv[0] + print "%s (--target=...) " % sys.argv[0] print " time " print " sched " def main(): - if len(sys.argv) < 4: + parser = optparse.OptionParser() + parser.add_option('--target', default="http://positive-cocoa-90213.appspot.com") + options, args = parser.parse_args() + if len(args) < 3: usage() sys.exit(1) - if sys.argv[1] == "time": - test_time(sys.argv[2], float(sys.argv[3])) - elif sys.argv[1] == "sched": - test_sched(sys.argv[2], int(sys.argv[3]), int(sys.argv[4])) + if args[0] == "time": + test_time(options.target, args[1], float(args[2])) + elif args[0] == "sched": + test_sched(options.target, args[1], int(args[2]), int(args[3])) else: usage() diff --git a/scheduler/main.py b/scheduler/main.py index f509f0e1f..8907e202d 100644 --- a/scheduler/main.py +++ b/scheduler/main.py @@ -23,6 +23,19 @@ class Test(ndb.Model): total_run_time = ndb.FloatProperty(default=0.) # Not total, but a EWMA total_runs = ndb.IntegerProperty(default=0) + def parallelism(self): + name = self.key.string_id() + m = re.search('(\d+)_test.sh$', name) + if m is None: + return 1 + else: + return int(m.group(1)) + + def cost(self): + p = self.parallelism() + logging.info("Test %s has parallelism %d and avg run time %s", self.key.string_id(), p, self.total_run_time) + return self.parallelism() * self.total_run_time + class Schedule(ndb.Model): shards = ndb.JsonProperty() @@ -52,7 +65,7 @@ def schedule(test_run, shard_count, shard): test_times = ndb.get_multi(ndb.Key(Test, test_name) for test_name in test_names) def avg(test): if test is not None: - return test.total_run_time + return test.cost() return 1 test_times = [(test_name, avg(test)) for test_name, test in zip(test_names, test_times)] test_times_dict = dict(test_times) diff --git a/shell-lint b/shell-lint new file mode 100755 index 000000000..5cc77cb2a --- /dev/null +++ b/shell-lint @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +# Lint all shell files in given directories with `shellcheck`. +# +# e.g. +# $ shell-lint infra k8s +# +# Depends on: +# - shellcheck +# - files-with-type +# - file >= 5.22 + +"$(dirname "${BASH_SOURCE[0]}")/files-with-type" text/x-shellscript "$@" | xargs shellcheck diff --git a/socks/connect.sh b/socks/connect.sh index 0d5ef84da..b6af8a6c4 100755 --- a/socks/connect.sh +++ b/socks/connect.sh @@ -10,14 +10,17 @@ fi HOST=$1 echo "Starting proxy container..." -PROXY_CONTAINER=$(ssh $HOST weave run -d weaveworks/socksproxy) +PROXY_CONTAINER=$(ssh "$HOST" weave run -d weaveworks/socksproxy) function finish { echo "Removing proxy container.." - ssh $HOST docker rm -f $PROXY_CONTAINER + # shellcheck disable=SC2029 + ssh "$HOST" docker rm -f "$PROXY_CONTAINER" } trap finish EXIT -PROXY_IP=$(ssh $HOST -- "docker inspect --format='{{.NetworkSettings.IPAddress}}' $PROXY_CONTAINER") +# shellcheck disable=SC2029 +PROXY_IP=$(ssh "$HOST" -- "docker inspect --format='{{.NetworkSettings.IPAddress}}' $PROXY_CONTAINER") echo 'Please configure your browser for proxy http://localhost:8080/proxy.pac' -ssh -L8000:$PROXY_IP:8000 -L8080:$PROXY_IP:8080 $HOST docker attach $PROXY_CONTAINER +# shellcheck disable=SC2029 +ssh "-L8000:$PROXY_IP:8000" "-L8080:$PROXY_IP:8080" "$HOST" docker attach "$PROXY_CONTAINER" diff --git a/test b/test index 5b88d5270..61791772b 100755 --- a/test +++ b/test @@ -1,9 +1,9 @@ #!/bin/bash -set -e +set -ex DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GO_TEST_ARGS="-tags netgo -cpu 4 -timeout 8m" +GO_TEST_ARGS=( -tags netgo -cpu 4 -timeout 8m ) SLOW= NO_GO_GET= @@ -28,66 +28,67 @@ while [ $# -gt 0 ]; do esac done -if [ -n "$SLOW" -o -n "$CIRCLECI" ]; then +if [ -n "$SLOW" ] || [ -n "$CIRCLECI" ]; then SLOW=true fi if [ -n "$SLOW" ]; then - GO_TEST_ARGS="$GO_TEST_ARGS -race -covermode=atomic" + GO_TEST_ARGS=( "${GO_TEST_ARGS[@]}" -race -covermode=atomic ) + # shellcheck disable=SC2153 if [ -n "$COVERDIR" ] ; then coverdir="$COVERDIR" else coverdir=$(mktemp -d coverage.XXXXXXXXXX) fi - mkdir -p $coverdir + mkdir -p "$coverdir" fi fail=0 -TESTDIRS=$(find . -type f -name '*_test.go' | xargs -n1 dirname | grep -vE '^\./(\.git|vendor|prog|experimental)/' | sort -u) +TESTDIRS=( $(find . -type f -name '*_test.go' -print0 | xargs -0 -n1 dirname | grep -vE '^\./(\.git|vendor|prog|experimental)/' | sort -u) ) # If running on circle, use the scheduler to work out what tests to run on what shard -if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" -a -x "$DIR/sched" ]; then +if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ] && [ -x "$DIR/sched" ]; then PREFIX=$(go list -e ./ | sed -e 's/\//-/g') - TESTDIRS=$(echo $TESTDIRS | "$DIR/sched" sched $PREFIX-$CIRCLE_BUILD_NUM $CIRCLE_NODE_TOTAL $CIRCLE_NODE_INDEX) - echo $TESTDIRS + TESTDIRS=( $(echo "${TESTDIRS[@]}" | "$DIR/sched" sched "$PREFIX-$CIRCLE_BUILD_NUM" "$CIRCLE_NODE_TOTAL" "$CIRCLE_NODE_INDEX") ) + echo "${TESTDIRS[@]}" fi PACKAGE_BASE=$(go list -e ./) -# Speed up the tests by compiling and installing their dependancies first. -go test -i $GO_TEST_ARGS $TESTDIRS +# Speed up the tests by compiling and installing their dependencies first. +go test -i "${GO_TEST_ARGS[@]}" "${TESTDIRS[@]}" -for dir in $TESTDIRS; do +for dir in "${TESTDIRS[@]}"; do if [ -z "$NO_GO_GET" ]; then - go get -t -tags netgo $dir + go get -t -tags netgo "$dir" fi - GO_TEST_ARGS_RUN="$GO_TEST_ARGS" + GO_TEST_ARGS_RUN=( "${GO_TEST_ARGS[@]}" ) if [ -n "$SLOW" ]; then - COVERPKGS=$( (go list $dir; go list -f '{{join .Deps "\n"}}' $dir | grep -v "vendor" | grep "^$PACKAGE_BASE/") | paste -s -d, -) - output=$(mktemp $coverdir/unit.XXXXXXXXXX) - GO_TEST_ARGS_RUN="$GO_TEST_ARGS -coverprofile=$output -coverpkg=$COVERPKGS" + COVERPKGS=$( (go list "$dir"; go list -f '{{join .Deps "\n"}}' "$dir" | grep -v "vendor" | grep "^$PACKAGE_BASE/") | paste -s -d, -) + output=$(mktemp "$coverdir/unit.XXXXXXXXXX") + GO_TEST_ARGS_RUN=( "${GO_TEST_ARGS[@]}" -coverprofile=$output -coverpkg=$COVERPKGS ) fi START=$(date +%s) - if ! go test $GO_TEST_ARGS_RUN $dir; then + if ! go test "${GO_TEST_ARGS_RUN[@]}" "$dir"; then fail=1 fi - RUNTIME=$(( $(date +%s) - $START )) + RUNTIME=$(( $(date +%s) - START )) # Report test runtime when running on circle, to help scheduler - if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" -a -x "$DIR/sched" ]; then - "$DIR/sched" time $dir $RUNTIME + if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ] && [ -x "$DIR/sched" ]; then + "$DIR/sched" time "$dir" $RUNTIME fi done -if [ -n "$SLOW" -a -z "$COVERDIR" ] ; then +if [ -n "$SLOW" ] && [ -z "$COVERDIR" ] ; then go get github.com/weaveworks/tools/cover - cover $coverdir/* >profile.cov - rm -rf $coverdir + cover "$coverdir"/* >profile.cov + rm -rf "$coverdir" go tool cover -html=profile.cov -o=coverage.html go tool cover -func=profile.cov | tail -n1 fi From ddf80d05cb270bc0d0884af779d5f5c7fc0a4755 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Fri, 8 Jul 2016 15:54:26 +0100 Subject: [PATCH 2/3] Actually run shell-lint on tools --- Makefile | 1 + backend/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8bf0445d1..170d64a2f 100644 --- a/Makefile +++ b/Makefile @@ -102,6 +102,7 @@ tests: $(SCOPE_BACKEND_BUILD_UPTODATE) lint: $(SCOPE_BACKEND_BUILD_UPTODATE) ./tools/lint -ignorespelling "agre " -ignorespelling "AGRE " . + ./tools/shell-lint tools prog/static.go: $(SCOPE_BACKEND_BUILD_UPTODATE) esc -o $@ -prefix client/build client/build diff --git a/backend/Dockerfile b/backend/Dockerfile index e809ba719..a6718ed66 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,6 +1,6 @@ FROM golang:1.6.2 RUN apt-get update && \ - apt-get install -y libpcap-dev python-requests time file && \ + apt-get install -y libpcap-dev python-requests time file shellcheck && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN go clean -i net && \ go install -tags netgo std && \ From ca9e0387d063905e681e44a3fbd2ddf1b1682f91 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Tue, 12 Jul 2016 18:07:42 +0100 Subject: [PATCH 3/3] Squashed 'tools/' changes from 8c6170d..0620e58 0620e58 Review tweaks git-subtree-dir: tools git-subtree-split: 0620e589fa0bf14f17790e8b9e81c2076b68c899 --- rebuild-image | 2 +- runner/runner.go | 3 --- socks/connect.sh | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/rebuild-image b/rebuild-image index 772ecb934..a579a47de 100755 --- a/rebuild-image +++ b/rebuild-image @@ -31,7 +31,7 @@ has_changes() { local rev1=$1 local rev2=$2 local changes - changes=$(git diff --oneline "$rev1..$rev2" -- "${INPUTFILES[@]}" | wc -l) + changes=$(git diff --oneline "$rev1..$rev2" -- "${INPUTFILES[@]}" | wc -l) [ "$changes" -gt 0 ] } diff --git a/runner/runner.go b/runner/runner.go index 57769d3d0..707369d81 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -155,18 +155,15 @@ func getSchedule(tests []string) ([]string, error) { shardID = os.Getenv("CIRCLE_NODE_INDEX") requestBody = &bytes.Buffer{} ) - fmt.Printf("getSchedule: %v", tests) if err := json.NewEncoder(requestBody).Encode(schedule{tests}); err != nil { return []string{}, err } url := fmt.Sprintf("http://%s/schedule/%s/%s/%s", schedulerHost, testRun, shardCount, shardID) - fmt.Printf("POSTing to %v: %v", url, requestBody) resp, err := http.Post(url, jsonContentType, requestBody) if err != nil { return []string{}, err } var sched schedule - fmt.Printf("Got response: %v", resp.Body) if err := json.NewDecoder(resp.Body).Decode(&sched); err != nil { return []string{}, err } diff --git a/socks/connect.sh b/socks/connect.sh index b6af8a6c4..25dcf8221 100755 --- a/socks/connect.sh +++ b/socks/connect.sh @@ -14,7 +14,7 @@ PROXY_CONTAINER=$(ssh "$HOST" weave run -d weaveworks/socksproxy) function finish { echo "Removing proxy container.." - # shellcheck disable=SC2029 + # shellcheck disable=SC2029 ssh "$HOST" docker rm -f "$PROXY_CONTAINER" } trap finish EXIT