github/weave-scope
1
0
Fork 0
mirror of https://github.com/weaveworks/scope.git synced 2026-03-04 10:41:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1658 from weaveworks/tools-with-shellcheck

Browse Source 
Lint shellscripts from tools
This commit is contained in:
Jonathan Lange
2016-07-13 15:10:49 +01:00
committed by GitHub
parent 78bf236a9c 7c94ade7f4
commit 53735eda27
22 changed files with 542 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile
View File

@@ -102,6 +102,7 @@ tests: $(SCOPE_BACKEND_BUILD_UPTODATE)
lint: $(SCOPE_BACKEND_BUILD_UPTODATE)
./tools/lint -ignorespelling "agre " -ignorespelling "AGRE " .
./tools/shell-lint tools
prog/static.go: $(SCOPE_BACKEND_BUILD_UPTODATE)
esc -o $@ -prefix client/build client/build

2
backend/Dockerfile
View File

@@ -1,6 +1,6 @@
FROM golang:1.6.2
RUN apt-get update && \
apt-get install -y libpcap-dev python-requests time file && \
apt-get install -y libpcap-dev python-requests time file shellcheck && \
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
RUN go clean -i net && \
go install -tags netgo std && \

2
tools/.gitignore vendored
View File

@@ -2,3 +2,5 @@ cover/cover
socks/proxy
socks/image.tar
runner/runner
*.pyc
*~

4
tools/README.md
View File

@@ -4,11 +4,15 @@ Included in this repo are tools shared by weave.git and scope.git. They include
- ```cover```: a tool which merges overlapping coverage reports generated by go
test
- ```files-with-type```: a tool to search directories for files of a given
MIME type
- ```lint```: a script to lint Go project; runs various tools like golint, go
vet, errcheck etc
- ```rebuild-image```: a script to rebuild docker images when their input files
change; useful when you using docker images to build your software, but you
don't want to build the image every time.
- ```shell-lint```: a script to lint multiple shell files with
[shellcheck](http://www.shellcheck.net/)
- ```socks```: a simple, dockerised SOCKS proxy for getting your laptop onto
the Weave network
- ```test```: a script to run all go unit tests in subdirectories, gather the

178
tools/cmd/wcloud/cli.go Normal file
View File

@@ -0,0 +1,178 @@
package main
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"time"
"github.com/olekukonko/tablewriter"
"gopkg.in/yaml.v2"
)
func env(key, def string) string {
if val, ok := os.LookupEnv(key); ok {
return val
}
return def
}
var (
token = env("SERVICE_TOKEN", "")
baseURL = env("BASE_URL", "https://cloud.weave.works")
)
func usage() {
fmt.Println(`Usage:
deploy <image>:<version> Deploy image to your configured env
list List recent deployments
config (<filename>) Get (or set) the configured env
logs <deploy> Show lots for the given deployment`)
}
func main() {
if len(os.Args) <= 1 {
usage()
os.Exit(1)
}
c := NewClient(token, baseURL)
switch os.Args[1] {
case "deploy":
deploy(c, os.Args[2:])
case "list":
list(c, os.Args[2:])
case "config":
config(c, os.Args[2:])
case "logs":
logs(c, os.Args[2:])
case "help":
usage()
default:
usage()
}
}
func deploy(c Client, args []string) {
if len(args) != 1 {
usage()
return
}
parts := strings.SplitN(args[0], ":", 2)
if len(parts) < 2 {
usage()
return
}
deployment := Deployment{
ImageName: parts[0],
Version: parts[1],
}
if err := c.Deploy(deployment); err != nil {
fmt.Println(err.Error())
os.Exit(1)
}
}
func list(c Client, args []string) {
flags := flag.NewFlagSet("list", flag.ContinueOnError)
page := flags.Int("page", 0, "Zero based index of page to list.")
pagesize := flags.Int("page-size", 10, "Number of results per page")
if err := flags.Parse(args); err != nil {
usage()
return
}
deployments, err := c.GetDeployments(*page, *pagesize)
if err != nil {
fmt.Println(err.Error())
os.Exit(1)
}
table := tablewriter.NewWriter(os.Stdout)
table.SetHeader([]string{"Created", "ID", "Image", "Version", "State"})
table.SetBorder(false)
table.SetColumnSeparator(" ")
for _, deployment := range deployments {
table.Append([]string{
deployment.CreatedAt.Format(time.RFC822),
deployment.ID,
deployment.ImageName,
deployment.Version,
deployment.State,
})
}
table.Render()
}
func loadConfig(filename string) (*Config, error) {
extension := filepath.Ext(filename)
var config Config
buf, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
if extension == ".yaml" || extension == ".yml" {
if err := yaml.Unmarshal(buf, &config); err != nil {
return nil, err
}
} else {
if err := json.NewDecoder(bytes.NewReader(buf)).Decode(&config); err != nil {
return nil, err
}
}
return &config, nil
}
func config(c Client, args []string) {
if len(args) > 1 {
usage()
return
}
if len(args) == 1 {
config, err := loadConfig(args[0])
if err != nil {
fmt.Println("Error reading config:", err)
os.Exit(1)
}
if err := c.SetConfig(config); err != nil {
fmt.Println(err.Error())
os.Exit(1)
}
} else {
config, err := c.GetConfig()
if err != nil {
fmt.Println(err.Error())
os.Exit(1)
}
buf, err := yaml.Marshal(config)
if err != nil {
fmt.Println(err.Error())
os.Exit(1)
}
fmt.Println(string(buf))
}
}
func logs(c Client, args []string) {
if len(args) != 1 {
usage()
return
}
output, err := c.GetLogs(args[0])
if err != nil {
fmt.Println(err.Error())
os.Exit(1)
}
fmt.Println(string(output))
}

131
tools/cmd/wcloud/client.go Normal file
View File

@@ -0,0 +1,131 @@
package main
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
)
// Client for the deployment service
type Client struct {
token string
baseURL string
}
// NewClient makes a new Client
func NewClient(token, baseURL string) Client {
return Client{
token: token,
baseURL: baseURL,
}
}
func (c Client) newRequest(method, path string, body io.Reader) (*http.Request, error) {
req, err := http.NewRequest(method, c.baseURL+path, body)
if err != nil {
return nil, err
}
req.Header.Add("Authorization", fmt.Sprintf("Scope-Probe token=%s", c.token))
return req, nil
}
// Deploy notifies the deployment service about a new deployment
func (c Client) Deploy(deployment Deployment) error {
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(deployment); err != nil {
return err
}
req, err := c.newRequest("POST", "/api/deploy", &buf)
if err != nil {
return err
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
if res.StatusCode != 204 {
return fmt.Errorf("Error making request: %s", res.Status)
}
return nil
}
// GetDeployments returns a list of deployments
func (c Client) GetDeployments(page, pagesize int) ([]Deployment, error) {
req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy?page=%d&pagesize=%d", page, pagesize), nil)
if err != nil {
return nil, err
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
if res.StatusCode != 200 {
return nil, fmt.Errorf("Error making request: %s", res.Status)
}
var response struct {
Deployments []Deployment `json:"deployments"`
}
if err := json.NewDecoder(res.Body).Decode(&response); err != nil {
return nil, err
}
return response.Deployments, nil
}
// GetConfig returns the current Config
func (c Client) GetConfig() (*Config, error) {
req, err := c.newRequest("GET", "/api/config/deploy", nil)
if err != nil {
return nil, err
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
if res.StatusCode != 200 {
return nil, fmt.Errorf("Error making request: %s", res.Status)
}
var config Config
if err := json.NewDecoder(res.Body).Decode(&config); err != nil {
return nil, err
}
return &config, nil
}
// SetConfig sets the current Config
func (c Client) SetConfig(config *Config) error {
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(config); err != nil {
return err
}
req, err := c.newRequest("POST", "/api/config/deploy", &buf)
if err != nil {
return err
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
if res.StatusCode != 204 {
return fmt.Errorf("Error making request: %s", res.Status)
}
return nil
}
// GetLogs returns the logs for a given deployment.
func (c Client) GetLogs(deployID string) ([]byte, error) {
req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/%s/log", deployID), nil)
if err != nil {
return nil, err
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
if res.StatusCode != 200 {
return nil, fmt.Errorf("Error making request: %s", res.Status)
}
return ioutil.ReadAll(res.Body)
}

24
tools/cmd/wcloud/types.go Normal file
View File

@@ -0,0 +1,24 @@
package main
import (
"time"
)
// Deployment describes a deployment
type Deployment struct {
ID string `json:"id"`
CreatedAt time.Time `json:"created_at"`
ImageName string `json:"image_name"`
Version string `json:"version"`
Priority int `json:"priority"`
State string `json:"status"`
LogKey string `json:"-"`
}
// Config for the deployment system for a user.
type Config struct {
RepoURL string `json:"repo_url" yaml:"repo_url"`
RepoPath string `json:"repo_path" yaml:"repo_path"`
RepoKey string `json:"repo_key" yaml:"repo_key"`
KubeconfigPath string `json:"kubeconfig_path" yaml:"kubeconfig_path"`
}

11
tools/cover/gather_coverage.sh
View File

@@ -3,19 +3,18 @@
# merges them and produces a complete report.
set -ex
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DESTINATION=$1
FROMDIR=$2
mkdir -p $DESTINATION
mkdir -p "$DESTINATION"
if [ -n "$CIRCLECI" ]; then
for i in $(seq 1 $(($CIRCLE_NODE_TOTAL - 1))); do
scp node$i:$FROMDIR/* $DESTINATION || true
for i in $(seq 1 $((CIRCLE_NODE_TOTAL - 1))); do
scp "node$i:$FROMDIR"/* "$DESTINATION" || true
done
fi
go get github.com/weaveworks/build-tools/cover
cover $DESTINATION/* >profile.cov
cover "$DESTINATION"/* >profile.cov
go tool cover -html=profile.cov -o coverage.html
go tool cover -func=profile.cov -o coverage.txt
tar czf coverage.tar.gz $DESTINATION
tar czf coverage.tar.gz "$DESTINATION"

13
tools/files-with-type Executable file
View File

@@ -0,0 +1,13 @@
#!/usr/bin/env bash
#
# Find all files with a given MIME type.
#
# e.g.
# $ files-with-type text/x-shellscript k8s infra
#
# Assumes `find`, `xargs`, and `file` are all installed.
mime_type=$1
shift
find "$@" -print0 -type f |xargs -0 file --mime-type | grep "${mime_type}" | sed -e 's/:.*$//'

20
tools/integration/assert.sh
View File

@@ -24,14 +24,14 @@ export INVARIANT=${INVARIANT:-}
export CONTINUE=${CONTINUE:-}
args="$(getopt -n "$0" -l \
verbose,help,stop,discover,invariant,continue vhxdic $*)" \
verbose,help,stop,discover,invariant,continue vhxdic "$@")" \
|| exit -1
for arg in $args; do
case "$arg" in
-h)
echo "$0 [-vxidc]" \
"[--verbose] [--stop] [--invariant] [--discover] [--continue]"
echo "`sed 's/./ /g' <<< "$0"` [-h] [--help]"
echo "$(sed 's/./ /g' <<< "$0") [-h] [--help]"
exit 0;;
--help)
cat <<EOF
@@ -83,17 +83,17 @@ assert_end() {
# to get report_time split tests_time on 2 substrings:
# ${tests_time:0:${#tests_time}-9} - seconds
# ${tests_time:${#tests_time}-9:3} - milliseconds
[[ -z "$INVARIANT" ]] \
&& report_time=" in ${tests_time:0:${#tests_time}-9}.${tests_time:${#tests_time}-9:3}s" \
|| report_time=
if [[ -z "$INVARIANT" ]]; then
report_time=" in ${tests_time:0:${#tests_time}-9}.${tests_time:${#tests_time}-9:3}s"
else
report_time=
fi
if [[ "$tests_failed" -eq 0 ]]; then
echo "all $tests passed$report_time."
else
for error in "${tests_errors[@]}"; do echo "$error"; done
echo "$tests_failed of $tests failed$report_time."
fi
tests_failed_previous=$tests_failed
[[ $tests_failed -gt 0 ]] && tests_suite_status=1
_assert_reset
}
@@ -103,7 +103,7 @@ assert() {
(( tests_ran++ )) || :
[[ -z "$DISCOVERONLY" ]] || return
expected=$(echo -ne "${2:-}")
result="$(eval 2>/dev/null $1 <<< ${3:-})" || true
result="$(eval 2>/dev/null "$1" <<< "${3:-}")" || true
if [[ "$result" == "$expected" ]]; then
[[ -z "$DEBUG" ]] || echo -n .
return
@@ -119,7 +119,7 @@ assert_raises() {
(( tests_ran++ )) || :
[[ -z "$DISCOVERONLY" ]] || return
status=0
(eval $1 <<< ${3:-}) > /dev/null 2>&1 || status=$?
(eval "$1" <<< "${3:-}") > /dev/null 2>&1 || status=$?
expected=${2:-0}
if [[ "$status" -eq "$expected" ]]; then
[[ -z "$DEBUG" ]] || echo -n .
@@ -143,7 +143,7 @@ _assert_fail() {
skip_if() {
# skip_if <command ..>
(eval $@) > /dev/null 2>&1 && status=0 || status=$?
(eval "$@") > /dev/null 2>&1 && status=0 || status=$?
[[ "$status" -eq 0 ]] || return
skip
}

88
tools/integration/gce.sh
View File

@@ -7,15 +7,15 @@
set -e
: ${KEY_FILE:=/tmp/gce_private_key.json}
: ${SSH_KEY_FILE:=$HOME/.ssh/gce_ssh_key}
: ${IMAGE:=ubuntu-14-04}
: ${ZONE:=us-central1-a}
: ${PROJECT:=}
: ${TEMPLATE_NAME:=}
: ${NUM_HOSTS:=}
: "${KEY_FILE:=/tmp/gce_private_key.json}"
: "${SSH_KEY_FILE:=$HOME/.ssh/gce_ssh_key}"
: "${IMAGE:=ubuntu-14-04}"
: "${ZONE:=us-central1-a}"
: "${PROJECT:=}"
: "${TEMPLATE_NAME:=}"
: "${NUM_HOSTS:=}"
if [ -z "${PROJECT}" -o -z "${NUM_HOSTS}" -o -z "${TEMPLATE_NAME}" ]; then
if [ -z "${PROJECT}" ] || [ -z "${NUM_HOSTS}" ] || [ -z "${TEMPLATE_NAME}" ]; then
echo "Must specify PROJECT, NUM_HOSTS and TEMPLATE_NAME"
exit 1
fi
@@ -26,21 +26,21 @@ if [ -n "$CIRCLECI" ]; then
fi
# Setup authentication
gcloud auth activate-service-account --key-file $KEY_FILE 1>/dev/null
gcloud config set project $PROJECT
gcloud auth activate-service-account --key-file "$KEY_FILE" 1>/dev/null
gcloud config set project "$PROJECT"
function vm_names {
local names=
for i in $(seq 1 $NUM_HOSTS); do
names="host$i$SUFFIX $names"
for i in $(seq 1 "$NUM_HOSTS"); do
names=( "host$i$SUFFIX" "${names[@]}" )
done
echo "$names"
echo "${names[@]}"
}
# Delete all vms in this account
function destroy {
names="$(vm_names)"
if [ $(gcloud compute instances list --zone $ZONE -q $names | wc -l) -le 1 ] ; then
if [ "$(gcloud compute instances list --zone "$ZONE" -q "$names" | wc -l)" -le 1 ] ; then
return 0
fi
for i in {0..10}; do
@@ -60,23 +60,23 @@ function destroy {
}
function internal_ip {
jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].networkIP" $1
jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].networkIP" "$1"
}
function external_ip {
jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].accessConfigs[0].natIP" $1
jq -r ".[] | select(.name == \"$2\") | .networkInterfaces[0].accessConfigs[0].natIP" "$1"
}
function try_connect {
for i in {0..10}; do
ssh -t $1 true && return
ssh -t "$1" true && return
sleep 2
done
}
function install_docker_on {
name=$1
ssh -t $name sudo bash -x -s <<EOF
ssh -t "$name" sudo bash -x -s <<EOF
curl -sSL https://get.docker.com/gpg | sudo apt-key add -
curl -sSL https://get.docker.com/ | sh
apt-get update -qq;
@@ -87,72 +87,72 @@ service docker restart
EOF
# It seems we need a short delay for docker to start up, so I put this in
# a separate ssh connection. This installs nsenter.
ssh -t $name sudo docker run --rm -v /usr/local/bin:/target jpetazzo/nsenter
ssh -t "$name" sudo docker run --rm -v /usr/local/bin:/target jpetazzo/nsenter
}
function copy_hosts {
hostname=$1
hosts=$2
cat $hosts | ssh -t "$hostname" "sudo -- sh -c \"cat >>/etc/hosts\""
ssh -t "$hostname" "sudo -- sh -c \"cat >>/etc/hosts\"" < "$hosts"
}
# Create new set of VMs
function setup {
destroy
names="$(vm_names)"
gcloud compute instances create $names --image $TEMPLATE_NAME --zone $ZONE
gcloud compute config-ssh --ssh-key-file $SSH_KEY_FILE
names=( $(vm_names) )
gcloud compute instances create "${names[@]}" --image "$TEMPLATE_NAME" --zone "$ZONE"
gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE"
sed -i '/UserKnownHostsFile=\/dev\/null/d' ~/.ssh/config
# build an /etc/hosts file for these vms
hosts=$(mktemp hosts.XXXXXXXXXX)
json=$(mktemp json.XXXXXXXXXX)
gcloud compute instances list --format=json >$json
for name in $names; do
echo "$(internal_ip $json $name) $name.$ZONE.$PROJECT" >>$hosts
gcloud compute instances list --format=json > "$json"
for name in "${names[@]}"; do
echo "$(internal_ip "$json" "$name") $name.$ZONE.$PROJECT" >> "$hosts"
done
for name in $names; do
for name in "${names[@]}"; do
hostname="$name.$ZONE.$PROJECT"
# Add the remote ip to the local /etc/hosts
sudo sed -i "/$hostname/d" /etc/hosts
sudo sh -c "echo \"$(external_ip $json $name) $hostname\" >>/etc/hosts"
try_connect $hostname
sudo sh -c "echo \"$(external_ip "$json" "$name") $hostname\" >>/etc/hosts"
try_connect "$hostname"
copy_hosts $hostname $hosts &
copy_hosts "$hostname" "$hosts" &
done
wait
rm $hosts $json
rm "$hosts" "$json"
}
function make_template {
gcloud compute instances create $TEMPLATE_NAME --image $IMAGE --zone $ZONE
gcloud compute config-ssh --ssh-key-file $SSH_KEY_FILE
gcloud compute instances create "$TEMPLATE_NAME" --image "$IMAGE" --zone "$ZONE"
gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE"
name="$TEMPLATE_NAME.$ZONE.$PROJECT"
try_connect $name
install_docker_on $name
gcloud -q compute instances delete $TEMPLATE_NAME --keep-disks boot --zone $ZONE
gcloud compute images create $TEMPLATE_NAME --source-disk $TEMPLATE_NAME --source-disk-zone $ZONE
try_connect "$name"
install_docker_on "$name"
gcloud -q compute instances delete "$TEMPLATE_NAME" --keep-disks boot --zone "$ZONE"
gcloud compute images create "$TEMPLATE_NAME" --source-disk "$TEMPLATE_NAME" --source-disk-zone "$ZONE"
}
function hosts {
hosts=
args=
json=$(mktemp json.XXXXXXXXXX)
gcloud compute instances list --format=json >$json
gcloud compute instances list --format=json > "$json"
for name in $(vm_names); do
hostname="$name.$ZONE.$PROJECT"
hosts="$hostname $hosts"
args="--add-host=$hostname:$(internal_ip $json $name) $args"
hosts=( $hostname "${hosts[@]}" )
args=( "--add-host=$hostname:$(internal_ip "$json" "$name")" "${args[@]}" )
done
echo export SSH=\"ssh -l vagrant\"
echo export HOSTS=\"$hosts\"
echo export ADD_HOST_ARGS=\"$args\"
rm $json
echo "export HOSTS=\"${hosts[*]}\""
echo "export ADD_HOST_ARGS=\"${args[*]}\""
rm "$json"
}
case "$1" in
@@ -170,7 +170,7 @@ destroy)
make_template)
# see if template exists
if ! gcloud compute images list | grep $PROJECT | grep $TEMPLATE_NAME; then
if ! gcloud compute images list | grep "$PROJECT" | grep "$TEMPLATE_NAME"; then
make_template
fi
esac

20
tools/integration/run_all.sh
View File

@@ -1,6 +1,9 @@
#!/bin/bash
set -ex
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1090
. "$DIR/config.sh"
whitely echo Sanity checks
@@ -10,18 +13,19 @@ if ! bash "$DIR/sanity_check.sh"; then
fi
whitely echo ...ok
TESTS="${@:-$(find . -name '*_test.sh')}"
RUNNER_ARGS=""
# shellcheck disable=SC2068
TESTS=( ${@:-$(find . -name '*_test.sh')} )
RUNNER_ARGS=( )
# If running on circle, use the scheduler to work out what tests to run
if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" ]; then
RUNNER_ARGS="$RUNNER_ARGS -scheduler"
if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ]; then
RUNNER_ARGS=( "${RUNNER_ARGS[@]}" -scheduler )
fi
# If running on circle or PARALLEL is not empty, run tests in parallel
if [ -n "$CIRCLECI" -o -n "$PARALLEL" ]; then
RUNNER_ARGS="$RUNNER_ARGS -parallel"
if [ -n "$CIRCLECI" ] || [ -n "$PARALLEL" ]; then
RUNNER_ARGS=( "${RUNNER_ARGS[@]}" -parallel )
fi
make -C ${DIR}/../runner
HOSTS="$HOSTS" "${DIR}/../runner/runner" $RUNNER_ARGS $TESTS
make -C "${DIR}/../runner"
HOSTS="$HOSTS" "${DIR}/../runner/runner" "${RUNNER_ARGS[@]}" "${TESTS[@]}"

14
tools/integration/sanity_check.sh
View File

@@ -1,5 +1,5 @@
#! /bin/bash
# shellcheck disable=SC1091
. ./config.sh
set -e
@@ -7,7 +7,7 @@ set -e
whitely echo Ping each host from the other
for host in $HOSTS; do
for other in $HOSTS; do
[ $host = $other ] || run_on $host $PING $other
[ "$host" = "$other" ] || run_on "$host" "$PING" "$other"
done
done
@@ -15,12 +15,12 @@ whitely echo Check we can reach docker
for host in $HOSTS; do
echo
echo Host Version Info: $host
echo =====================================
echo "Host Version Info: $host"
echo "====================================="
echo "# docker version"
docker_on $host version
docker_on "$host" version
echo "# docker info"
docker_on $host info
docker_on "$host" info
echo "# weave version"
weave_on $host version
weave_on "$host" version
done

14
tools/lint
View File

@@ -1,7 +1,7 @@
#!/bin/bash
# This scipt lints go files for common errors.
#
# Its runs gofmt and go vet, and optionally golint and
# It runs gofmt and go vet, and optionally golint and
# gocyclo, if they are installed.
#
# With no arguments, it lints the current files staged
@@ -41,10 +41,10 @@ function spell_check {
local lint_result=0
# we don't want to spell check tar balls or binaries
if file $filename | grep executable >/dev/null 2>&1; then
if file "$filename" | grep executable >/dev/null 2>&1; then
return $lint_result
fi
if [[ $filename == *".tar" ]]; then
if [[ $filename == *".tar" || $filename == *".gz" ]]; then
return $lint_result
fi
@@ -63,11 +63,11 @@ function spell_check {
function test_mismatch {
filename="$1"
package=$(grep '^package ' $filename | awk '{print $2}')
package=$(grep '^package ' "$filename" | awk '{print $2}')
local lint_result=0
if [[ $package == "main" ]]; then
continue # in package main, all bets are off
return # in package main, all bets are off
fi
if [[ $filename == *"_internal_test.go" ]]; then
@@ -115,7 +115,7 @@ function lint_go {
# don't have it installed. Also never blocks a commit,
# it just warns.
if type gocyclo >/dev/null 2>&1; then
gocyclo -over 25 "${filename}" | while read line; do
gocyclo -over 25 "${filename}" | while read -r line; do
echo "${filename}": higher than 25 cyclomatic complexity - "${line}"
done
fi
@@ -158,7 +158,7 @@ function lint {
function lint_files {
local lint_result=0
while read filename; do
while read -r filename; do
lint "${filename}" || lint_result=1
done
exit $lint_result

2
tools/publish-site
View File

@@ -3,7 +3,7 @@
set -e
set -o pipefail
: ${PRODUCT:=}
: "${PRODUCT:=}"
fatal() {
echo "$@" >&2

26
tools/rebuild-image
View File

@@ -5,37 +5,39 @@
set -eux
IMAGENAME=$1
SAVEDNAME=$(echo $IMAGENAME | sed "s/[\/\-]/\./g")
# shellcheck disable=SC2001
SAVEDNAME=$(echo "$IMAGENAME" | sed "s/[\/\-]/\./g")
IMAGEDIR=$2
shift 2
INPUTFILES=$@
INPUTFILES=( "$@" )
CACHEDIR=$HOME/docker/
# Rebuild the image
rebuild() {
mkdir -p $CACHEDIR
rm $CACHEDIR/$SAVEDNAME* || true
docker build -t $IMAGENAME $IMAGEDIR
docker save $IMAGENAME:latest | gzip - > $CACHEDIR/$SAVEDNAME-$CIRCLE_SHA1.gz
mkdir -p "$CACHEDIR"
rm "$CACHEDIR/$SAVEDNAME"* || true
docker build -t "$IMAGENAME" "$IMAGEDIR"
docker save "$IMAGENAME:latest" | gzip - > "$CACHEDIR/$SAVEDNAME-$CIRCLE_SHA1.gz"
}
# Get the revision the cached image was build at
cached_image_rev() {
find $CACHEDIR -name "$SAVEDNAME-*" -type f | sed -n 's/^[^\-]*\-\([a-z0-9]*\).gz$/\1/p'
find "$CACHEDIR" -name "$SAVEDNAME-*" -type f | sed -n 's/^[^\-]*\-\([a-z0-9]*\).gz$/\1/p'
}
# Have there been any revision between $1 and $2
has_changes() {
local rev1=$1
local rev2=$2
local changes=$(git diff --oneline $rev1..$rev2 -- $INPUTFILES | wc -l)
local changes
changes=$(git diff --oneline "$rev1..$rev2" -- "${INPUTFILES[@]}" | wc -l)
[ "$changes" -gt 0 ]
}
commit_timestamp() {
local rev=$1
git show -s --format=%ct $rev
git show -s --format=%ct "$rev"
}
cached_revision=$(cached_image_rev)
@@ -46,14 +48,14 @@ if [ -z "$cached_revision" ]; then
fi
echo ">>> Found cached image rev $cached_revision"
if has_changes $cached_revision $CIRCLE_SHA1 ; then
if has_changes "$cached_revision" "$CIRCLE_SHA1" ; then
echo ">>> Found changes, rebuilding"
rebuild
exit 0
fi
IMAGE_TIMEOUT="$(( 3 * 24 * 60 * 60 ))"
if [ "$(commit_timestamp $cached_revision)" -lt "${IMAGE_TIMEOUT}" ]; then
if [ "$(commit_timestamp "$cached_revision")" -lt "${IMAGE_TIMEOUT}" ]; then
echo ">>> Image is more the 24hrs old; rebuilding"
rebuild
exit 0
@@ -61,4 +63,4 @@ fi
# we didn't rebuild; import cached version
echo ">>> No changes found, importing cached image"
zcat $CACHEDIR/$SAVEDNAME-$cached_revision.gz | docker load
zcat "$CACHEDIR/$SAVEDNAME-$cached_revision.gz" | docker load

18
tools/runner/runner.go
View File

@@ -33,6 +33,7 @@ var (
useScheduler = false
runParallel = false
verbose = false
timeout = 180 // In seconds. Three minutes ought to be enough for any test
consoleLock = sync.Mutex{}
)
@@ -96,7 +97,16 @@ func (t test) run(hosts []string) bool {
}
start := time.Now()
err := cmd.Run()
var err error
c := make(chan error, 1)
go func() { c <- cmd.Run() }()
select {
case err = <-c:
case <-time.After(time.Duration(timeout) * time.Second):
err = fmt.Errorf("timed out")
}
duration := float64(time.Now().Sub(start)) / float64(time.Second)
consoleLock.Lock()
@@ -245,15 +255,17 @@ func main() {
mflag.BoolVar(&runParallel, []string{"parallel"}, false, "Run tests in parallel on hosts where possible")
mflag.BoolVar(&verbose, []string{"v"}, false, "Print output from all tests (Also enabled via DEBUG=1)")
mflag.StringVar(&schedulerHost, []string{"scheduler-host"}, defaultSchedulerHost, "Hostname of scheduler.")
mflag.IntVar(&timeout, []string{"timeout"}, 180, "Max time to run one test for, in seconds")
mflag.Parse()
if len(os.Getenv("DEBUG")) > 0 {
verbose = true
}
tests, err := getTests(mflag.Args())
testArgs := mflag.Args()
tests, err := getTests(testArgs)
if err != nil {
fmt.Printf("Error parsing tests: %v\n", err)
fmt.Printf("Error parsing tests: %v (%v)\n", err, testArgs)
os.Exit(1)
}

26
tools/sched
View File

@@ -1,36 +1,38 @@
#!/usr/bin/python
import sys, string, json, urllib
import requests
import optparse
BASE_URL="http://positive-cocoa-90213.appspot.com"
def test_time(test_name, runtime):
r = requests.post(BASE_URL + "/record/%s/%f" % (urllib.quote(test_name, safe=""), runtime))
def test_time(target, test_name, runtime):
r = requests.post(target + "/record/%s/%f" % (urllib.quote(test_name, safe=""), runtime))
print r.text
assert r.status_code == 204
def test_sched(test_run, shard_count, shard_id):
def test_sched(target, test_run, shard_count, shard_id):
tests = json.dumps({'tests': string.split(sys.stdin.read())})
r = requests.post(BASE_URL + "/schedule/%s/%d/%d" % (test_run, shard_count, shard_id), data=tests)
r = requests.post(target + "/schedule/%s/%d/%d" % (test_run, shard_count, shard_id), data=tests)
assert r.status_code == 200
result = r.json()
for test in sorted(result['tests']):
print test
def usage():
print "%s <cmd> <args..>" % sys.argv[0]
print "%s (--target=...) <cmd> <args..>" % sys.argv[0]
print " time <test name> <run time>"
print " sched <test run> <num shards> <shard id>"
def main():
if len(sys.argv) < 4:
parser = optparse.OptionParser()
parser.add_option('--target', default="http://positive-cocoa-90213.appspot.com")
options, args = parser.parse_args()
if len(args) < 3:
usage()
sys.exit(1)
if sys.argv[1] == "time":
test_time(sys.argv[2], float(sys.argv[3]))
elif sys.argv[1] == "sched":
test_sched(sys.argv[2], int(sys.argv[3]), int(sys.argv[4]))
if args[0] == "time":
test_time(options.target, args[1], float(args[2]))
elif args[0] == "sched":
test_sched(options.target, args[1], int(args[2]), int(args[3]))
else:
usage()

15
tools/scheduler/main.py
View File

@@ -23,6 +23,19 @@ class Test(ndb.Model):
total_run_time = ndb.FloatProperty(default=0.) # Not total, but a EWMA
total_runs = ndb.IntegerProperty(default=0)
def parallelism(self):
name = self.key.string_id()
m = re.search('(\d+)_test.sh$', name)
if m is None:
return 1
else:
return int(m.group(1))
def cost(self):
p = self.parallelism()
logging.info("Test %s has parallelism %d and avg run time %s", self.key.string_id(), p, self.total_run_time)
return self.parallelism() * self.total_run_time
class Schedule(ndb.Model):
shards = ndb.JsonProperty()
@@ -52,7 +65,7 @@ def schedule(test_run, shard_count, shard):
test_times = ndb.get_multi(ndb.Key(Test, test_name) for test_name in test_names)
def avg(test):
if test is not None:
return test.total_run_time
return test.cost()
return 1
test_times = [(test_name, avg(test)) for test_name, test in zip(test_names, test_times)]
test_times_dict = dict(test_times)

13
tools/shell-lint Executable file
View File

@@ -0,0 +1,13 @@
#!/usr/bin/env bash
#
# Lint all shell files in given directories with `shellcheck`.
#
# e.g.
# $ shell-lint infra k8s
#
# Depends on:
# - shellcheck
# - files-with-type
# - file >= 5.22
"$(dirname "${BASH_SOURCE[0]}")/files-with-type" text/x-shellscript "$@" | xargs shellcheck

11
tools/socks/connect.sh
View File

@@ -10,14 +10,17 @@ fi
HOST=$1
echo "Starting proxy container..."
PROXY_CONTAINER=$(ssh $HOST weave run -d weaveworks/socksproxy)
PROXY_CONTAINER=$(ssh "$HOST" weave run -d weaveworks/socksproxy)
function finish {
echo "Removing proxy container.."
ssh $HOST docker rm -f $PROXY_CONTAINER
# shellcheck disable=SC2029
ssh "$HOST" docker rm -f "$PROXY_CONTAINER"
}
trap finish EXIT
PROXY_IP=$(ssh $HOST -- "docker inspect --format='{{.NetworkSettings.IPAddress}}' $PROXY_CONTAINER")
# shellcheck disable=SC2029
PROXY_IP=$(ssh "$HOST" -- "docker inspect --format='{{.NetworkSettings.IPAddress}}' $PROXY_CONTAINER")
echo 'Please configure your browser for proxy http://localhost:8080/proxy.pac'
ssh -L8000:$PROXY_IP:8000 -L8080:$PROXY_IP:8080 $HOST docker attach $PROXY_CONTAINER
# shellcheck disable=SC2029
ssh "-L8000:$PROXY_IP:8000" "-L8080:$PROXY_IP:8080" "$HOST" docker attach "$PROXY_CONTAINER"

49
tools/test
View File

@@ -1,9 +1,9 @@
#!/bin/bash
set -e
set -ex
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
GO_TEST_ARGS="-tags netgo -cpu 4 -timeout 8m"
GO_TEST_ARGS=( -tags netgo -cpu 4 -timeout 8m )
SLOW=
NO_GO_GET=
@@ -28,66 +28,67 @@ while [ $# -gt 0 ]; do
esac
done
if [ -n "$SLOW" -o -n "$CIRCLECI" ]; then
if [ -n "$SLOW" ] || [ -n "$CIRCLECI" ]; then
SLOW=true
fi
if [ -n "$SLOW" ]; then
GO_TEST_ARGS="$GO_TEST_ARGS -race -covermode=atomic"
GO_TEST_ARGS=( "${GO_TEST_ARGS[@]}" -race -covermode=atomic )
# shellcheck disable=SC2153
if [ -n "$COVERDIR" ] ; then
coverdir="$COVERDIR"
else
coverdir=$(mktemp -d coverage.XXXXXXXXXX)
fi
mkdir -p $coverdir
mkdir -p "$coverdir"
fi
fail=0
TESTDIRS=$(find . -type f -name '*_test.go' | xargs -n1 dirname | grep -vE '^\./(\.git|vendor|prog|experimental)/' | sort -u)
TESTDIRS=( $(find . -type f -name '*_test.go' -print0 | xargs -0 -n1 dirname | grep -vE '^\./(\.git|vendor|prog|experimental)/' | sort -u) )
# If running on circle, use the scheduler to work out what tests to run on what shard
if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" -a -x "$DIR/sched" ]; then
if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ] && [ -x "$DIR/sched" ]; then
PREFIX=$(go list -e ./ | sed -e 's/\//-/g')
TESTDIRS=$(echo $TESTDIRS | "$DIR/sched" sched $PREFIX-$CIRCLE_BUILD_NUM $CIRCLE_NODE_TOTAL $CIRCLE_NODE_INDEX)
echo $TESTDIRS
TESTDIRS=( $(echo "${TESTDIRS[@]}" | "$DIR/sched" sched "$PREFIX-$CIRCLE_BUILD_NUM" "$CIRCLE_NODE_TOTAL" "$CIRCLE_NODE_INDEX") )
echo "${TESTDIRS[@]}"
fi
PACKAGE_BASE=$(go list -e ./)
# Speed up the tests by compiling and installing their dependancies first.
go test -i $GO_TEST_ARGS $TESTDIRS
# Speed up the tests by compiling and installing their dependencies first.
go test -i "${GO_TEST_ARGS[@]}" "${TESTDIRS[@]}"
for dir in $TESTDIRS; do
for dir in "${TESTDIRS[@]}"; do
if [ -z "$NO_GO_GET" ]; then
go get -t -tags netgo $dir
go get -t -tags netgo "$dir"
fi
GO_TEST_ARGS_RUN="$GO_TEST_ARGS"
GO_TEST_ARGS_RUN=( "${GO_TEST_ARGS[@]}" )
if [ -n "$SLOW" ]; then
COVERPKGS=$( (go list $dir; go list -f '{{join .Deps "\n"}}' $dir | grep -v "vendor" | grep "^$PACKAGE_BASE/") | paste -s -d, -)
output=$(mktemp $coverdir/unit.XXXXXXXXXX)
GO_TEST_ARGS_RUN="$GO_TEST_ARGS -coverprofile=$output -coverpkg=$COVERPKGS"
COVERPKGS=$( (go list "$dir"; go list -f '{{join .Deps "\n"}}' "$dir" | grep -v "vendor" | grep "^$PACKAGE_BASE/") | paste -s -d, -)
output=$(mktemp "$coverdir/unit.XXXXXXXXXX")
GO_TEST_ARGS_RUN=( "${GO_TEST_ARGS[@]}" -coverprofile=$output -coverpkg=$COVERPKGS )
fi
START=$(date +%s)
if ! go test $GO_TEST_ARGS_RUN $dir; then
if ! go test "${GO_TEST_ARGS_RUN[@]}" "$dir"; then
fail=1
fi
RUNTIME=$(( $(date +%s) - $START ))
RUNTIME=$(( $(date +%s) - START ))
# Report test runtime when running on circle, to help scheduler
if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" -a -x "$DIR/sched" ]; then
"$DIR/sched" time $dir $RUNTIME
if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ] && [ -x "$DIR/sched" ]; then
"$DIR/sched" time "$dir" $RUNTIME
fi
done
if [ -n "$SLOW" -a -z "$COVERDIR" ] ; then
if [ -n "$SLOW" ] && [ -z "$COVERDIR" ] ; then
go get github.com/weaveworks/tools/cover
cover $coverdir/* >profile.cov
rm -rf $coverdir
cover "$coverdir"/* >profile.cov
rm -rf "$coverdir"
go tool cover -html=profile.cov -o=coverage.html
go tool cover -func=profile.cov | tail -n1
fi