Squashed 'tools/' changes from 4b7d5c6..a3b18bf

a3b18bf Merge pull request #65 from weaveworks/fix-integration-tests
ecb5602 Fix integration tests
f9dcbf6 ... without tab (clearly not my day)
a6215c3 Add break I forgot
0e6832d Remove incorrectly added tab
eb26c68 Merge pull request #64 from weaveworks/remove-test-package-linting
f088e83 Review feedback
2c6e83e Remove test package linting
2b3a1bb Merge pull request #62 from weaveworks/revert-61-test-defaults
8c3883a Revert "Make no-go-get the default, and don't assume -tags netgo"
e75c226 Fix bug in GC of firewall rules.
e49754e Merge pull request #51 from weaveworks/gc-firewall-rules
191f487 Add flag to enale/disable firewall rules' GC.
567905c Add GC of firewall rules for weave-net-tests to scheduler.
03119e1 Fix typo in GC of firewall rules.
bbe3844 Fix regular expression for firewall rules.
c5c23ce Pre-change refactoring: splitted gc_project function into smaller methods for better readability.
ed5529f GC firewall rules
ed8e757 Merge pull request #61 from weaveworks/test-defaults
57856e6 Merge pull request #56 from weaveworks/remove-wcloud
dd5f3e6 Add -p flag to test, run test in parallel
62f6f94 Make no-go-get the default, and don't assume -tags netgo
8946588 Merge pull request #60 from weaveworks/2647-gc-weave-net-tests
4085df9 Scheduler now also garbage-collects VMs from weave-net-tests.
d1a5e46 Remove wcloud cli tool

git-subtree-dir: tools
git-subtree-split: a3b18bfe932ddb8adaceb0adb2df35c579bf4ee4

.gitignore

@@ -2,6 +2,5 @@ cover/cover
 socks/proxy
 socks/image.tar
 runner/runner
-cmd/wcloud/wcloud
 *.pyc
 *~

circle.yml

@@ -27,5 +27,4 @@ test:
     - cd $SRCDIR/cover; make
     - cd $SRCDIR/socks; make
     - cd $SRCDIR/runner; make
-    - cd $SRCDIR/cmd/wcloud; make
 

cmd/wcloud/Makefile

@@ -1,11 +0,0 @@
-.PHONY: all clean
-
-all: wcloud
-
-wcloud: *.go
-	go get ./$(@D)
-	go build -o $@ ./$(@D)
-
-clean:
-	rm -rf wcloud
-	go clean ./...

cmd/wcloud/cli.go

@@ -1,238 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"encoding/json"
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"os/user"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"github.com/olekukonko/tablewriter"
-	"gopkg.in/yaml.v2"
-)
-
-// ArrayFlags allows you to collect repeated flags
-type ArrayFlags []string
-
-func (a *ArrayFlags) String() string {
-	return strings.Join(*a, ",")
-}
-
-// Set implements flags.Value
-func (a *ArrayFlags) Set(value string) error {
-	*a = append(*a, value)
-	return nil
-}
-
-func env(key, def string) string {
-	if val, ok := os.LookupEnv(key); ok {
-		return val
-	}
-	return def
-}
-
-var (
-	token   = env("SERVICE_TOKEN", "")
-	baseURL = env("BASE_URL", "https://cloud.weave.works")
-)
-
-func usage() {
-	fmt.Println(`Usage:
-	deploy <image>:<version>   Deploy image to your configured env
-	list                       List recent deployments
-	config (<filename>)        Get (or set) the configured env
-	logs <deploy>              Show lots for the given deployment`)
-}
-
-func main() {
-	if len(os.Args) <= 1 {
-		usage()
-		os.Exit(1)
-	}
-
-	c := NewClient(token, baseURL)
-
-	switch os.Args[1] {
-	case "deploy":
-		deploy(c, os.Args[2:])
-	case "list":
-		list(c, os.Args[2:])
-	case "config":
-		config(c, os.Args[2:])
-	case "logs":
-		logs(c, os.Args[2:])
-	case "events":
-		events(c, os.Args[2:])
-	case "help":
-		usage()
-	default:
-		usage()
-	}
-}
-
-func deploy(c Client, args []string) {
-	var (
-		flags    = flag.NewFlagSet("", flag.ContinueOnError)
-		username = flags.String("u", "", "Username to report to deploy service (default with be current user)")
-		services ArrayFlags
-	)
-	flags.Var(&services, "service", "Service to update (can be repeated)")
-	if err := flags.Parse(args); err != nil {
-		usage()
-		return
-	}
-	args = flags.Args()
-	if len(args) != 1 {
-		usage()
-		return
-	}
-	parts := strings.SplitN(args[0], ":", 2)
-	if len(parts) < 2 {
-		usage()
-		return
-	}
-	if *username == "" {
-		user, err := user.Current()
-		if err != nil {
-			fmt.Println(err.Error())
-			os.Exit(1)
-		}
-		*username = user.Username
-	}
-	deployment := Deployment{
-		ImageName:        parts[0],
-		Version:          parts[1],
-		TriggeringUser:   *username,
-		IntendedServices: services,
-	}
-	if err := c.Deploy(deployment); err != nil {
-		fmt.Println(err.Error())
-		os.Exit(1)
-	}
-}
-
-func list(c Client, args []string) {
-	var (
-		flags = flag.NewFlagSet("", flag.ContinueOnError)
-		since = flags.Duration("since", 7*24*time.Hour, "How far back to fetch results")
-	)
-	if err := flags.Parse(args); err != nil {
-		usage()
-		return
-	}
-	through := time.Now()
-	from := through.Add(-*since)
-	deployments, err := c.GetDeployments(from.Unix(), through.Unix())
-	if err != nil {
-		fmt.Println(err.Error())
-		os.Exit(1)
-	}
-
-	table := tablewriter.NewWriter(os.Stdout)
-	table.SetHeader([]string{"Created", "ID", "Image", "Version", "State"})
-	table.SetBorder(false)
-	table.SetColumnSeparator(" ")
-	for _, deployment := range deployments {
-		table.Append([]string{
-			deployment.CreatedAt.Format(time.RFC822),
-			deployment.ID,
-			deployment.ImageName,
-			deployment.Version,
-			deployment.State,
-		})
-	}
-	table.Render()
-}
-
-func events(c Client, args []string) {
-	var (
-		flags = flag.NewFlagSet("", flag.ContinueOnError)
-		since = flags.Duration("since", 7*24*time.Hour, "How far back to fetch results")
-	)
-	if err := flags.Parse(args); err != nil {
-		usage()
-		return
-	}
-	through := time.Now()
-	from := through.Add(-*since)
-	events, err := c.GetEvents(from.Unix(), through.Unix())
-	if err != nil {
-		fmt.Println(err.Error())
-		os.Exit(1)
-	}
-
-	fmt.Println("events: ", string(events))
-}
-
-func loadConfig(filename string) (*Config, error) {
-	extension := filepath.Ext(filename)
-	var config Config
-	buf, err := ioutil.ReadFile(filename)
-	if err != nil {
-		return nil, err
-	}
-	if extension == ".yaml" || extension == ".yml" {
-		if err := yaml.Unmarshal(buf, &config); err != nil {
-			return nil, err
-		}
-	} else {
-		if err := json.NewDecoder(bytes.NewReader(buf)).Decode(&config); err != nil {
-			return nil, err
-		}
-	}
-	return &config, nil
-}
-
-func config(c Client, args []string) {
-	if len(args) > 1 {
-		usage()
-		return
-	}
-
-	if len(args) == 1 {
-		config, err := loadConfig(args[0])
-		if err != nil {
-			fmt.Println("Error reading config:", err)
-			os.Exit(1)
-		}
-
-		if err := c.SetConfig(config); err != nil {
-			fmt.Println(err.Error())
-			os.Exit(1)
-		}
-	} else {
-		config, err := c.GetConfig()
-		if err != nil {
-			fmt.Println(err.Error())
-			os.Exit(1)
-		}
-
-		buf, err := yaml.Marshal(config)
-		if err != nil {
-			fmt.Println(err.Error())
-			os.Exit(1)
-		}
-
-		fmt.Println(string(buf))
-	}
-}
-
-func logs(c Client, args []string) {
-	if len(args) != 1 {
-		usage()
-		return
-	}
-
-	output, err := c.GetLogs(args[0])
-	if err != nil {
-		fmt.Println(err.Error())
-		os.Exit(1)
-	}
-
-	fmt.Println(string(output))
-}

cmd/wcloud/client.go

@@ -1,150 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-)
-
-// Client for the deployment service
-type Client struct {
-	token   string
-	baseURL string
-}
-
-// NewClient makes a new Client
-func NewClient(token, baseURL string) Client {
-	return Client{
-		token:   token,
-		baseURL: baseURL,
-	}
-}
-
-func (c Client) newRequest(method, path string, body io.Reader) (*http.Request, error) {
-	req, err := http.NewRequest(method, c.baseURL+path, body)
-	if err != nil {
-		return nil, err
-	}
-	req.Header.Add("Authorization", fmt.Sprintf("Scope-Probe token=%s", c.token))
-	return req, nil
-}
-
-// Deploy notifies the deployment service about a new deployment
-func (c Client) Deploy(deployment Deployment) error {
-	var buf bytes.Buffer
-	if err := json.NewEncoder(&buf).Encode(deployment); err != nil {
-		return err
-	}
-	req, err := c.newRequest("POST", "/api/deploy/deploy", &buf)
-	if err != nil {
-		return err
-	}
-	res, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return err
-	}
-	if res.StatusCode != 204 {
-		return fmt.Errorf("error making request: %s", res.Status)
-	}
-	return nil
-}
-
-// GetDeployments returns a list of deployments
-func (c Client) GetDeployments(from, through int64) ([]Deployment, error) {
-	req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/deploy?from=%d&through=%d", from, through), nil)
-	if err != nil {
-		return nil, err
-	}
-	res, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	if res.StatusCode != 200 {
-		return nil, fmt.Errorf("error making request: %s", res.Status)
-	}
-	var response struct {
-		Deployments []Deployment `json:"deployments"`
-	}
-	if err := json.NewDecoder(res.Body).Decode(&response); err != nil {
-		return nil, err
-	}
-	return response.Deployments, nil
-}
-
-// GetEvents returns the raw events.
-func (c Client) GetEvents(from, through int64) ([]byte, error) {
-	req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/event?from=%d&through=%d", from, through), nil)
-	if err != nil {
-		return nil, err
-	}
-	res, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	if res.StatusCode != 200 {
-		return nil, fmt.Errorf("error making request: %s", res.Status)
-	}
-	return ioutil.ReadAll(res.Body)
-}
-
-// GetConfig returns the current Config
-func (c Client) GetConfig() (*Config, error) {
-	req, err := c.newRequest("GET", "/api/config/deploy", nil)
-	if err != nil {
-		return nil, err
-	}
-	res, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	if res.StatusCode == 404 {
-		return nil, fmt.Errorf("no configuration uploaded yet")
-	}
-	if res.StatusCode != 200 {
-		return nil, fmt.Errorf("error making request: %s", res.Status)
-	}
-	var config Config
-	if err := json.NewDecoder(res.Body).Decode(&config); err != nil {
-		return nil, err
-	}
-	return &config, nil
-}
-
-// SetConfig sets the current Config
-func (c Client) SetConfig(config *Config) error {
-	var buf bytes.Buffer
-	if err := json.NewEncoder(&buf).Encode(config); err != nil {
-		return err
-	}
-	req, err := c.newRequest("POST", "/api/config/deploy", &buf)
-	if err != nil {
-		return err
-	}
-	res, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return err
-	}
-	if res.StatusCode != 204 {
-		return fmt.Errorf("error making request: %s", res.Status)
-	}
-	return nil
-}
-
-// GetLogs returns the logs for a given deployment.
-func (c Client) GetLogs(deployID string) ([]byte, error) {
-	req, err := c.newRequest("GET", fmt.Sprintf("/api/deploy/deploy/%s/log", deployID), nil)
-	if err != nil {
-		return nil, err
-	}
-	res, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	if res.StatusCode != 200 {
-		return nil, fmt.Errorf("error making request: %s", res.Status)
-	}
-	return ioutil.ReadAll(res.Body)
-}

cmd/wcloud/types.go

@@ -1,43 +0,0 @@
-package main
-
-import (
-	"time"
-)
-
-// Deployment describes a deployment
-type Deployment struct {
-	ID        string    `json:"id"`
-	CreatedAt time.Time `json:"created_at"`
-	ImageName string    `json:"image_name"`
-	Version   string    `json:"version"`
-	Priority  int       `json:"priority"`
-	State     string    `json:"status"`
-
-	TriggeringUser   string   `json:"triggering_user"`
-	IntendedServices []string `json:"intended_services"`
-}
-
-// Config for the deployment system for a user.
-type Config struct {
-	RepoURL        string `json:"repo_url" yaml:"repo_url"`
-	RepoBranch     string `json:"repo_branch" yaml:"repo_branch"`
-	RepoPath       string `json:"repo_path" yaml:"repo_path"`
-	RepoKey        string `json:"repo_key" yaml:"repo_key"`
-	KubeconfigPath string `json:"kubeconfig_path" yaml:"kubeconfig_path"`
-	AutoApply      bool   `json:"auto_apply" yaml:"auto_apply"`
-
-	Notifications []NotificationConfig `json:"notifications" yaml:"notifications"`
-
-	// Globs of files not to change, relative to the route of the repo
-	ConfigFileBlackList []string `json:"config_file_black_list" yaml:"config_file_black_list"`
-
-	CommitMessageTemplate string `json:"commit_message_template" yaml:"commit_message_template"` // See https://golang.org/pkg/text/template/
-}
-
-// NotificationConfig describes how to send notifications
-type NotificationConfig struct {
-	SlackWebhookURL      string `json:"slack_webhook_url" yaml:"slack_webhook_url"`
-	SlackUsername        string `json:"slack_username" yaml:"slack_username"`
-	MessageTemplate      string `json:"message_template" yaml:"message_template"`
-	ApplyMessageTemplate string `json:"apply_message_template" yaml:"apply_message_template"`
-}

integration/config.sh

@@ -81,7 +81,8 @@ run_on() {
     host=$1
     shift 1
     [ -z "$DEBUG" ] || greyly echo "Running on $host:" "$@" >&2
-    remote "$host" "$SSH" "$host" "$@"
+    # shellcheck disable=SC2086
+    remote "$host" $SSH "$host" "$@"
 }
 
 docker_on() {
@@ -117,7 +118,8 @@ start_suite() {
         PLUGIN_ID=$(docker_on "$host" ps -aq --filter=name=weaveplugin)
         PLUGIN_FILTER="cat"
         [ -n "$PLUGIN_ID" ] && PLUGIN_FILTER="grep -v $PLUGIN_ID"
-        rm_containers "$host" "$(docker_on "$host" ps -aq 2>/dev/null | "$PLUGIN_FILTER")"
+        # shellcheck disable=SC2046
+        rm_containers "$host" $(docker_on "$host" ps -aq 2>/dev/null | $PLUGIN_FILTER)
         run_on "$host" "docker network ls | grep -q ' weave ' && docker network rm weave" || true
         weave_on "$host" reset 2>/dev/null
     done

lint

@@ -18,7 +18,6 @@
 set -e
 
 IGNORE_LINT_COMMENT=
-IGNORE_TEST_PACKAGES=
 IGNORE_SPELLINGS=
 while true; do
     case "$1" in
@@ -27,7 +26,7 @@ while true; do
             shift 1
             ;;
         -notestpackage)
-            IGNORE_TEST_PACKAGES=1
+            # NOOP, still accepted for backwards compatibility
             shift 1
             ;;
         -ignorespelling)
@@ -65,30 +64,6 @@ spell_check() {
     return $lint_result
 }
 
-test_mismatch() {
-    local filename="$1"
-    local package=$(grep '^package ' "$filename" | awk '{print $2}')
-    local lint_result=0
-
-    if [[ $package == "main" ]]; then
-        return # in package main, all bets are off
-    fi
-
-    if [[ $filename == *"_internal_test.go" ]]; then
-        if [[ $package == *"_test" ]]; then
-            lint_result=1
-            echo "${filename}: should not be part of a _test package"
-        fi
-    else
-        if [[ ! $package == *"_test" ]]; then
-            lint_result=1
-            echo "${filename}: should be part of a _test package"
-        fi
-    fi
-
-    return $lint_result
-}
-
 lint_go() {
     local filename="$1"
     local lint_result=0
@@ -183,12 +158,6 @@ lint() {
         tf) lint_tf "${filename}" || lint_result=1 ;;
     esac
 
-    if [ -z "$IGNORE_TEST_PACKAGES" ]; then
-        if [[ "$filename" == *"_test.go" ]]; then
-            test_mismatch "${filename}" || lint_result=1
-        fi
-    fi
-
     spell_check "${filename}" || lint_result=1
 
     return $lint_result

scheduler/main.py

@@ -85,11 +85,25 @@ def schedule(test_run, shard_count, shard):
   schedule = Schedule.get_or_insert(schedule_id, shards=shards)
   return flask.json.jsonify(tests=schedule.shards[str(shard)])
 
-NAME_RE = re.compile(r'^host(?P<index>\d+)-(?P<build>\d+)-(?P<shard>\d+)$')
+FIREWALL_REGEXES = [
+  re.compile(r'^(?P<network>\w+)-allow-(?P<type>\w+)-(?P<build>\d+)-(?P<shard>\d+)$'),
+  re.compile(r'^(?P<network>\w+)-(?P<build>\d+)-(?P<shard>\d+)-allow-(?P<type>[\w\-]+)$'),
+]
+NAME_REGEXES = [
+  re.compile(r'^host(?P<index>\d+)-(?P<build>\d+)-(?P<shard>\d+)$'),
+  re.compile(r'^test-(?P<build>\d+)-(?P<shard>\d+)-(?P<index>\d+)$'),
+]
+
+def _matches_any_regex(name, regexes):
+  for regex in regexes:
+    matches = regex.match(name)
+    if matches:
+      return matches
 
 PROJECTS = [
-  ('weaveworks/weave', 'positive-cocoa-90213', 'us-central1-a'),
-  ('weaveworks/scope', 'scope-integration-tests', 'us-central1-a'),
+  ('weaveworks/weave', 'weave-net-tests',         'us-central1-a', True),
+  ('weaveworks/weave', 'positive-cocoa-90213',    'us-central1-a', True),
+  ('weaveworks/scope', 'scope-integration-tests', 'us-central1-a', False),
 ]
 
 @app.route('/tasks/gc')
@@ -98,34 +112,45 @@ def gc():
   credentials = GoogleCredentials.get_application_default()
   compute = discovery.build('compute', 'v1', credentials=credentials)
 
-  for repo, project, zone in PROJECTS:
-    gc_project(compute, repo, project, zone)
+  for repo, project, zone, gc_fw in PROJECTS:
+    gc_project(compute, repo, project, zone, gc_fw)
 
   return "Done"
 
-def gc_project(compute, repo, project, zone):
+def gc_project(compute, repo, project, zone, gc_fw):
   logging.info("GCing %s, %s, %s", repo, project, zone)
-  instances = compute.instances().list(project=project, zone=zone).execute()
-  if 'items' not in instances:
-    return
+  # Get list of builds, filter down to running builds:
+  running = _get_running_builds(repo)
+  # Stop VMs for builds that aren't running:
+  _gc_compute_engine_instances(compute, project, zone, running)
+  # Remove firewall rules for builds that aren't running:
+  if gc_fw:
+    _gc_firewall_rules(compute, project, running)
 
-  host_by_build = collections.defaultdict(list)
-  for instance in instances['items']:
-    matches = NAME_RE.match(instance['name'])
-    if matches is None:
-      continue
-    host_by_build[int(matches.group('build'))].append(instance['name'])
-  logging.info("Running VMs by build: %r", host_by_build)
-
-  # Get list of builds, filter down to runnning builds
+def _get_running_builds(repo):
   result = urlfetch.fetch('https://circleci.com/api/v1/project/%s' % repo,
     headers={'Accept': 'application/json'})
   assert result.status_code == 200
   builds = json.loads(result.content)
   running = {build['build_num'] for build in builds if not build.get('stop_time')}
   logging.info("Runnings builds: %r", running)
+  return running
 
-  # Stop VMs for builds that aren't running
+def _get_hosts_by_build(instances):
+  host_by_build = collections.defaultdict(list)
+  for instance in instances['items']:
+    matches = _matches_any_regex(instance['name'], NAME_REGEXES)
+    if not matches:
+      continue
+    host_by_build[int(matches.group('build'))].append(instance['name'])
+  logging.info("Running VMs by build: %r", host_by_build)
+  return host_by_build
+
+def _gc_compute_engine_instances(compute, project, zone, running):
+  instances = compute.instances().list(project=project, zone=zone).execute()
+  if 'items' not in instances:
+    return
+  host_by_build = _get_hosts_by_build(instances)
   stopped = []
   for build, names in host_by_build.iteritems():
     if build in running:
@@ -134,5 +159,17 @@ def gc_project(compute, repo, project, zone):
       stopped.append(name)
       logging.info("Stopping VM %s", name)
       compute.instances().delete(project=project, zone=zone, instance=name).execute()
+  return stopped
 
-  return
+def _gc_firewall_rules(compute, project, running):
+  firewalls = compute.firewalls().list(project=project).execute()
+  if 'items' not in firewalls:
+    return
+  for firewall in firewalls['items']:
+    matches = _matches_any_regex(firewall['name'], FIREWALL_REGEXES)
+    if not matches:
+      continue
+    if int(matches.group('build')) in running:
+      continue
+    logging.info("Deleting firewall rule %s", firewall['name'])
+    compute.firewalls().delete(project=project, firewall=firewall['name']).execute()