diff --git a/backend/Dockerfile b/backend/Dockerfile
index 307c425a5..ed5a9c054 100644
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -1,7 +1,7 @@
FROM golang:1.10.2-stretch
ENV SCOPE_SKIP_UI_ASSETS true
RUN apt-get update && \
- apt-get install -y libpcap-dev python-requests time file shellcheck git gcc-arm-linux-gnueabihf curl build-essential python-pip && \
+ apt-get install -y libpcap-dev time file shellcheck git gcc-arm-linux-gnueabihf curl build-essential python-pip && \
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
RUN go clean -i net && \
go install -tags netgo std && \
@@ -18,7 +18,7 @@ RUN go get -tags netgo \
github.com/client9/misspell/cmd/misspell && \
chmod a+wr --recursive /usr/local/go && \
rm -rf /go/pkg/ /go/src/
-RUN pip install yapf==0.16.2 flake8==3.3.0
+RUN pip install yapf==0.16.2 flake8==3.3.0 requests==2.19.1
COPY build.sh /
ENTRYPOINT ["/build.sh"]
diff --git a/scope b/scope
index 49aa7d3fc..5611e1f5d 100755
--- a/scope
+++ b/scope
@@ -245,7 +245,7 @@ case "$COMMAND" in
# is not making any attempt to do escaping at all, we might as well try.
# shellcheck disable=SC2039
quoted=$(printf '%q ' "$@" 2>/dev/null || true)
- # printf %q behaves oddly with zero args (it acts as though it recieved one empty arg)
+ # printf %q behaves oddly with zero args (it acts as though it received one empty arg)
# so we ignore that case.
if [ -z "$quoted" ] || [ $# -eq 0 ]; then
quoted="$*"
diff --git a/tools/COPYING.LGPL-3 b/tools/COPYING.LGPL-3
new file mode 100644
index 000000000..f01171d44
--- /dev/null
+++ b/tools/COPYING.LGPL-3
@@ -0,0 +1,175 @@
+./integration/assert.sh is a copy of
+
+ https://github.com/lehmannro/assert.sh/blob/master/assert.sh
+
+Since it was added to this codebase, it has only received cosmetic
+modifications. As it is licensed under the LGPL-3, here's the license
+text in its entirety:
+
+
+
+ GNU LESSER GENERAL PUBLIC LICENSE
+ Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc.
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+ This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+ 0. Additional Definitions.
+
+ As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+ "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+ An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+ A "Combined Work" is a work produced by combining or linking an
+Application with the Library. The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+ The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+ The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+ 1. Exception to Section 3 of the GNU GPL.
+
+ You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+ 2. Conveying Modified Versions.
+
+ If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+ a) under this License, provided that you make a good faith effort to
+ ensure that, in the event an Application does not supply the
+ function or data, the facility still operates, and performs
+ whatever part of its purpose remains meaningful, or
+
+ b) under the GNU GPL, with none of the additional permissions of
+ this License applicable to that copy.
+
+ 3. Object Code Incorporating Material from Library Header Files.
+
+ The object code form of an Application may incorporate material from
+a header file that is part of the Library. You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+ a) Give prominent notice with each copy of the object code that the
+ Library is used in it and that the Library and its use are
+ covered by this License.
+
+ b) Accompany the object code with a copy of the GNU GPL and this license
+ document.
+
+ 4. Combined Works.
+
+ You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+ a) Give prominent notice with each copy of the Combined Work that
+ the Library is used in it and that the Library and its use are
+ covered by this License.
+
+ b) Accompany the Combined Work with a copy of the GNU GPL and this license
+ document.
+
+ c) For a Combined Work that displays copyright notices during
+ execution, include the copyright notice for the Library among
+ these notices, as well as a reference directing the user to the
+ copies of the GNU GPL and this license document.
+
+ d) Do one of the following:
+
+ 0) Convey the Minimal Corresponding Source under the terms of this
+ License, and the Corresponding Application Code in a form
+ suitable for, and under terms that permit, the user to
+ recombine or relink the Application with a modified version of
+ the Linked Version to produce a modified Combined Work, in the
+ manner specified by section 6 of the GNU GPL for conveying
+ Corresponding Source.
+
+ 1) Use a suitable shared library mechanism for linking with the
+ Library. A suitable mechanism is one that (a) uses at run time
+ a copy of the Library already present on the user's computer
+ system, and (b) will operate properly with a modified version
+ of the Library that is interface-compatible with the Linked
+ Version.
+
+ e) Provide Installation Information, but only if you would otherwise
+ be required to provide such information under section 6 of the
+ GNU GPL, and only to the extent that such information is
+ necessary to install and execute a modified version of the
+ Combined Work produced by recombining or relinking the
+ Application with a modified version of the Linked Version. (If
+ you use option 4d0, the Installation Information must accompany
+ the Minimal Corresponding Source and Corresponding Application
+ Code. If you use option 4d1, you must provide the Installation
+ Information in the manner specified by section 6 of the GNU GPL
+ for conveying Corresponding Source.)
+
+ 5. Combined Libraries.
+
+ You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+ a) Accompany the combined library with a copy of the same work based
+ on the Library, uncombined with any other library facilities,
+ conveyed under the terms of this License.
+
+ b) Give prominent notice with the combined library that part of it
+ is a work based on the Library, and explaining where to find the
+ accompanying uncombined form of the same work.
+
+ 6. Revised Versions of the GNU Lesser General Public License.
+
+ The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+ If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.
diff --git a/tools/LICENSE b/tools/LICENSE
new file mode 100644
index 000000000..9cd1640ba
--- /dev/null
+++ b/tools/LICENSE
@@ -0,0 +1,13 @@
+Copyright 2018 Weaveworks. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/tools/README.md b/tools/README.md
index 9092b8e24..8154aa988 100644
--- a/tools/README.md
+++ b/tools/README.md
@@ -2,6 +2,7 @@
Included in this repo are tools shared by weave.git and scope.git. They include
+- ```bazel-rules```: Bazel build rules used in our projects
- ```build```: a set of docker base-images for building weave
projects. These should be used instead of giving each project its
own build image.
@@ -32,7 +33,11 @@ Included in this repo are tools shared by weave.git and scope.git. They include
## Requirements
- ```lint``` requires shfmt to lint sh files; get shfmt with
- ```go get -u gopkg.in/mvdan/sh.v1/cmd/shfmt```
+```
+curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64
+chmod +x shfmt
+```
+ (we pin that version, and it doesn't build from the source repo any more)
## Using build-tools.git
@@ -50,3 +55,14 @@ To update the code in build-tools.git, the process is therefore:
- PR into build-tools.git, go through normal review process etc.
- Do `git subtree pull --prefix tools https://github.com/weaveworks/build-tools.git master --squash`
in your repo, and PR that.
+
+## Getting Help
+
+If you have any questions about, feedback for or problems with `build-tools`:
+
+- Invite yourself to the #weave-community slack channel.
+- Ask a question on the #weave-community slack channel.
+- Send an email to weave-users@weave.works
+- File an issue.
+
+Your feedback is always welcome!
diff --git a/tools/bazel-rules/BUILD.bazel b/tools/bazel-rules/BUILD.bazel
new file mode 100644
index 000000000..751b37073
--- /dev/null
+++ b/tools/bazel-rules/BUILD.bazel
@@ -0,0 +1,26 @@
+load("@io_bazel_rules_go//proto:compiler.bzl", "go_proto_compiler")
+
+go_proto_compiler(
+ name = "gogo_proto",
+ deps = [
+ "//vendor/github.com/gogo/protobuf/gogoproto:go_default_library",
+ "//vendor/github.com/gogo/protobuf/proto:go_default_library",
+ "//vendor/github.com/gogo/protobuf/sortkeys:go_default_library",
+ ],
+ plugin = "@com_github_gogo_protobuf//protoc-gen-gogoslick",
+ visibility = ["//visibility:public"],
+)
+
+go_proto_compiler(
+ name = "gogo_grpc",
+ deps = [
+ "//vendor/github.com/gogo/protobuf/gogoproto:go_default_library",
+ "//vendor/github.com/gogo/protobuf/proto:go_default_library",
+ "//vendor/github.com/gogo/protobuf/sortkeys:go_default_library",
+ "//vendor/google.golang.org/grpc:go_default_library",
+ "//vendor/golang.org/x/net/context:go_default_library",
+ ],
+ plugin = "@com_github_gogo_protobuf//protoc-gen-gogoslick",
+ options = ["plugins=grpc"],
+ visibility = ["//visibility:public"],
+)
diff --git a/tools/bazel-rules/gogo.bzl b/tools/bazel-rules/gogo.bzl
new file mode 100644
index 000000000..82f244616
--- /dev/null
+++ b/tools/bazel-rules/gogo.bzl
@@ -0,0 +1,36 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_repository")
+
+
+_BUILD_FILE = """
+proto_library(
+ name = "gogoproto",
+ srcs = ["gogo.proto"],
+ deps = [
+ "@com_google_protobuf//:descriptor_proto",
+ ],
+ visibility = ["//visibility:public"],
+)
+"""
+
+def _go_repository_impl(ctx):
+ ctx.file("BUILD.bazel", content="")
+ ctx.file("github.com/gogo/protobuf/gogoproto/BUILD.bazel", content=_BUILD_FILE)
+ ctx.template("github.com/gogo/protobuf/gogoproto/gogo.proto", ctx.attr._proto)
+
+_gogo_proto_repository = repository_rule(
+ implementation = _go_repository_impl,
+ attrs = {
+ "_proto": attr.label(default="//vendor/github.com/gogo/protobuf/gogoproto:gogo.proto"),
+ },
+)
+
+def gogo_dependencies():
+ go_repository(
+ name = "com_github_gogo_protobuf",
+ importpath = "github.com/gogo/protobuf",
+ urls = ["https://codeload.github.com/ianthehat/protobuf/zip/2adc21fd136931e0388e278825291678e1d98309"],
+ strip_prefix = "protobuf-2adc21fd136931e0388e278825291678e1d98309",
+ type = "zip",
+ build_file_proto_mode="disable",
+ )
+ _gogo_proto_repository(name = "internal_gogo_proto_repository")
diff --git a/tools/build/Makefile b/tools/build/Makefile
index cea049be5..a84198c38 100644
--- a/tools/build/Makefile
+++ b/tools/build/Makefile
@@ -5,13 +5,14 @@
# All this must go at top of file I'm afraid.
IMAGE_PREFIX := quay.io/weaveworks/build-
IMAGE_TAG := $(shell ../image-tag)
+GIT_REVISION := $(shell git rev-parse HEAD)
UPTODATE := .uptodate
# Every directory with a Dockerfile in it builds an image called
# $(IMAGE_PREFIX). Dependencies (i.e. things that go in the image)
# still need to be explicitly declared.
%/$(UPTODATE): %/Dockerfile %/*
- $(SUDO) docker build -t $(IMAGE_PREFIX)$(shell basename $(@D)) $(@D)/
+ $(SUDO) docker build --build-arg=revision=$(GIT_REVISION) -t $(IMAGE_PREFIX)$(shell basename $(@D)) $(@D)/
$(SUDO) docker tag $(IMAGE_PREFIX)$(shell basename $(@D)) $(IMAGE_PREFIX)$(shell basename $(@D)):$(IMAGE_TAG)
touch $@
diff --git a/tools/build/golang/Dockerfile b/tools/build/golang/Dockerfile
index 8ef1d2b04..23fb6ac83 100644
--- a/tools/build/golang/Dockerfile
+++ b/tools/build/golang/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.8.0-stretch
+FROM golang:1.10.0-stretch
RUN apt-get update && \
apt-get install -y \
curl \
@@ -11,9 +11,10 @@ RUN apt-get update && \
python-pip \
python-requests \
python-yaml \
+ shellcheck \
unzip && \
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-RUN pip install attrs pyhcl
+RUN pip install attrs pyhcl yapf==0.16.2 flake8==3.3.0
RUN curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
echo "b1925c2c405458811f0c227266402cf1868b4de529f114722c2e3a5af4ac7bb2 shfmt" | sha256sum -c && \
chmod +x shfmt && \
@@ -47,3 +48,10 @@ RUN mkdir -p /var/run/secrets/kubernetes.io/serviceaccount && \
touch /var/run/secrets/kubernetes.io/serviceaccount/token
COPY build.sh /
ENTRYPOINT ["/build.sh"]
+
+ARG revision
+LABEL maintainer="Weaveworks " \
+ org.opencontainers.image.title="golang" \
+ org.opencontainers.image.source="https://github.com/weaveworks/build-tools/tree/master/build/golang" \
+ org.opencontainers.image.revision="${revision}" \
+ org.opencontainers.image.vendor="Weaveworks"
diff --git a/tools/build/haskell/Dockerfile b/tools/build/haskell/Dockerfile
index 8d40c6624..79f34a80a 100644
--- a/tools/build/haskell/Dockerfile
+++ b/tools/build/haskell/Dockerfile
@@ -2,3 +2,10 @@ FROM fpco/stack-build:lts-8.9
COPY build.sh /
COPY copy-libraries /usr/local/bin/
ENTRYPOINT ["/build.sh"]
+
+ARG revision
+LABEL maintainer="Weaveworks " \
+ org.opencontainers.image.title="haskell" \
+ org.opencontainers.image.source="https://github.com/weaveworks/build-tools/tree/master/build/haskell" \
+ org.opencontainers.image.revision="${revision}" \
+ org.opencontainers.image.vendor="Weaveworks"
diff --git a/tools/circle.yml b/tools/circle.yml
index 976a68cc9..68976ff09 100644
--- a/tools/circle.yml
+++ b/tools/circle.yml
@@ -13,27 +13,16 @@ dependencies:
- go install -tags netgo std
- mkdir -p $(dirname $SRCDIR)
- cp -r $(pwd)/ $SRCDIR
- - |
- curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
- echo "b1925c2c405458811f0c227266402cf1868b4de529f114722c2e3a5af4ac7bb2 shfmt" | sha256sum -c && \
- chmod +x shfmt && \
- sudo mv shfmt /usr/bin
- - |
- cd $SRCDIR;
- go get \
- github.com/fzipp/gocyclo \
- github.com/golang/lint/golint \
- github.com/kisielk/errcheck \
- github.com/fatih/hclfmt
- - pip install yapf==0.16.2 flake8==3.3.0
test:
override:
- - cd $SRCDIR; ./lint .
+ - |
+ cd $SRCDIR/build && \
+ make && \
+ docker run --rm -v "$SRCDIR:$SRCDIR" -w "$SRCDIR" --entrypoint sh quay.io/weaveworks/build-golang -c ./lint .
- cd $SRCDIR/cover; make
- cd $SRCDIR/socks; make
- cd $SRCDIR/runner; make
- - cd $SRCDIR/build; make
deployment:
snapshot:
diff --git a/tools/config_management/roles/dev-tools/tasks/main.yml b/tools/config_management/roles/dev-tools/tasks/main.yml
index 96ac3a219..a9cb99ddd 100644
--- a/tools/config_management/roles/dev-tools/tasks/main.yml
+++ b/tools/config_management/roles/dev-tools/tasks/main.yml
@@ -38,11 +38,3 @@
dest: /usr/bin
mode: 0555
creates: /usr/bin/terraform
-
-# Ubuntu runs an apt update process that will run on first boot from image.
-# This is of questionable value when the machines are only going to live for a few minutes.
-# If you leave them on they will run the process daily.
-# Also we have seen the update process create a 'defunct' process which then throws off Weave Net smoke-test checks.
-# So, we override the 'persistent' setting so it will still run at the scheduled time but will not try to catch up on first boot.
-- name: copy apt daily override
- copy: src=apt-daily.timer.conf dest=/etc/systemd/system/apt-daily.timer.d/
diff --git a/tools/config_management/roles/dev-tools/files/apt-daily.timer.conf b/tools/config_management/roles/setup-apt/files/apt-daily.timer.conf
similarity index 100%
rename from tools/config_management/roles/dev-tools/files/apt-daily.timer.conf
rename to tools/config_management/roles/setup-apt/files/apt-daily.timer.conf
diff --git a/tools/config_management/roles/setup-apt/tasks/main.yml b/tools/config_management/roles/setup-apt/tasks/main.yml
new file mode 100644
index 000000000..3593cf709
--- /dev/null
+++ b/tools/config_management/roles/setup-apt/tasks/main.yml
@@ -0,0 +1,10 @@
+---
+# Set up apt
+
+# Ubuntu runs an apt update process that will run on first boot from image.
+# This is of questionable value when the machines are only going to live for a few minutes.
+# If you leave them on they will run the process daily.
+# Also we have seen the update process create a 'defunct' process which then throws off Weave Net smoke-test checks.
+# So, we override the 'persistent' setting so it will still run at the scheduled time but will not try to catch up on first boot.
+- name: copy apt daily override
+ copy: src=apt-daily.timer.conf dest=/etc/systemd/system/apt-daily.timer.d/
diff --git a/tools/config_management/setup_bare_docker.yml b/tools/config_management/setup_bare_docker.yml
new file mode 100644
index 000000000..fac8405f1
--- /dev/null
+++ b/tools/config_management/setup_bare_docker.yml
@@ -0,0 +1,16 @@
+---
+################################################################################
+# Install Docker from Docker's official repository
+################################################################################
+
+- name: install docker
+ hosts: all
+ gather_facts: false # required in case Python is not available on the host
+ become: true
+ become_user: root
+
+ pre_tasks:
+ - include: library/setup_ansible_dependencies.yml
+
+ roles:
+ - docker-install
diff --git a/tools/config_management/setup_weave-net_dev.yml b/tools/config_management/setup_weave-net_dev.yml
index bdfa08e90..1923d011e 100644
--- a/tools/config_management/setup_weave-net_dev.yml
+++ b/tools/config_management/setup_weave-net_dev.yml
@@ -13,6 +13,7 @@
- include: library/setup_ansible_dependencies.yml
roles:
+ - setup-apt
- dev-tools
- golang-from-tarball
- docker-install
diff --git a/tools/config_management/setup_weave-net_test.yml b/tools/config_management/setup_weave-net_test.yml
index fbd155df7..7125d054f 100644
--- a/tools/config_management/setup_weave-net_test.yml
+++ b/tools/config_management/setup_weave-net_test.yml
@@ -13,6 +13,7 @@
- include: library/setup_ansible_dependencies.yml
roles:
+ - setup-apt
- docker-install
- weave-net-utilities
- kubernetes-install
diff --git a/tools/dependencies/list_os_images.sh b/tools/dependencies/list_os_images.sh
index 00db0d061..139a08144 100755
--- a/tools/dependencies/list_os_images.sh
+++ b/tools/dependencies/list_os_images.sh
@@ -57,7 +57,7 @@ fi
case "$1" in
'gcp')
- gcloud compute images list --standard-images --regexp=".*?$2.*" \
+ gcloud compute images list --standard-images --filter="name~'.*?$2.*'" \
--format="csv[no-heading][separator=/](selfLink.map().scope(projects).segment(0),family)" \
| sort -d
;;
diff --git a/tools/lint b/tools/lint
index 63c506618..9db7970d1 100755
--- a/tools/lint
+++ b/tools/lint
@@ -6,7 +6,8 @@
#
# For shell files, it runs shfmt. If you don't have that installed, you can get
# it with:
-# go get -u gopkg.in/mvdan/sh.v1/cmd/shfmt
+# curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64
+# chmod +x shfmt
#
# With no arguments, it lints the current files staged
# for git commit. Or you can pass it explicit filenames
@@ -50,14 +51,6 @@ spell_check() {
local filename="$1"
local lint_result=0
- # we don't want to spell check tar balls, binaries, Makefile and json files
- if file "$filename" | grep executable >/dev/null 2>&1; then
- return $lint_result
- fi
- if [[ $filename == *".tar" || $filename == *".gz" || $filename == *".json" || $(basename "$filename") == "Makefile" ]]; then
- return $lint_result
- fi
-
# misspell is completely optional. If you don't like it
# don't have it installed.
if ! type misspell >/dev/null 2>&1; then
@@ -113,9 +106,12 @@ lint_sh() {
local filename="$1"
local lint_result=0
- if ! diff -u "${filename}" <(shfmt -i 4 "${filename}"); then
- lint_result=1
- echo "${filename}: run shfmt -i 4 -w ${filename}"
+ # Skip shfmt validation, if not installed
+ if type shfmt >/dev/null 2>&1; then
+ if ! diff -u "${filename}" <(shfmt -i 4 "${filename}"); then
+ lint_result=1
+ echo "${filename}: run shfmt -i 4 -w ${filename}"
+ fi
fi
# the shellcheck is completely optional. If you don't like it
@@ -159,7 +155,7 @@ lint_py() {
if yapf --diff "${filename}" | grep -qE '^[+-]'; then
lint_result=1
- echo "${filename}: run yapf --in-place ${filename}"
+ echo "${filename} needs reformatting. Run: yapf --in-place ${filename}"
else
# Only run flake8 if yapf passes, since they pick up a lot of similar issues
flake8 "${filename}" || lint_result=1
@@ -185,19 +181,24 @@ lint() {
*.pb.go) return ;;
esac
- if [[ "$(file --mime-type "${filename}" | awk '{print $2}')" == "text/x-shellscript" ]]; then
- ext="sh"
- fi
+ mimetype=$(file --mime-type "${filename}" | awk '{print $2}')
- case "$ext" in
- go) lint_go "${filename}" || lint_result=1 ;;
- sh) lint_sh "${filename}" || lint_result=1 ;;
- tf) lint_tf "${filename}" || lint_result=1 ;;
- md) lint_md "${filename}" || lint_result=1 ;;
- py) lint_py "${filename}" || lint_result=1 ;;
+ case "$mimetype.$ext" in
+ text/x-shellscript.*) lint_sh "${filename}" || lint_result=1 ;;
+ *.go) lint_go "${filename}" || lint_result=1 ;;
+ *.tf) lint_tf "${filename}" || lint_result=1 ;;
+ *.md) lint_md "${filename}" || lint_result=1 ;;
+ *.py) lint_py "${filename}" || lint_result=1 ;;
esac
- spell_check "${filename}" || lint_result=1
+ # we don't want to spell check tar balls, binaries, Makefile and json files
+ case "$mimetype.$ext" in
+ *.tar | *.gz | *.json) ;;
+ *.req | *.key | *.pem | *.crt) ;;
+ application/x-executable.*) ;;
+ text/x-makefile.*) ;;
+ *) spell_check "${filename}" || lint_result=1 ;;
+ esac
return $lint_result
}
@@ -240,7 +241,7 @@ filter_out() {
list_files() {
if [ $# -gt 0 ]; then
- find "$@" | grep -vE '(^|/)vendor/'
+ find "$@" \( -name vendor -o -name .git \) -prune -o -type f
else
git ls-files --exclude-standard | grep -vE '(^|/)vendor/'
fi
diff --git a/tools/provisioning/README.md b/tools/provisioning/README.md
index 627bb42e3..6ff739cad 100755
--- a/tools/provisioning/README.md
+++ b/tools/provisioning/README.md
@@ -16,16 +16,15 @@ You can then use these machines as is or run various Ansible playbooks from `../
* On macOS: `brew install vagrant`
* On Linux (via Aptitude): `sudo apt install vagrant`
- * If you need a specific version:
-
- curl -fsS https://releases.hashicorp.com/terraform/x.y.z/terraform_x.y.z_linux_amd64.zip | gunzip > terraform && chmod +x terraform && sudo mv terraform /usr/bin
-
* For other platforms or more details, see [here](https://www.vagrantup.com/docs/installation/)
* You will need [Terraform](https://www.terraform.io) installed on your machine and added to your `PATH` in order to be able to provision cloud-hosted machines automatically.
* On macOS: `brew install terraform`
* On Linux (via Aptitude): `sudo apt install terraform`
+ * If you need a specific version:
+
+ curl -fsS https://releases.hashicorp.com/terraform/x.y.z/terraform_x.y.z_linux_amd64.zip | gunzip > terraform && chmod +x terraform && sudo mv terraform /usr/bin
* For other platforms or more details, see [here](https://www.terraform.io/intro/getting-started/install.html)
* Depending on the cloud provider, you may have to create an account, manually onboard, create and register SSH keys, etc.
diff --git a/tools/provisioning/gcp/main.tf b/tools/provisioning/gcp/main.tf
index abfddb7df..af5a22ebc 100755
--- a/tools/provisioning/gcp/main.tf
+++ b/tools/provisioning/gcp/main.tf
@@ -77,3 +77,17 @@ resource "google_compute_firewall" "fw-allow-esp" {
source_ranges = ["${var.gcp_network_global_cidr}"]
}
+
+# Required for WKS Kubernetes API server access
+resource "google_compute_firewall" "fw-allow-kube-apiserver" {
+ name = "${var.name}-allow-kube-apiserver"
+ network = "${var.gcp_network}"
+ target_tags = ["${var.name}"]
+
+ allow {
+ protocol = "tcp"
+ ports = ["6443"]
+ }
+
+ source_ranges = ["${var.client_ip}"]
+}
diff --git a/tools/provisioning/gcp/outputs.tf b/tools/provisioning/gcp/outputs.tf
index 9aa1e33e8..210398ba5 100755
--- a/tools/provisioning/gcp/outputs.tf
+++ b/tools/provisioning/gcp/outputs.tf
@@ -6,6 +6,10 @@ output "public_ips" {
value = ["${google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.assigned_nat_ip}"]
}
+output "private_ips" {
+ value = ["${google_compute_instance.tf_test_vm.*.network_interface.0.address}"]
+}
+
output "hostnames" {
value = "${join("\n",
"${formatlist("%v.%v.%v",
diff --git a/tools/provisioning/setup.sh b/tools/provisioning/setup.sh
index 456878e0e..965ee28fd 100755
--- a/tools/provisioning/setup.sh
+++ b/tools/provisioning/setup.sh
@@ -18,7 +18,8 @@ function decrypt() {
echo >&2 "Failed to decode and decrypt $2: no secret key was provided."
return 1
fi
- echo "$3" | openssl base64 -d | openssl enc -d -aes256 -pass "pass:$1"
+ # Set md5 because existing keys were encrypted that way and openssl default changed
+ echo "$3" | openssl base64 -d | openssl enc -md md5 -d -aes256 -pass "pass:$1"
}
function ssh_private_key() {
diff --git a/tools/push-images b/tools/push-images
index 1871c3782..913a8c318 100755
--- a/tools/push-images
+++ b/tools/push-images
@@ -26,25 +26,28 @@ while [ $# -gt 0 ]; do
esac
done
-push_image() {
- local image="$1"
- docker push "${image}:${IMAGE_TAG}"
-}
-
+pids=""
for image in ${IMAGES}; do
if [[ "$image" == *"build"* ]]; then
continue
fi
echo "Will push ${image}:${IMAGE_TAG}"
- push_image "${image}" &
+ docker push "${image}:${IMAGE_TAG}" &
+ pids="$pids $!"
if [ -z "$NO_DOCKER_HUB" ]; then
# remove the quey prefix and push to docker hub
docker_hub_image=${image#$QUAY_PREFIX}
docker tag "${image}:${IMAGE_TAG}" "${docker_hub_image}:${IMAGE_TAG}"
echo "Will push ${docker_hub_image}:${IMAGE_TAG}"
- docker push "${docker_hub_image}:${IMAGE_TAG}"
+ docker push "${docker_hub_image}:${IMAGE_TAG}" &
+ pids="$pids $!"
fi
done
+# Wait individually for tasks so we fail-exit on any non-zero return code
+for p in $pids; do
+ wait "$p"
+done
+
wait
diff --git a/tools/rebuild-image b/tools/rebuild-image
index 1f0bb1091..cfa4ced8e 100755
--- a/tools/rebuild-image
+++ b/tools/rebuild-image
@@ -9,6 +9,7 @@ IMAGENAME=$1
SAVEDNAME=$(echo "$IMAGENAME" | sed "s/[\/\-]/\./g")
IMAGEDIR=$2
shift 2
+GIT_REVISION="$(git rev-parse HEAD)"
INPUTFILES=("$@")
CACHEDIR=$HOME/docker/
@@ -17,7 +18,7 @@ CACHEDIR=$HOME/docker/
rebuild() {
mkdir -p "$CACHEDIR"
rm "$CACHEDIR/$SAVEDNAME"* || true
- docker build -t "$IMAGENAME" "$IMAGEDIR"
+ docker build --build-arg=revision="$GIT_REVISION" -t "$IMAGENAME" "$IMAGEDIR"
docker save "$IMAGENAME:latest" | gzip - >"$CACHEDIR/$SAVEDNAME-$CIRCLE_SHA1.gz"
}
diff --git a/tools/sched b/tools/sched
index a282558f1..179c650a5 100755
--- a/tools/sched
+++ b/tools/sched
@@ -1,16 +1,31 @@
#!/usr/bin/env python
import sys, string, urllib
import requests
+from requests.packages.urllib3.util.retry import Retry
+from requests.adapters import HTTPAdapter
import optparse
+session = requests.Session()
+adapter = HTTPAdapter(
+ max_retries=Retry(
+ connect=5,
+ status=5,
+ backoff_factor=0.1,
+ status_forcelist=[500, 502, 503, 504]
+ )
+)
+session.mount('http://', adapter)
+session.mount('https://', adapter)
+
+
def test_time(target, test_name, runtime):
- r = requests.post(target + "/record/%s/%f" % (urllib.quote(test_name, safe=""), runtime))
+ r = session.post(target + "/record/%s/%f" % (urllib.quote(test_name, safe=""), runtime))
print r.text.encode('utf-8')
assert r.status_code == 204
def test_sched(target, test_run, shard_count, shard_id):
tests = {'tests': string.split(sys.stdin.read())}
- r = requests.post(target + "/schedule/%s/%d/%d" % (test_run, shard_count, shard_id), json=tests)
+ r = session.post(target + "/schedule/%s/%d/%d" % (test_run, shard_count, shard_id), json=tests)
assert r.status_code == 200
result = r.json()
for test in sorted(result['tests']):
diff --git a/tools/scheduler/main.py b/tools/scheduler/main.py
index 3b540b54a..de3a1288d 100644
--- a/tools/scheduler/main.py
+++ b/tools/scheduler/main.py
@@ -107,8 +107,12 @@ FIREWALL_REGEXES = [
r'(?P[\w\-]+)$'),
]
NAME_REGEXES = [
- re.compile(r'^host(?P\d+)-(?P\d+)-(?P\d+)$'),
- re.compile(r'^test-(?P\d+)-(?P\d+)-(?P\d+)$'),
+ re.compile(pat)
+ for pat in (
+ r'^host(?P\d+)-(?P\d+)-(?P\d+)$',
+ r'^host(?P\d+)-(?P[a-zA-Z0-9-]+)-(?P\d+)'
+ r'-(?P\d+)$',
+ r'^test-(?P\d+)-(?P\d+)-(?P\d+)$', )
]
diff --git a/tools/socks/Dockerfile b/tools/socks/Dockerfile
index 867cd6bc5..ad0b8938f 100644
--- a/tools/socks/Dockerfile
+++ b/tools/socks/Dockerfile
@@ -1,7 +1,13 @@
FROM gliderlabs/alpine
-MAINTAINER Weaveworks Inc
WORKDIR /
COPY proxy /
EXPOSE 8000
EXPOSE 8080
ENTRYPOINT ["/proxy"]
+
+ARG revision
+LABEL maintainer="Weaveworks " \
+ org.opencontainers.image.title="socks" \
+ org.opencontainers.image.source="https://github.com/weaveworks/build-tools/tree/master/socks" \
+ org.opencontainers.image.revision="${revision}" \
+ org.opencontainers.image.vendor="Weaveworks"
diff --git a/tools/socks/Makefile b/tools/socks/Makefile
index 2daeda643..b33586492 100644
--- a/tools/socks/Makefile
+++ b/tools/socks/Makefile
@@ -2,6 +2,7 @@
IMAGE_TAR=image.tar
IMAGE_NAME=weaveworks/socksproxy
+GIT_REVISION := $(shell git rev-parse HEAD)
PROXY_EXE=proxy
NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
rm $@; \
@@ -15,7 +16,7 @@ NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
all: $(IMAGE_TAR)
$(IMAGE_TAR): Dockerfile $(PROXY_EXE)
- docker build -t $(IMAGE_NAME) .
+ docker build --build-arg=revision=$(GIT_REVISION) -t $(IMAGE_NAME) .
docker save $(IMAGE_NAME):latest > $@
$(PROXY_EXE): *.go
diff --git a/tools/socks/main.go b/tools/socks/main.go
index 7cd8c7086..ff56a0c88 100644
--- a/tools/socks/main.go
+++ b/tools/socks/main.go
@@ -15,19 +15,20 @@ import (
)
type pacFileParameters struct {
- HostMatch string
- Aliases map[string]string
+ HostMatch string
+ SocksDestination string
+ Aliases map[string]string
}
const (
pacfile = `
function FindProxyForURL(url, host) {
if(shExpMatch(host, "{{.HostMatch}}")) {
- return "SOCKS5 localhost:8000";
+ return "SOCKS5 {{.SocksDestination}}";
}
{{range $key, $value := .Aliases}}
if (host == "{{$key}}") {
- return "SOCKS5 localhost:8000";
+ return "SOCKS5 {{.SocksDestination}}";
}
{{end}}
return "DIRECT";
@@ -37,11 +38,13 @@ function FindProxyForURL(url, host) {
func main() {
var (
- as []string
- hostMatch string
+ as []string
+ hostMatch string
+ socksDestination string
)
mflagext.ListVar(&as, []string{"a", "-alias"}, []string{}, "Specify hostname aliases in the form alias:hostname. Can be repeated.")
mflag.StringVar(&hostMatch, []string{"h", "-host-match"}, "*.weave.local", "Specify main host shExpMatch expression in pacfile")
+ mflag.StringVar(&socksDestination, []string{"d", "-socks-destination"}, "localhost:8000", "Specify destination host:port in pacfile")
mflag.Parse()
var aliases = map[string]string{}
@@ -60,7 +63,7 @@ func main() {
t := template.Must(template.New("pacfile").Parse(pacfile))
http.HandleFunc("/proxy.pac", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/x-ns-proxy-autoconfig")
- t.Execute(w, pacFileParameters{hostMatch, aliases})
+ t.Execute(w, pacFileParameters{hostMatch, socksDestination, aliases})
})
if err := http.ListenAndServe(":8080", nil); err != nil {
diff --git a/tools/test b/tools/test
index c87bdd073..c284e4947 100755
--- a/tools/test
+++ b/tools/test
@@ -9,6 +9,7 @@ TAGS=
PARALLEL=
RACE="-race -covermode=atomic"
TIMEOUT=1m
+VERBOSE=
usage() {
echo "$0 [-slow] [-in-container foo] [-netgo] [-(no-)go-get] [-timeout 1m]"
@@ -16,6 +17,10 @@ usage() {
while [ $# -gt 0 ]; do
case "$1" in
+ "-v")
+ VERBOSE="-v"
+ shift 1
+ ;;
"-slow")
SLOW=true
shift 1
@@ -55,7 +60,7 @@ while [ $# -gt 0 ]; do
esac
done
-GO_TEST_ARGS=(-tags "${TAGS[@]}" -cpu 4 -timeout $TIMEOUT)
+GO_TEST_ARGS=(-tags "${TAGS[@]}" -cpu 4 -timeout $TIMEOUT $VERBOSE)
if [ -n "$SLOW" ] || [ -n "$CIRCLECI" ]; then
SLOW=true