diff --git a/backend/Dockerfile b/backend/Dockerfile index 777cbc99d..128366fcb 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -10,8 +10,10 @@ RUN go get -tags netgo \ github.com/golang/lint/golint \ github.com/kisielk/errcheck \ gopkg.in/mvdan/sh.v1/cmd/shfmt \ + github.com/fatih/hclfmt \ github.com/mjibson/esc \ github.com/client9/misspell/cmd/misspell && \ + chmod a+wr --recursive /usr/local/go/pkg && \ rm -rf /go/pkg/ /go/src/ COPY build.sh / ENTRYPOINT ["/build.sh"] diff --git a/bin/setup-circleci-secrets b/bin/setup-circleci-secrets index e6b83318a..f5de31d2c 100755 --- a/bin/setup-circleci-secrets +++ b/bin/setup-circleci-secrets @@ -5,9 +5,9 @@ set -eu # openssl enc -in do-setup-circleci-secrets.orig -out setup-circleci-secrets.orig -e -aes256 -pass stdin # openssl base64 < setup-circleci-secrets.orig -openssl base64 -d < -i "localhost", -c local setup_weave-kube.yml +``` + +### Vagrant + +Provision your local VM using Vagrant: + +``` +cd $(mktemp -d -t XXX) +vagrant init ubuntu/xenial64 # or, e.g. centos/7 +vagrant up +``` + +then set the following environment variables by extracting the output of `vagrant ssh-config`: + +``` +eval $(vagrant ssh-config | sed \ +-ne 's/\ *HostName /vagrant_ssh_host=/p' \ +-ne 's/\ *User /vagrant_ssh_user=/p' \ +-ne 's/\ *Port /vagrant_ssh_port=/p' \ +-ne 's/\ *IdentityFile /vagrant_ssh_id_file=/p') +``` + +and finally run: + +``` +ansible-playbook --private-key=$vagrant_ssh_id_file -u $vagrant_ssh_user \ +--ssh-extra-args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ +-i "$vagrant_ssh_host:$vagrant_ssh_port," setup_weave-kube.yml +``` + +or, for specific versions of Kubernetes and Docker: + +``` +ansible-playbook --private-key=$vagrant_ssh_id_file -u $vagrant_ssh_user \ +--ssh-extra-args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ +-i "$vagrant_ssh_host:$vagrant_ssh_port," setup_weave-kube.yml \ +--extra-vars "docker_version=1.12.3 kubernetes_version=1.4.4" +``` + +NOTE: Kubernetes APT repo includes only the latest version, so currently +retrieving an older version will fail. + +### Terraform + +Provision your machine using the Terraform scripts from `../provisioning`, then run: + +``` +terraform output ansible_inventory > /tmp/ansible_inventory +``` + +and + +``` +ansible-playbook \ + --private-key="$(terraform output private_key_path)" \ + -u "$(terraform output username)" \ + -i /tmp/ansible_inventory \ + --ssh-extra-args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \ + ../../config_management/setup_weave-kube.yml + +``` + +To specify versions of Kubernetes and Docker see Vagrant examples above. + +N.B.: `--ssh-extra-args` is used to provide: + +* `StrictHostKeyChecking=no`: as VMs come and go, the same IP can be used by a different machine, so checking the host's SSH key may fail. Note that this introduces a risk of a man-in-the-middle attack. +* `UserKnownHostsFile=/dev/null`: if you previously connected a VM with the same IP but a different public key, and added it to `~/.ssh/known_hosts`, SSH may still fail to connect, hence we use `/dev/null` instead of `~/.ssh/known_hosts`. + +## Resources + +* [https://www.vagrantup.com/docs/provisioning/ansible.html](https://www.vagrantup.com/docs/provisioning/ansible.html) +* [http://docs.ansible.com/ansible/guide_vagrant.html](http://docs.ansible.com/ansible/guide_vagrant.html) diff --git a/tools/config_management/group_vars/all b/tools/config_management/group_vars/all new file mode 100644 index 000000000..969719980 --- /dev/null +++ b/tools/config_management/group_vars/all @@ -0,0 +1,10 @@ +--- +go_version: 1.7.4 +terraform_version: 0.8.5 +docker_version: 1.11.2 +kubernetes_version: 1.5.2 +kubernetes_cni_version: 0.3.0.1 +kubernetes_token: 123456.0123456789123456 +etcd_container_version: 2.2.5 +kube_discovery_container_version: 1.0 +pause_container_version: 3.0 diff --git a/tools/config_management/library/setup_ansible_dependencies.yml b/tools/config_management/library/setup_ansible_dependencies.yml new file mode 100644 index 000000000..50263369a --- /dev/null +++ b/tools/config_management/library/setup_ansible_dependencies.yml @@ -0,0 +1,33 @@ +--- +################################################################################ +# Install Ansible's dependencies: python and lsb_release, required respectively +# to run Ansible modules and gather Ansible facts. +# +# See also: +# - http://docs.ansible.com/ansible/intro_installation.html#managed-node-requirements +# - http://docs.ansible.com/ansible/setup_module.html +################################################################################ + +- name: check if python is installed (as required by ansible modules) + raw: test -e /usr/bin/python + register: is_python_installed + failed_when: is_python_installed.rc not in [0, 1] + changed_when: false # never mutates state. + +- name: install python if missing (as required by ansible modules) + when: is_python_installed|failed # skip otherwise + raw: (test -e /usr/bin/apt-get && apt-get update && apt-get install -y python-minimal) || (test -e /usr/bin/yum && yum update && yum install -y python) + changed_when: is_python_installed.rc == 1 + +- name: check if lsb_release is installed (as required for ansible facts) + raw: test -e /usr/bin/lsb_release + register: is_lsb_release_installed + failed_when: is_lsb_release_installed.rc not in [0, 1] + changed_when: false # never mutates state. + +- name: install lsb_release if missing (as required for ansible facts) + when: is_lsb_release_installed|failed # skip otherwise + raw: (test -e /usr/bin/apt-get && apt-get install -y lsb_release) || (test -e /usr/bin/yum && yum install -y redhat-lsb-core) + changed_when: is_lsb_release_installed.rc == 1 + +- setup: # gather 'facts', i.e. compensates for 'gather_facts: false' in calling playbook. diff --git a/tools/config_management/roles/dev-tools/tasks/main.yml b/tools/config_management/roles/dev-tools/tasks/main.yml new file mode 100644 index 000000000..422344df1 --- /dev/null +++ b/tools/config_management/roles/dev-tools/tasks/main.yml @@ -0,0 +1,40 @@ +--- +# Set up Development Environment. + +- name: install development tools + package: + name: "{{ item }}" + state: present + with_items: + # weave net dependencies + - make + - vagrant + # ansible dependencies + - python-pip + - python-dev + - libffi-dev + - libssl-dev + # terraform dependencies + - unzip + # other potentially useful tools: + - aufs-tools + - ethtool + - iputils-arping + - libpcap-dev + - git + - mercurial + - bc + - jq + +- name: install ansible + pip: + name: ansible + state: present + +- name: install terraform + unarchive: + src: 'https://releases.hashicorp.com/terraform/{{ terraform_version }}/terraform_{{ terraform_version }}_linux_{{ {"x86_64": "amd64", "i386": "386"}[ansible_architecture] }}.zip' + remote_src: yes + dest: /usr/bin + mode: 0555 + creates: /usr/bin/terraform diff --git a/tools/config_management/roles/docker-configuration/files/docker_over_tcp.conf b/tools/config_management/roles/docker-configuration/files/docker_over_tcp.conf new file mode 100644 index 000000000..c1598245b --- /dev/null +++ b/tools/config_management/roles/docker-configuration/files/docker_over_tcp.conf @@ -0,0 +1,3 @@ +[Service] +ExecStart= +ExecStart=/usr/bin/docker daemon -H fd:// -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay diff --git a/tools/config_management/roles/docker-configuration/tasks/main.yml b/tools/config_management/roles/docker-configuration/tasks/main.yml new file mode 100644 index 000000000..a9d5c1998 --- /dev/null +++ b/tools/config_management/roles/docker-configuration/tasks/main.yml @@ -0,0 +1,34 @@ +--- +# Configure Docker +# See also: https://docs.docker.com/engine/installation/linux/ubuntulinux/#install + +- name: ensure docker group is present (or create it) + group: + name: docker + state: present + +- name: add user to docker group (avoids sudo-ing) + user: + name: "{{ ansible_user }}" + group: docker + state: present + +- name: ensure docker's systemd directory exists + file: + path: /etc/systemd/system/docker.service.d + state: directory + recurse: yes + +- name: enable docker remote api over tcp + copy: + src: "{{ role_path }}/files/docker_over_tcp.conf" + dest: /etc/systemd/system/docker.service.d/docker_over_tcp.conf + register: docker_over_tcp + +- name: restart docker service + systemd: + name: docker + state: restarted + daemon_reload: yes # ensure docker_over_tcp.conf is picked up. + enabled: yes + when: docker_over_tcp.changed diff --git a/tools/config_management/roles/docker-from-docker-repo/tasks/debian.yml b/tools/config_management/roles/docker-from-docker-repo/tasks/debian.yml new file mode 100644 index 000000000..28bf8d96b --- /dev/null +++ b/tools/config_management/roles/docker-from-docker-repo/tasks/debian.yml @@ -0,0 +1,35 @@ +--- +# Debian / Ubuntu specific: + +- name: install dependencies for docker repository + package: + name: "{{ item }}" + state: present + with_items: + - apt-transport-https + - ca-certificates + +- name: add apt key for the docker repository + apt_key: + keyserver: hkp://ha.pool.sks-keyservers.net:80 + id: 58118E89F3A912897C070ADBF76221572C52609D + state: present + register: apt_key_docker_repo + +- name: add docker's apt repository ({{ ansible_distribution | lower }}-{{ ansible_distribution_release }}) + apt_repository: + repo: deb https://apt.dockerproject.org/repo {{ ansible_distribution | lower }}-{{ ansible_distribution_release }} main + state: present + register: apt_docker_repo + +- name: update apt's cache + apt: + update_cache: yes + when: apt_key_docker_repo.changed or apt_docker_repo.changed + +- name: install docker-engine + package: + name: "{{ item }}" + state: present + with_items: + - docker-engine={{ docker_version }}* diff --git a/tools/config_management/roles/docker-from-docker-repo/tasks/main.yml b/tools/config_management/roles/docker-from-docker-repo/tasks/main.yml new file mode 100644 index 000000000..c3707a1c4 --- /dev/null +++ b/tools/config_management/roles/docker-from-docker-repo/tasks/main.yml @@ -0,0 +1,16 @@ +--- +# Set up Docker +# See also: https://docs.docker.com/engine/installation/linux/ubuntulinux/#install + +- include_role: + name: docker-prerequisites + +# Distribution-specific tasks: +- include: debian.yml + when: ansible_os_family == "Debian" + +- include: redhat.yml + when: ansible_os_family == "RedHat" + +- include_role: + name: docker-configuration diff --git a/tools/config_management/roles/docker-from-docker-repo/tasks/redhat.yml b/tools/config_management/roles/docker-from-docker-repo/tasks/redhat.yml new file mode 100644 index 000000000..ae491300f --- /dev/null +++ b/tools/config_management/roles/docker-from-docker-repo/tasks/redhat.yml @@ -0,0 +1,25 @@ +--- +# RedHat / CentOS specific: + +- name: add docker' yum repository (centos/{{ ansible_lsb.major_release }}) + yum_repository: + name: docker + description: Docker YUM repo + file: external_repos + baseurl: https://yum.dockerproject.org/repo/main/centos/{{ ansible_lsb.major_release }} + enabled: yes + gpgkey: https://yum.dockerproject.org/gpg + gpgcheck: yes + state: present + +- name: update yum's cache + yum: + name: "*" + update_cache: yes + +- name: install docker-engine + package: + name: "{{ item }}" + state: present + with_items: + - docker-engine-{{ docker_version }} diff --git a/tools/config_management/roles/docker-from-get.docker.com/tasks/main.yml b/tools/config_management/roles/docker-from-get.docker.com/tasks/main.yml new file mode 100644 index 000000000..ef47b6f75 --- /dev/null +++ b/tools/config_management/roles/docker-from-get.docker.com/tasks/main.yml @@ -0,0 +1,15 @@ +--- +# Set up Docker +# See also: legacy gce.sh script + +- include_role: + name: docker-prerequisites + +- name: add apt key for the docker repository + shell: curl -sSL https://get.docker.com/gpg | sudo apt-key add - + +- name: install docker + shell: 'curl -sSL https://get.docker.com/ | sed -e s/docker-engine/docker-engine={{ docker_version }}*/ | sh' + +- include_role: + name: docker-configuration diff --git a/tools/config_management/roles/docker-from-tarball/tasks/main.yml b/tools/config_management/roles/docker-from-tarball/tasks/main.yml new file mode 100644 index 000000000..90d4c8d10 --- /dev/null +++ b/tools/config_management/roles/docker-from-tarball/tasks/main.yml @@ -0,0 +1,61 @@ +--- +# Set up Docker +# See also: +# - https://docs.docker.com/engine/installation/linux/ubuntulinux/#install +# - https://github.com/docker/docker/releases + +- include_role: + name: docker-prerequisites + +- name: install daemon + package: + name: daemon + state: present + +- name: 'create directory {{ docker_dir }}/{{ docker_version }}' + file: + path: '{{ docker_dir }}/{{ docker_version }}' + state: directory + mode: 0755 + +- name: download and extract docker + unarchive: + src: 'https://get.docker.com/builds/Linux/x86_64/docker-{{ docker_version }}.tgz' + remote_src: yes + dest: '{{ docker_dir }}/{{ docker_version }}' + extra_opts: '--strip-components=1' + mode: 0555 + creates: '{{ docker_dir }}/{{ docker_version }}/docker' + +- name: create symlink to current version + file: + src: '{{ docker_dir }}/{{ docker_version }}' + dest: '{{ docker_dir }}/current' + state: link + mode: 0555 + +- name: list all files to symlink + find: + paths: '{{ docker_dir }}/current' + file_type: file + register: binaries + changed_when: false + +- name: create symlinks to all binaries + file: + src: '{{ item }}' + dest: /usr/bin/{{ item | basename }} + state: link + with_items: "{{ binaries.files | map(attribute='path') | list }}" + +- name: killall docker + command: killall docker + register: killall + failed_when: false + changed_when: killall.rc == 0 + +- name: start dockerd + command: daemon -- /usr/bin/dockerd + +- include_role: + name: docker-configuration diff --git a/tools/config_management/roles/docker-from-tarball/vars/main.yml b/tools/config_management/roles/docker-from-tarball/vars/main.yml new file mode 100644 index 000000000..d41066848 --- /dev/null +++ b/tools/config_management/roles/docker-from-tarball/vars/main.yml @@ -0,0 +1,5 @@ +--- +docker_dir: '/opt/docker' +docker_url: '{{ "rc" in {{ docker_version }} | ternary( > + "https://test.docker.com/builds/Linux/x86_64/docker-{{ docker_version }}.tgz", > + "https://get.docker.com/builds/Linux/x86_64/docker-{{ docker_version }}.tgz") }}' diff --git a/tools/config_management/roles/docker-prerequisites/tasks/debian.yml b/tools/config_management/roles/docker-prerequisites/tasks/debian.yml new file mode 100644 index 000000000..ef2ea1484 --- /dev/null +++ b/tools/config_management/roles/docker-prerequisites/tasks/debian.yml @@ -0,0 +1,11 @@ +--- +# Install Docker's dependencies +# See also: https://docs.docker.com/engine/installation/linux/ubuntulinux/#install + +- name: install linux-image-extra-*/virtual + package: + name: "{{ item }}" + state: present + with_items: + - linux-image-extra-{{ ansible_kernel }} + - linux-image-extra-virtual diff --git a/tools/config_management/roles/docker-prerequisites/tasks/main.yml b/tools/config_management/roles/docker-prerequisites/tasks/main.yml new file mode 100644 index 000000000..a81773720 --- /dev/null +++ b/tools/config_management/roles/docker-prerequisites/tasks/main.yml @@ -0,0 +1,5 @@ +--- + +# Distribution-specific tasks: +- include: debian.yml + when: ansible_os_family == "Debian" diff --git a/tools/config_management/roles/golang-from-tarball/tasks/main.yml b/tools/config_management/roles/golang-from-tarball/tasks/main.yml new file mode 100644 index 000000000..1b607e1ff --- /dev/null +++ b/tools/config_management/roles/golang-from-tarball/tasks/main.yml @@ -0,0 +1,36 @@ +--- +# Set up Go. + +- name: install go + unarchive: + src: 'https://storage.googleapis.com/golang/go{{ go_version }}.linux-{{ {"x86_64": "amd64", "i386": "386"}[ansible_architecture] }}.tar.gz' + remote_src: yes + dest: /usr/local + mode: 0777 + creates: /usr/local/go/bin/go + +- name: set go env. vars. and add go to path + blockinfile: + dest: '$HOME/.bashrc' + block: | + export PATH=$PATH:/usr/local/go/bin + export GOPATH=$HOME + state: present + create: yes + mode: 0644 + become: '{{ item }}' + with_items: + - true # Run as root + - false # Run as SSH user + +- name: source ~/.bashrc from ~/.bash_profile + lineinfile: + dest: '$HOME/.bash_profile' + line: '[ -r $HOME/.bashrc ] && source $HOME/.bashrc' + state: present + create: yes + mode: 0644 + become: '{{ item }}' + with_items: + - true # Run as root + - false # Run as SSH user diff --git a/tools/config_management/roles/kubelet-stop/tasks/main.yml b/tools/config_management/roles/kubelet-stop/tasks/main.yml new file mode 100644 index 000000000..405c1549a --- /dev/null +++ b/tools/config_management/roles/kubelet-stop/tasks/main.yml @@ -0,0 +1,14 @@ +--- + +- name: check if kubelet service exists + stat: + path: /etc/init.d/kubelet + register: kubelet + +# avoids having weave-net and weave-kube conflict in some test cases (e.g. 130_expose_test.sh) +- name: stop kubelet service + systemd: + name: kubelet + state: stopped + enabled: no + when: kubelet.stat.exists diff --git a/tools/config_management/roles/kubernetes-install/tasks/debian.yml b/tools/config_management/roles/kubernetes-install/tasks/debian.yml new file mode 100644 index 000000000..532356d1b --- /dev/null +++ b/tools/config_management/roles/kubernetes-install/tasks/debian.yml @@ -0,0 +1,28 @@ +--- +# Debian / Ubuntu specific: + +- name: add apt key for the kubernetes repository + apt_key: + url: https://packages.cloud.google.com/apt/doc/apt-key.gpg + state: present + register: apt_key_k8s_repo + +- name: add kubernetes' apt repository (kubernetes-{{ ansible_distribution_release }}) + apt_repository: + repo: deb http://apt.kubernetes.io/ kubernetes-{{ ansible_distribution_release }} main + state: present + register: apt_k8s_repo + +- name: update apt's cache + apt: + update_cache: yes + when: apt_key_k8s_repo.changed or apt_k8s_repo.changed + +- name: install kubelet and kubectl + package: + name: "{{ item }}" + state: present + with_items: + - kubelet={{ kubernetes_version }}* + - kubectl={{ kubernetes_version }}* + - kubernetes-cni={{ kubernetes_cni_version }}* diff --git a/tools/config_management/roles/kubernetes-install/tasks/main.yml b/tools/config_management/roles/kubernetes-install/tasks/main.yml new file mode 100644 index 000000000..c602b2cdd --- /dev/null +++ b/tools/config_management/roles/kubernetes-install/tasks/main.yml @@ -0,0 +1,17 @@ +--- +# Install Kubernetes + +# Distribution-specific tasks: +- include: debian.yml + when: ansible_os_family == "Debian" + +- include: redhat.yml + when: ansible_os_family == "RedHat" + +- name: install ebtables and kubeadm + package: + name: "{{ item }}" + state: present + with_items: + - ebtables + - kubeadm diff --git a/tools/config_management/roles/kubernetes-install/tasks/redhat.yml b/tools/config_management/roles/kubernetes-install/tasks/redhat.yml new file mode 100644 index 000000000..f85b872e9 --- /dev/null +++ b/tools/config_management/roles/kubernetes-install/tasks/redhat.yml @@ -0,0 +1,29 @@ +--- +# RedHat / CentOS specific: + +- name: add kubernetes' yum repository (kubernetes-el{{ ansible_lsb.major_release }}-x86-64) + yum_repository: + name: kubernetes + description: Kubernetes YUM repo + file: external_repos + baseurl: https://packages.cloud.google.com/yum/repos/kubernetes-el{{ ansible_lsb.major_release }}-x86_64 + enabled: yes + gpgkey: https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg + gpgcheck: yes + state: present + register: yum_k8s_repo + +- name: update yum's cache + yum: + name: "*" + update_cache: yes + when: yum_k8s_repo.changed + +- name: install kubelet and kubectl + package: + name: "{{ item }}" + state: present + with_items: + - kubelet-{{ kubernetes_version }} + - kubectl-{{ kubernetes_version }} + - kubernetes-cni-{{ kubernetes_cni_version }}* diff --git a/tools/config_management/roles/kubernetes-start/tasks/main.yml b/tools/config_management/roles/kubernetes-start/tasks/main.yml new file mode 100644 index 000000000..7910f1403 --- /dev/null +++ b/tools/config_management/roles/kubernetes-start/tasks/main.yml @@ -0,0 +1,37 @@ +--- +# Start Kubernetes + +- name: kubeadm reset + command: kubeadm reset + +- name: restart kubelet service + systemd: + name: kubelet + state: restarted + enabled: yes + +- name: kubeadm init on the master + command: 'kubeadm init --token={{ kubernetes_token }}' + when: ' {{ play_hosts[0] == inventory_hostname }}' + +- name: allow pods to be run on the master (if only node) + command: kubectl taint nodes --all dedicated- + when: '{{ play_hosts | length }} == 1' + +- name: kubeadm join on workers + command: "kubeadm join --token={{ kubernetes_token }} {{ hostvars[play_hosts[0]].private_ip }}" + when: ' {{ play_hosts[0] != inventory_hostname }}' + +- name: list kubernetes' pods + command: kubectl get pods --all-namespaces + when: ' {{ play_hosts[0] == inventory_hostname }}' + changed_when: false + register: kubectl_get_pods + tags: + - output + +- name: print outpout of `kubectl get pods --all-namespaces` + debug: msg="{{ kubectl_get_pods.stdout_lines }}" + when: ' {{ play_hosts[0] == inventory_hostname }}' + tags: + - output diff --git a/tools/config_management/roles/setup-ansible/pre_tasks/main.yml b/tools/config_management/roles/setup-ansible/pre_tasks/main.yml new file mode 100644 index 000000000..efb154917 --- /dev/null +++ b/tools/config_management/roles/setup-ansible/pre_tasks/main.yml @@ -0,0 +1,26 @@ +--- +# Set machine up to be able to run ansible playbooks. + +- name: check if python is installed (as required by ansible modules) + raw: test -e /usr/bin/python + register: is_python_installed + failed_when: is_python_installed.rc not in [0, 1] + changed_when: false # never mutates state. + +- name: install python if missing (as required by ansible modules) + when: is_python_installed|failed # skip otherwise + raw: (test -e /usr/bin/apt-get && apt-get install -y python-minimal) || (test -e /usr/bin/yum && yum install -y python) + changed_when: is_python_installed.rc == 1 + +- name: check if lsb_release is installed (as required for ansible facts) + raw: test -e /usr/bin/lsb_release + register: is_lsb_release_installed + failed_when: is_lsb_release_installed.rc not in [0, 1] + changed_when: false # never mutates state. + +- name: install lsb_release if missing (as required for ansible facts) + when: is_lsb_release_installed|failed # skip otherwise + raw: (test -e /usr/bin/apt-get && apt-get install -y lsb_release) || (test -e /usr/bin/yum && yum install -y lsb_release) + changed_when: is_lsb_release_installed.rc == 1 + +- setup: # gather 'facts', i.e. compensates for the above 'gather_facts: false'. diff --git a/tools/config_management/roles/sock-shop/tasks/tasks.yml b/tools/config_management/roles/sock-shop/tasks/tasks.yml new file mode 100644 index 000000000..fc00e8096 --- /dev/null +++ b/tools/config_management/roles/sock-shop/tasks/tasks.yml @@ -0,0 +1,34 @@ +--- +# Set up sock-shop on top of Kubernetes. +# Dependencies on other roles: +# - kubernetes + +- name: create sock-shop namespace in k8s + command: kubectl create namespace sock-shop + +- name: create sock-shop in k8s + command: kubectl apply -n sock-shop -f "https://github.com/microservices-demo/microservices-demo/blob/master/deploy/kubernetes/complete-demo.yaml?raw=true" + +- name: describe front-end service + command: kubectl describe svc front-end -n sock-shop + changed_when: false + register: kubectl_describe_svc_frontend + tags: + - output + +- name: print outpout of `kubectl describe svc front-end -n sock-shop` + debug: msg="{{ kubectl_describe_svc_frontend.stdout_lines }}" + tags: + - output + +- name: list sock-shop k8s' pods + command: kubectl get pods -n sock-shop + changed_when: false + register: kubectl_get_pods + tags: + - output + +- name: print outpout of `kubectl get pods -n sock-shop` + debug: msg="{{ kubectl_get_pods.stdout_lines }}" + tags: + - output diff --git a/tools/config_management/roles/weave-kube/tasks/main.yml b/tools/config_management/roles/weave-kube/tasks/main.yml new file mode 100644 index 000000000..fcaa7e20f --- /dev/null +++ b/tools/config_management/roles/weave-kube/tasks/main.yml @@ -0,0 +1,20 @@ +--- +# Set up Weave Kube on top of Kubernetes. + +- name: configure weave net's cni plugin + command: kubectl apply -f https://git.io/weave-kube + when: ' {{ play_hosts[0] == inventory_hostname }}' + +- name: list kubernetes' pods + command: kubectl get pods --all-namespaces + when: ' {{ play_hosts[0] == inventory_hostname }}' + changed_when: false + register: kubectl_get_pods + tags: + - output + +- name: print outpout of `kubectl get pods --all-namespaces` + debug: msg="{{ kubectl_get_pods.stdout_lines }}" + when: ' {{ play_hosts[0] == inventory_hostname }}' + tags: + - output diff --git a/tools/config_management/roles/weave-net-sources/tasks/main.yml b/tools/config_management/roles/weave-net-sources/tasks/main.yml new file mode 100644 index 000000000..f753cd41b --- /dev/null +++ b/tools/config_management/roles/weave-net-sources/tasks/main.yml @@ -0,0 +1,24 @@ +--- +# Set up Development Environment for Weave Net. + +- name: check if weave net has been checked out + become: false # Run as SSH-user + stat: + path: $HOME/src/github.com/weaveworks/weave + register: weave + failed_when: false + changed_when: false + +- name: git clone weave net + become: false # Run as SSH-user + git: + repo: https://github.com/weaveworks/weave.git + dest: $HOME/src/github.com/weaveworks/weave + when: not weave.stat.exists + +- name: create a convenience symlink to $HOME/src/github.com/weaveworks/weave + become: false # Run as SSH-user + file: + src: $HOME/src/github.com/weaveworks/weave + dest: $HOME/weave + state: link diff --git a/tools/config_management/roles/weave-net-utilities/tasks/main.yml b/tools/config_management/roles/weave-net-utilities/tasks/main.yml new file mode 100644 index 000000000..8032f6a2d --- /dev/null +++ b/tools/config_management/roles/weave-net-utilities/tasks/main.yml @@ -0,0 +1,53 @@ +--- + +- name: install jq + package: + name: "{{ item }}" + state: present + with_items: + - jq + +- name: install ethtool (used by the weave script) + package: + name: "{{ item }}" + install_recommends: no + state: present + with_items: + - ethtool + +- name: install nsenter (used by the weave script) + command: docker run --rm -v /usr/local/bin:/target jpetazzo/nsenter + +- name: install pip (for docker-py) + package: + name: "{{ item }}" + state: present + with_items: + - python-pip + +- name: install docker-py (for docker_image) + pip: + name: docker-py + state: present + +- name: docker pull images used by tests + docker_image: + name: '{{ item }}' + state: present + with_items: + - alpine + - aanand/docker-dnsutils + - weaveworks/hello-world + +- name: docker pull images used by k8s tests + docker_image: + name: '{{ item }}' + state: present + with_items: + - gcr.io/google_containers/etcd-amd64:{{ etcd_container_version }} + - gcr.io/google_containers/kube-apiserver-amd64:v{{ kubernetes_version }} + - gcr.io/google_containers/kube-controller-manager-amd64:v{{ kubernetes_version }} + - gcr.io/google_containers/kube-proxy-amd64:v{{ kubernetes_version }} + - gcr.io/google_containers/kube-scheduler-amd64:v{{ kubernetes_version }} + - gcr.io/google_containers/kube-discovery-amd64:{{ kube_discovery_container_version }} + - gcr.io/google_containers/pause-amd64:{{ pause_container_version }} diff --git a/tools/config_management/roles/weave-net/tasks/main.yml b/tools/config_management/roles/weave-net/tasks/main.yml new file mode 100644 index 000000000..dfd370404 --- /dev/null +++ b/tools/config_management/roles/weave-net/tasks/main.yml @@ -0,0 +1,26 @@ +--- +# Set up Weave Net. + +- name: install weave net + get_url: + url: https://git.io/weave + dest: /usr/local/bin/weave + mode: 0555 + +- name: stop weave net + command: weave stop + +- name: start weave net + command: weave launch + +- name: get weave net's status + command: weave status + changed_when: false + register: weave_status + tags: + - output + +- name: print outpout of `weave status` + debug: msg="{{ weave_status.stdout_lines }}" + tags: + - output diff --git a/tools/config_management/setup_weave-kube.yml b/tools/config_management/setup_weave-kube.yml new file mode 100644 index 000000000..14a73124c --- /dev/null +++ b/tools/config_management/setup_weave-kube.yml @@ -0,0 +1,26 @@ +--- +################################################################################ +# Install Docker and Kubernetes, and configure Kubernetes to +# use Weave Net's CNI plugin (a.k.a. Weave Kube). +# +# See also: +# - http://kubernetes.io/docs/getting-started-guides/kubeadm/ +# - https://github.com/weaveworks/weave-kube +################################################################################ + +- name: install docker, kubernetes and weave-kube + hosts: all + gather_facts: false # required in case Python is not available on the host + become: true + become_user: root + + pre_tasks: + - include: library/setup_ansible_dependencies.yml + + roles: + - docker-from-get.docker.com + - kubernetes-install + - weave-net-utilities + - kubelet-stop + - kubernetes-start + - weave-kube diff --git a/tools/config_management/setup_weave-net_dev.yml b/tools/config_management/setup_weave-net_dev.yml new file mode 100644 index 000000000..f72149d04 --- /dev/null +++ b/tools/config_management/setup_weave-net_dev.yml @@ -0,0 +1,20 @@ +--- +################################################################################ +# Install Docker from Docker's official repository and Weave Net. +################################################################################ + +- name: install docker and weave net for development + hosts: all + gather_facts: false # required in case Python is not available on the host + become: true + become_user: root + + pre_tasks: + - include: library/setup_ansible_dependencies.yml + + roles: + - dev-tools + - golang-from-tarball + - docker-from-get.docker.com + # Do not run this role when building with Vagrant, as sources have been already checked out: + - { role: weave-net-sources, when: "ansible_user != 'vagrant'" } diff --git a/tools/config_management/setup_weave-net_test.yml b/tools/config_management/setup_weave-net_test.yml new file mode 100644 index 000000000..9fb46be68 --- /dev/null +++ b/tools/config_management/setup_weave-net_test.yml @@ -0,0 +1,19 @@ +--- +################################################################################ +# Install Docker from Docker's official repository and Weave Net. +################################################################################ + +- name: install docker and weave net for testing + hosts: all + gather_facts: false # required in case Python is not available on the host + become: true + become_user: root + + pre_tasks: + - include: library/setup_ansible_dependencies.yml + + roles: + - docker-from-get.docker.com + - kubernetes-install + - weave-net-utilities + - kubelet-stop diff --git a/tools/dependencies/cross_versions.py b/tools/dependencies/cross_versions.py new file mode 100755 index 000000000..bc93cf317 --- /dev/null +++ b/tools/dependencies/cross_versions.py @@ -0,0 +1,93 @@ +#!/usr/bin/python + +# Generate the cross product of latest versions of Weave Net's dependencies: +# - Go +# - Docker +# - Kubernetes +# +# Dependencies: +# - python +# - git +# - list_versions.py +# +# Testing: +# $ python -m doctest -v cross_versions.py + +from os import linesep +from sys import argv, exit, stdout, stderr +from getopt import getopt, GetoptError +from list_versions import DEPS, get_versions_from, filter_versions +from itertools import product + +# See also: /usr/include/sysexits.h +_ERROR_RUNTIME=1 +_ERROR_ILLEGAL_ARGS=64 + +def _usage(error_message=None): + if error_message: + stderr.write('ERROR: ' + error_message + linesep) + stdout.write(linesep.join([ + 'Usage:', + ' cross_versions.py [OPTION]...', + 'Examples:', + ' cross_versions.py', + ' cross_versions.py -r', + ' cross_versions.py --rc', + ' cross_versions.py -l', + ' cross_versions.py --latest', + 'Options:', + '-l/--latest Include only the latest version of each major and minor versions sub-tree.', + '-r/--rc Include release candidate versions.', + '-h/--help Prints this!', + '' + ])) + +def _validate_input(argv): + try: + config = { + 'rc': False, + 'latest': False + } + opts, args = getopt(argv, 'hlr', ['help', 'latest', 'rc']) + for opt, value in opts: + if opt in ('-h', '--help'): + _usage() + exit() + if opt in ('-l', '--latest'): + config['latest'] = True + if opt in ('-r', '--rc'): + config['rc'] = True + if len(args) != 0: + raise ValueError('Unsupported argument(s): %s.' % args) + return config + except GetoptError as e: + _usage(str(e)) + exit(_ERROR_ILLEGAL_ARGS) + except ValueError as e: + _usage(str(e)) + exit(_ERROR_ILLEGAL_ARGS) + +def _versions(dependency, config): + return map(str, + filter_versions( + get_versions_from(DEPS[dependency]['url'], DEPS[dependency]['re']), + DEPS[dependency]['min'], + **config + ) + ) + +def cross_versions(config): + docker_versions = _versions('docker', config) + k8s_versions = _versions('kubernetes', config) + return product(docker_versions, k8s_versions) + +def main(argv): + try: + config = _validate_input(argv) + print(linesep.join('\t'.join(triple) for triple in cross_versions(config))) + except Exception as e: + print(str(e)) + exit(_ERROR_RUNTIME) + +if __name__ == '__main__': + main(argv[1:]) diff --git a/tools/dependencies/list_os_images.sh b/tools/dependencies/list_os_images.sh new file mode 100755 index 000000000..7d2495da3 --- /dev/null +++ b/tools/dependencies/list_os_images.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +function usage() { + cat <&2 "No AWS owner ID for $1." + exit 1 +} + +if [ -z "$1" ]; then + echo >&2 "No specified provider." + usage + exit 1 +fi + +if [ -z "$2" ]; then + if [ "$1" == "help" ]; then + usage + exit 0 + else + echo >&2 "No specified operating system." + usage + exit 1 + fi +fi + +case "$1" in + 'gcp') + gcloud compute images list --standard-images --regexp=".*?$2.*" \ + --format="csv[no-heading][separator=/](selfLink.map().scope(projects).segment(0),family)" \ + | sort -d + ;; + 'aws') + aws --region "${3:-us-east-1}" ec2 describe-images \ + --owners "$(find_aws_owner_id "$2")" \ + --filters "Name=name,Values=$2*" \ + --query 'Images[*].{name:Name,id:ImageId}' + # Other examples: + # - CentOS: aws --region us-east-1 ec2 describe-images --owners aws-marketplace --filters Name=product-code,Values=aw0evgkw8e5c1q413zgy5pjce + # - Debian: aws --region us-east-1 ec2 describe-images --owners 379101102735 --filters "Name=architecture,Values=x86_64" "Name=name,Values=debian-jessie-*" "Name=root-device-type,Values=ebs" "Name=virtualization-type,Values=hvm" + ;; + 'do') + curl -s -X GET \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \ + "https://api.digitalocean.com/v2/images?page=1&per_page=999999" \ + | jq --raw-output ".images | .[] | .slug" | grep "$2" | sort -d + ;; + *) + echo >&2 "Unknown provider [$1]." + usage + exit 1 + ;; +esac diff --git a/tools/dependencies/list_versions.py b/tools/dependencies/list_versions.py new file mode 100755 index 000000000..5b0d9c42d --- /dev/null +++ b/tools/dependencies/list_versions.py @@ -0,0 +1,266 @@ +#!/usr/bin/python + +# List all available versions of Weave Net's dependencies: +# - Go +# - Docker +# - Kubernetes +# +# Depending on the parameters passed, it can gather the equivalent of the below bash one-liners: +# git ls-remote --tags https://github.com/golang/go | grep -oP '(?<=refs/tags/go)[\.\d]+$' | sort --version-sort +# git ls-remote --tags https://github.com/golang/go | grep -oP '(?<=refs/tags/go)[\.\d]+rc\d+$' | sort --version-sort | tail -n 1 +# git ls-remote --tags https://github.com/docker/docker | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+$' | sort --version-sort +# git ls-remote --tags https://github.com/docker/docker | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+\-rc\d*$' | sort --version-sort | tail -n 1 +# git ls-remote --tags https://github.com/kubernetes/kubernetes | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+$' | sort --version-sort +# git ls-remote --tags https://github.com/kubernetes/kubernetes | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+\-beta\.\d+$' | sort --version-sort | tail -n 1 +# +# Dependencies: +# - python +# - git +# +# Testing: +# $ python -m doctest -v list_versions.py + +from os import linesep, path +from sys import argv, exit, stdout, stderr +from getopt import getopt, GetoptError +from subprocess import Popen, PIPE, STDOUT +from pkg_resources import parse_version +from itertools import groupby +from six.moves import filter +import shlex +import re + +# See also: /usr/include/sysexits.h +_ERROR_RUNTIME=1 +_ERROR_ILLEGAL_ARGS=64 + +_TAG_REGEX='^[0-9a-f]{40}\s+refs/tags/%s$' +_VERSION='version' +DEPS={ + 'go': { + 'url': 'https://github.com/golang/go', + 're': 'go(?P<%s>[\d\.]+(?:rc\d)*)' % _VERSION, + 'min': None + }, + 'docker': { + 'url': 'https://github.com/docker/docker', + 're': 'v(?P<%s>\d+\.\d+\.\d+(?:\-rc\d)*)' % _VERSION, + # Weave Net only works with Docker from 1.6.0 onwards, so we ignore all previous versions: + 'min': '1.6.0' + }, + 'kubernetes': { + 'url': 'https://github.com/kubernetes/kubernetes', + 're': 'v(?P<%s>\d+\.\d+\.\d+(?:\-beta\.\d)*)' % _VERSION, + # Weave Kube requires Kubernetes 1.4.2+, so we ignore all previous versions: + 'min': '1.4.2' + } +} + +class Version(object): + ''' Helper class to parse and manipulate (sort, filter, group) software versions. ''' + def __init__(self, version): + self.version = version + self.digits = [int(x) if x else 0 for x in re.match('(\d*)\.?(\d*)\.?(\d*).*?', version).groups()] + self.major, self.minor, self.patch = self.digits + self.__parsed = parse_version(version) + self.is_rc = self.__parsed.is_prerelease + def __lt__ (self, other): + return self.__parsed.__lt__(other.__parsed) + def __gt__ (self, other): + return self.__parsed.__gt__(other.__parsed) + def __le__ (self, other): + return self.__parsed.__le__(other.__parsed) + def __ge__ (self, other): + return self.__parsed.__ge__(other.__parsed) + def __eq__ (self, other): + return self.__parsed.__eq__(other.__parsed) + def __ne__ (self, other): + return self.__parsed.__ne__(other.__parsed) + def __str__(self): + return self.version + def __repr__(self): + return self.version + +def _read_go_version_from_dockerfile(): + # Read Go version from weave/build/Dockerfile + dockerfile_path = path.join(path.dirname(path.dirname(path.dirname(path.realpath(__file__)))), 'build', 'Dockerfile') + with open(dockerfile_path, 'r') as f: + for line in f: + m = re.match('^FROM golang:(\S*)$', line) + if m: + return m.group(1) + raise RuntimeError("Failed to read Go version from weave/build/Dockerfile. You may be running this script from somewhere else than weave/tools.") + +def _try_set_min_go_version(): + ''' Set the current version of Go used to build Weave Net's containers as the minimum version. ''' + try: + DEPS['go']['min'] = _read_go_version_from_dockerfile() + except IOError as e: + stderr.write('WARNING: No minimum Go version set. Root cause: %s%s' % (e, linesep)) + +def _sanitize(out): + return out.decode('ascii').strip().split(linesep) + +def _parse_tag(tag, version_pattern, debug=False): + ''' Parse Git tag output's line using the provided `version_pattern`, e.g.: + >>> _parse_tag('915b77eb4efd68916427caf8c7f0b53218c5ea4a refs/tags/v1.4.6', 'v(?P\d+\.\d+\.\d+(?:\-beta\.\d)*)') + '1.4.6' + ''' + pattern = _TAG_REGEX % version_pattern + m = re.match(pattern, tag) + if m: + return m.group(_VERSION) + elif debug: + stderr.write('ERROR: Failed to parse version out of tag [%s] using [%s].%s' % (tag, pattern, linesep)) + +def get_versions_from(git_repo_url, version_pattern): + ''' Get release and release candidates' versions from the provided Git repository. ''' + git = Popen(shlex.split('git ls-remote --tags %s' % git_repo_url), stdout=PIPE) + out, err = git.communicate() + status_code = git.returncode + if status_code != 0: + raise RuntimeError('Failed to retrieve git tags from %s. Status code: %s. Output: %s. Error: %s' % (git_repo_url, status_code, out, err)) + return list(filter(None, (_parse_tag(line, version_pattern) for line in _sanitize(out)))) + +def _tree(versions, level=0): + ''' Group versions by major, minor and patch version digits. ''' + if not versions or level >= len(versions[0].digits): + return # Empty versions or no more digits to group by. + versions_tree = [] + for _, versions_group in groupby(versions, lambda v: v.digits[level]): + subtree = _tree(list(versions_group), level+1) + if subtree: + versions_tree.append(subtree) + # Return the current subtree if non-empty, or the list of "leaf" versions: + return versions_tree if versions_tree else versions + +def _is_iterable(obj): + ''' + Check if the provided object is an iterable collection, i.e. not a string, e.g. a list, a generator: + >>> _is_iterable('string') + False + >>> _is_iterable([1, 2, 3]) + True + >>> _is_iterable((x for x in [1, 2, 3])) + True + ''' + return hasattr(obj, '__iter__') and not isinstance(obj, str) + +def _leaf_versions(tree, rc): + ''' + Recursively traverse the versions tree in a depth-first fashion, + and collect the last node of each branch, i.e. leaf versions. + ''' + versions = [] + if _is_iterable(tree): + for subtree in tree: + versions.extend(_leaf_versions(subtree, rc)) + if not versions: + if rc: + last_rc = next(filter(lambda v: v.is_rc, reversed(tree)), None) + last_prod = next(filter(lambda v: not v.is_rc, reversed(tree)), None) + if last_rc and last_prod and (last_prod < last_rc): + versions.extend([last_prod, last_rc]) + elif not last_prod: + versions.append(last_rc) + else: + # Either there is no RC, or we ignore the RC as older than the latest production version: + versions.append(last_prod) + else: + versions.append(tree[-1]) + return versions + +def filter_versions(versions, min_version=None, rc=False, latest=False): + ''' Filter provided versions + + >>> filter_versions(['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], min_version=None, latest=False, rc=False) + [1.0.0, 1.0.1, 1.1.1, 2.0.0] + + >>> filter_versions(['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], min_version=None, latest=True, rc=False) + [1.0.1, 1.1.1, 2.0.0] + + >>> filter_versions(['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], min_version=None, latest=False, rc=True) + [1.0.0-beta.1, 1.0.0, 1.0.1, 1.1.1, 1.1.2-rc1, 2.0.0] + + >>> filter_versions(['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], min_version='1.1.0', latest=False, rc=True) + [1.1.1, 1.1.2-rc1, 2.0.0] + + >>> filter_versions(['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], min_version=None, latest=True, rc=True) + [1.0.1, 1.1.1, 1.1.2-rc1, 2.0.0] + + >>> filter_versions(['1.0.0-beta.1', '1.0.0', '1.0.1', '1.1.1', '1.1.2-rc1', '2.0.0'], min_version='1.1.0', latest=True, rc=True) + [1.1.1, 1.1.2-rc1, 2.0.0] + ''' + versions = sorted([Version(v) for v in versions]) + if min_version: + min_version = Version(min_version) + versions = [v for v in versions if v >= min_version] + if not rc: + versions = [v for v in versions if not v.is_rc] + if latest: + versions_tree = _tree(versions) + return _leaf_versions(versions_tree, rc) + else: + return versions + +def _usage(error_message=None): + if error_message: + stderr.write('ERROR: ' + error_message + linesep) + stdout.write(linesep.join([ + 'Usage:', + ' list_versions.py [OPTION]... [DEPENDENCY]', + 'Examples:', + ' list_versions.py go', + ' list_versions.py -r docker', + ' list_versions.py --rc docker', + ' list_versions.py -l kubernetes', + ' list_versions.py --latest kubernetes', + 'Options:', + '-l/--latest Include only the latest version of each major and minor versions sub-tree.', + '-r/--rc Include release candidate versions.', + '-h/--help Prints this!', + '' + ])) + +def _validate_input(argv): + try: + config = { + 'rc': False, + 'latest': False + } + opts, args = getopt(argv, 'hlr', ['help', 'latest', 'rc']) + for opt, value in opts: + if opt in ('-h', '--help'): + _usage() + exit() + if opt in ('-l', '--latest'): + config['latest'] = True + if opt in ('-r', '--rc'): + config['rc'] = True + if len(args) != 1: + raise ValueError('Please provide a dependency to get versions of. Expected 1 argument but got %s: %s.' % (len(args), args)) + dependency=args[0].lower() + if dependency not in DEPS.keys(): + raise ValueError('Please provide a valid dependency. Supported one dependency among {%s} but got: %s.' % (', '.join(DEPS.keys()), dependency)) + return dependency, config + except GetoptError as e: + _usage(str(e)) + exit(_ERROR_ILLEGAL_ARGS) + except ValueError as e: + _usage(str(e)) + exit(_ERROR_ILLEGAL_ARGS) + +def main(argv): + try: + dependency, config = _validate_input(argv) + if dependency == 'go': + _try_set_min_go_version() + versions = get_versions_from(DEPS[dependency]['url'], DEPS[dependency]['re']) + versions = filter_versions(versions, DEPS[dependency]['min'], **config) + print(linesep.join(map(str, versions))) + except Exception as e: + print(str(e)) + exit(_ERROR_RUNTIME) + +if __name__ == '__main__': + main(argv[1:]) diff --git a/tools/integration/config.sh b/tools/integration/config.sh index 3dfa3cac4..6bf208606 100644 --- a/tools/integration/config.sh +++ b/tools/integration/config.sh @@ -130,4 +130,4 @@ end_suite() { whitely assert_end } -WEAVE=$DIR/../weave +WEAVE=$DIR/../../integration/weave diff --git a/tools/integration/gce.sh b/tools/integration/gce.sh index 5129916ed..e52a85739 100755 --- a/tools/integration/gce.sh +++ b/tools/integration/gce.sh @@ -9,7 +9,9 @@ set -e : "${KEY_FILE:=/tmp/gce_private_key.json}" : "${SSH_KEY_FILE:=$HOME/.ssh/gce_ssh_key}" -: "${IMAGE:=ubuntu-14-04}" +: "${IMAGE_FAMILY:=ubuntu-1404-lts}" +: "${IMAGE_PROJECT:=ubuntu-os-cloud}" +: "${USER_ACCOUNT:=ubuntu}" : "${ZONE:=us-central1-a}" : "${PROJECT:=}" : "${TEMPLATE_NAME:=}" @@ -22,7 +24,9 @@ fi SUFFIX="" if [ -n "$CIRCLECI" ]; then - SUFFIX="-${CIRCLE_BUILD_NUM}-$CIRCLE_NODE_INDEX" + SUFFIX="-${CIRCLE_PROJECT_USERNAME}-${CIRCLE_PROJECT_REPONAME}-${CIRCLE_BUILD_NUM}-$CIRCLE_NODE_INDEX" +else + SUFFIX="-${USER}" fi # Setup authentication @@ -41,7 +45,8 @@ function vm_names() { function destroy() { local names names="$(vm_names)" - if [ "$(gcloud compute instances list --zone "$ZONE" -q "$names" | wc -l)" -le 1 ]; then + # shellcheck disable=SC2086 + if [ "$(gcloud compute instances list --zones "$ZONE" -q $names | wc -l)" -le 1 ]; then return 0 fi for i in {0..10}; do @@ -82,12 +87,16 @@ function try_connect() { function install_docker_on() { name=$1 + echo "Installing Docker on $name for user ${USER_ACCOUNT}" + # shellcheck disable=SC2087 ssh -t "$name" sudo bash -x -s <> /etc/default/docker; service docker restart EOF @@ -136,7 +145,7 @@ function setup() { } function make_template() { - gcloud compute instances create "$TEMPLATE_NAME" --image "$IMAGE" --zone "$ZONE" + gcloud compute instances create "$TEMPLATE_NAME" --image-family "$IMAGE_FAMILY" --image-project "$IMAGE_PROJECT" --zone "$ZONE" gcloud compute config-ssh --ssh-key-file "$SSH_KEY_FILE" name="$TEMPLATE_NAME.$ZONE.$PROJECT" try_connect "$name" @@ -155,7 +164,7 @@ function hosts() { hosts=($hostname "${hosts[@]}") args=("--add-host=$hostname:$(internal_ip "$json" "$name")" "${args[@]}") done - echo export SSH=\"ssh -l vagrant\" + echo export SSH=\"ssh -l "${USER_ACCOUNT}"\" echo "export HOSTS=\"${hosts[*]}\"" echo "export ADD_HOST_ARGS=\"${args[*]}\"" rm "$json" @@ -178,6 +187,9 @@ case "$1" in # see if template exists if ! gcloud compute images list | grep "$PROJECT" | grep "$TEMPLATE_NAME"; then make_template + else + echo "Reusing existing template:" + gcloud compute images describe "$TEMPLATE_NAME" | grep "^creationTimestamp" fi ;; esac diff --git a/tools/integration/sanity_check.sh b/tools/integration/sanity_check.sh index c88337fe8..192112de8 100755 --- a/tools/integration/sanity_check.sh +++ b/tools/integration/sanity_check.sh @@ -1,6 +1,6 @@ #! /bin/bash -# shellcheck disable=SC1091 -. ./config.sh +# shellcheck disable=SC1090,SC1091 +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/config.sh" set -e diff --git a/tools/lint b/tools/lint index 25f191a53..b9428d9f2 100755 --- a/tools/lint +++ b/tools/lint @@ -6,7 +6,7 @@ # # For shell files, it runs shfmt. If you don't have that installed, you can get # it with: -# go get -u github.com/mvdan/sh/cmd/shfmt +# go get -u gopkg.in/mvdan/sh.v1/cmd/shfmt # # With no arguments, it lints the current files staged # for git commit. Or you can pass it explicit filenames @@ -17,8 +17,11 @@ set -e +LINT_IGNORE_FILE=${LINT_IGNORE_FILE:-".lintignore"} + IGNORE_LINT_COMMENT= IGNORE_SPELLINGS= +PARALLEL= while true; do case "$1" in -nocomment) @@ -33,6 +36,10 @@ while true; do IGNORE_SPELLINGS="$2,$IGNORE_SPELLINGS" shift 2 ;; + -p) + PARALLEL=1 + shift 1 + ;; *) break ;; @@ -142,10 +149,11 @@ lint() { return fi - # Don't lint static.go + # Don't lint specific files case "$(basename "${filename}")" in static.go) return ;; coverage.html) return ;; + *.pb.go) return ;; esac if [[ "$(file --mime-type "${filename}" | awk '{print $2}')" == "text/x-shellscript" ]]; then @@ -171,12 +179,46 @@ lint_files() { exit $lint_result } -list_files() { - if [ $# -gt 0 ]; then - git ls-files --exclude-standard | grep -vE '(^|/)vendor/' +matches_any() { + local filename="$1" + local patterns="$2" + while read -r pattern; do + # shellcheck disable=SC2053 + # Use the [[ operator without quotes on $pattern + # in order to "glob" the provided filename: + [[ "$filename" == $pattern ]] && return 0 + done <<<"$patterns" + return 1 +} + +filter_out() { + local patterns_file="$1" + if [ -n "$patterns_file" ] && [ -r "$patterns_file" ]; then + local patterns + patterns=$(sed '/^#.*$/d ; /^\s*$/d' "$patterns_file") # Remove blank lines and comments before we start iterating. + [ -n "$DEBUG" ] && echo >&2 "> Filters:" && echo >&2 "$patterns" + local filtered_out=() + while read -r filename; do + matches_any "$filename" "$patterns" && filtered_out+=("$filename") || echo "$filename" + done + [ -n "$DEBUG" ] && echo >&2 "> Files filtered out (i.e. NOT linted):" && printf >&2 '%s\n' "${filtered_out[@]}" else - git diff --cached --name-only + cat # No patterns provided: simply propagate stdin to stdout. fi } -list_files "$@" | lint_files +list_files() { + if [ $# -gt 0 ]; then + find "$@" | grep -vE '(^|/)vendor/' + else + git ls-files --exclude-standard | grep -vE '(^|/)vendor/' + fi +} + +if [ $# = 1 ] && [ -f "$1" ]; then + lint "$1" +elif [ -n "$PARALLEL" ]; then + list_files "$@" | filter_out "$LINT_IGNORE_FILE" | xargs -n1 -P16 "$0" +else + list_files "$@" | filter_out "$LINT_IGNORE_FILE" | lint_files +fi diff --git a/tools/provisioning/README.md b/tools/provisioning/README.md new file mode 100755 index 000000000..627bb42e3 --- /dev/null +++ b/tools/provisioning/README.md @@ -0,0 +1,55 @@ +# Weaveworks provisioning + +## Introduction + +This project allows you to get hold of some machine either locally or on one of the below cloud providers: + +* Amazon Web Services +* Digital Ocean +* Google Cloud Platform + +You can then use these machines as is or run various Ansible playbooks from `../config_management` to set up Weave Net, Kubernetes, etc. + +## Set up + +* You will need [Vagrant](https://www.vagrantup.com) installed on your machine and added to your `PATH` in order to be able to provision local (virtual) machines automatically. + + * On macOS: `brew install vagrant` + * On Linux (via Aptitude): `sudo apt install vagrant` + * If you need a specific version: + + curl -fsS https://releases.hashicorp.com/terraform/x.y.z/terraform_x.y.z_linux_amd64.zip | gunzip > terraform && chmod +x terraform && sudo mv terraform /usr/bin + + * For other platforms or more details, see [here](https://www.vagrantup.com/docs/installation/) + +* You will need [Terraform](https://www.terraform.io) installed on your machine and added to your `PATH` in order to be able to provision cloud-hosted machines automatically. + + * On macOS: `brew install terraform` + * On Linux (via Aptitude): `sudo apt install terraform` + * For other platforms or more details, see [here](https://www.terraform.io/intro/getting-started/install.html) + +* Depending on the cloud provider, you may have to create an account, manually onboard, create and register SSH keys, etc. + Please refer to the `README.md` in each sub-folder for more details. + +## Usage in scripts + +Source `setup.sh`, set the `SECRET_KEY` environment variable, and depending on the cloud provider you want to use, call either: + +* `gcp_on` / `gcp_off` +* `do_on` / `do_off` +* `aws_on` / `aws_off` + +## Usage in shell + +Source `setup.sh`, set the `SECRET_KEY` environment variable, and depending on the cloud provider you want to use, call either: + +* `gcp_on` / `gcp_off` +* `do_on` / `do_off` +* `aws_on` / `aws_off` + +Indeed, the functions defined in `setup.sh` are also exported as aliases, so you can call them from your shell directly. + +Other aliases are also defined, in order to make your life easier: + +* `tf_ssh`: to ease SSH-ing into the virtual machines, reading the username and IP address to use from Terraform, as well as setting default SSH options. +* `tf_ansi`: to ease applying an Ansible playbook to a set of virtual machines, dynamically creating the inventory, as well as setting default SSH options. diff --git a/tools/provisioning/aws/README.md b/tools/provisioning/aws/README.md new file mode 100644 index 000000000..f4f018f97 --- /dev/null +++ b/tools/provisioning/aws/README.md @@ -0,0 +1,90 @@ +# Amazon Web Services + +## Introduction + +This project allows you to get hold of some machine on Amazon Web Services. +You can then use these machines as is or run various Ansible playbooks from `../config_management` to set up Weave Net, Kubernetes, etc. + +## Setup + +* Log in [weaveworks.signin.aws.amazon.com/console](https://weaveworks.signin.aws.amazon.com/console/) with your account. + +* Go to `Services` > `IAM` > `Users` > Click on your username > `Security credentials` > `Create access key`. + Your access key and secret key will appear on the screen. Set these as environment variables: + +``` +export AWS_ACCESS_KEY_ID= +export AWS_SECRET_ACCESS_KEY= +``` + +* Go to `Services` > `EC2` > Select the availability zone you want to use (see top right corner, e.g. `us-east-1`) > `Import Key Pair`. + Enter your SSH public key and the name for it, and click `Import`. + Set the path to your private key as an environment variable: + +``` +export TF_VAR_aws_public_key_name= +export TF_VAR_aws_private_key_path="$HOME/.ssh/id_rsa" +``` + +* Set your current IP address as an environment variable: + +``` +export TF_VAR_client_ip=$(curl -s -X GET http://checkip.amazonaws.com/) +``` + + or pass it as a Terraform variable: + +``` +$ terraform -var 'client_ip=$(curl -s -X GET http://checkip.amazonaws.com/)' +``` + +### Bash aliases + +You can set the above variables temporarily in your current shell, permanently in your `~/.bashrc` file, or define aliases to activate/deactivate them at will with one single command by adding the below to your `~/.bashrc` file: + +``` +function _aws_on() { + export AWS_ACCESS_KEY_ID="" # Replace with appropriate value. + export AWS_SECRET_ACCESS_KEY="" # Replace with appropriate value. + export TF_VAR_aws_public_key_name="" # Replace with appropriate value. + export TF_VAR_aws_private_key_path="$HOME/.ssh/id_rsa" # Replace with appropriate value. +} +alias _aws_on='_aws_on' +function _aws_off() { + unset AWS_ACCESS_KEY_ID + unset AWS_SECRET_ACCESS_KEY + unset TF_VAR_aws_public_key_name + unset TF_VAR_aws_private_key_path +} +alias _aws_off='_aws_off' +``` + +N.B.: + +* sourcing `../setup.sh` defines aliases called `aws_on` and `aws_off`, similarly to the above (however, notice no `_` in front of the name, as opposed to the ones above); +* `../setup.sh`'s `aws_on` alias needs the `SECRET_KEY` environment variable to be set in order to decrypt sensitive information. + +## Usage + +* Create the machine: `terraform apply` +* Show the machine's status: `terraform show` +* Stop and destroy the machine: `terraform destroy` +* SSH into the newly-created machine: + +``` +$ ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no `terraform output username`@`terraform output public_ips` +# N.B.: the default username will differ depending on the AMI/OS you installed, e.g. ubuntu for Ubuntu, ec2-user for Red Hat, etc. +``` + +or + +``` +source ../setup.sh +tf_ssh 1 # Or the nth machine, if multiple VMs are provisioned. +``` + +## Resources + +* [https://www.terraform.io/docs/providers/aws/](https://www.terraform.io/docs/providers/aws/) +* [https://www.terraform.io/docs/providers/aws/r/instance.html](https://www.terraform.io/docs/providers/aws/r/instance.html) +* [Terraform variables](https://www.terraform.io/intro/getting-started/variables.html) diff --git a/tools/provisioning/aws/main.tf b/tools/provisioning/aws/main.tf new file mode 100755 index 000000000..f4be8c345 --- /dev/null +++ b/tools/provisioning/aws/main.tf @@ -0,0 +1,137 @@ +# Specify the provider and access details +provider "aws" { + # Access key, secret key and region are sourced from environment variables or input arguments -- see README.md + region = "${var.aws_dc}" +} + +resource "aws_security_group" "allow_ssh" { + name = "${var.name}_allow_ssh" + description = "AWS security group to allow SSH-ing onto AWS EC2 instances (created using Terraform)." + + # Open TCP port for SSH: + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["${var.client_ip}/32"] + } + + tags { + Name = "${var.name}_allow_ssh" + App = "${var.app}" + CreatedBy = "terraform" + } +} + +resource "aws_security_group" "allow_docker" { + name = "${var.name}_allow_docker" + description = "AWS security group to allow communication with Docker on AWS EC2 instances (created using Terraform)." + + # Open TCP port for Docker: + ingress { + from_port = 2375 + to_port = 2375 + protocol = "tcp" + cidr_blocks = ["${var.client_ip}/32"] + } + + tags { + Name = "${var.name}_allow_docker" + App = "${var.app}" + CreatedBy = "terraform" + } +} + +resource "aws_security_group" "allow_weave" { + name = "${var.name}_allow_weave" + description = "AWS security group to allow communication with Weave on AWS EC2 instances (created using Terraform)." + + # Open TCP port for Weave: + ingress { + from_port = 12375 + to_port = 12375 + protocol = "tcp" + cidr_blocks = ["${var.client_ip}/32"] + } + + tags { + Name = "${var.name}_allow_weave" + App = "${var.app}" + CreatedBy = "terraform" + } +} + +resource "aws_security_group" "allow_private_ingress" { + name = "${var.name}_allow_private_ingress" + description = "AWS security group to allow all private ingress traffic on AWS EC2 instances (created using Terraform)." + + # Full inbound local network access on both TCP and UDP + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["${var.aws_vpc_cidr_block}"] + } + + tags { + Name = "${var.name}_allow_private_ingress" + App = "${var.app}" + CreatedBy = "terraform" + } +} + +resource "aws_security_group" "allow_all_egress" { + name = "${var.name}_allow_all_egress" + description = "AWS security group to allow all egress traffic on AWS EC2 instances (created using Terraform)." + + # Full outbound internet access on both TCP and UDP + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags { + Name = "${var.name}_allow_all_egress" + App = "${var.app}" + CreatedBy = "terraform" + } +} + +resource "aws_instance" "tf_test_vm" { + instance_type = "${var.aws_size}" + count = "${var.num_hosts}" + + # Lookup the correct AMI based on the region we specified + ami = "${lookup(var.aws_amis, var.aws_dc)}" + + key_name = "${var.aws_public_key_name}" + + security_groups = [ + "${aws_security_group.allow_ssh.name}", + "${aws_security_group.allow_docker.name}", + "${aws_security_group.allow_weave.name}", + "${aws_security_group.allow_private_ingress.name}", + "${aws_security_group.allow_all_egress.name}", + ] + + # Wait for machine to be SSH-able: + provisioner "remote-exec" { + inline = ["exit"] + + connection { + type = "ssh" + + # Lookup the correct username based on the AMI we specified + user = "${lookup(var.aws_usernames, "${lookup(var.aws_amis, var.aws_dc)}")}" + private_key = "${file("${var.aws_private_key_path}")}" + } + } + + tags { + Name = "${var.name}-${count.index}" + App = "${var.app}" + CreatedBy = "terraform" + } +} diff --git a/tools/provisioning/aws/outputs.tf b/tools/provisioning/aws/outputs.tf new file mode 100755 index 000000000..587986b8b --- /dev/null +++ b/tools/provisioning/aws/outputs.tf @@ -0,0 +1,54 @@ +output "username" { + value = "${lookup(var.aws_usernames, "${lookup(var.aws_amis, var.aws_dc)}")}" +} + +output "public_ips" { + value = ["${aws_instance.tf_test_vm.*.public_ip}"] +} + +output "hostnames" { + value = "${join("\n", + "${formatlist("%v.%v.%v", + aws_instance.tf_test_vm.*.tags.Name, + aws_instance.tf_test_vm.*.availability_zone, + var.app + )}" + )}" +} + +# /etc/hosts file for the Droplets: +output "private_etc_hosts" { + value = "${join("\n", + "${formatlist("%v %v.%v.%v", + aws_instance.tf_test_vm.*.private_ip, + aws_instance.tf_test_vm.*.tags.Name, + aws_instance.tf_test_vm.*.availability_zone, + var.app + )}" + )}" +} + +# /etc/hosts file for the client: +output "public_etc_hosts" { + value = "${join("\n", + "${formatlist("%v %v.%v.%v", + aws_instance.tf_test_vm.*.public_ip, + aws_instance.tf_test_vm.*.tags.Name, + aws_instance.tf_test_vm.*.availability_zone, + var.app + )}" + )}" +} + +output "ansible_inventory" { + value = "${format("[all]\n%s", join("\n", + "${formatlist("%v private_ip=%v", + aws_instance.tf_test_vm.*.public_ip, + aws_instance.tf_test_vm.*.private_ip, + )}" + ))}" +} + +output "private_key_path" { + value = "${var.aws_private_key_path}" +} diff --git a/tools/provisioning/aws/variables.tf b/tools/provisioning/aws/variables.tf new file mode 100755 index 000000000..6abaff0be --- /dev/null +++ b/tools/provisioning/aws/variables.tf @@ -0,0 +1,63 @@ +variable "client_ip" { + description = "IP address of the client machine" +} + +variable "app" { + description = "Name of the application using the created EC2 instance(s)." + default = "default" +} + +variable "name" { + description = "Name of the EC2 instance(s)." + default = "test" +} + +variable "num_hosts" { + description = "Number of EC2 instance(s)." + default = 1 +} + +variable "aws_vpc_cidr_block" { + description = "AWS VPC CIDR block to use to attribute private IP addresses." + default = "172.31.0.0/16" +} + +variable "aws_public_key_name" { + description = "Name of the SSH keypair to use in AWS." +} + +variable "aws_private_key_path" { + description = "Path to file containing private key" + default = "~/.ssh/id_rsa" +} + +variable "aws_dc" { + description = "The AWS region to create things in." + default = "us-east-1" +} + +variable "aws_amis" { + default = { + # Ubuntu Server 16.04 LTS (HVM), SSD Volume Type: + "us-east-1" = "ami-40d28157" + "eu-west-2" = "ami-23d0da47" + + # Red Hat Enterprise Linux 7.3 (HVM), SSD Volume Type: + + #"us-east-1" = "ami-b63769a1" + } +} + +variable "aws_usernames" { + description = "User to SSH as into the AWS instance." + + default = { + "ami-40d28157" = "ubuntu" # Ubuntu Server 16.04 LTS (HVM) + "ami-b63769a1" = "ec2-user" # Red Hat Enterprise Linux 7.3 (HVM) + } +} + +variable "aws_size" { + description = "AWS' selected machine size" + default = "t2.medium" # Instance with 2 cores & 4 GB memory +} diff --git a/tools/provisioning/do/README.md b/tools/provisioning/do/README.md new file mode 100755 index 000000000..d958f18d8 --- /dev/null +++ b/tools/provisioning/do/README.md @@ -0,0 +1,98 @@ +# Digital Ocean + +## Introduction + +This project allows you to get hold of some machine on Digital Ocean. +You can then use these machines as is or run various Ansible playbooks from `../config_management` to set up Weave Net, Kubernetes, etc. + +## Setup + +* Log in [cloud.digitalocean.com](https://cloud.digitalocean.com) with your account. + +* Go to `Settings` > `Security` > `SSH keys` > `Add SSH Key`. + Enter your SSH public key and the name for it, and click `Add SSH Key`. + Set the path to your private key as an environment variable: + +``` +export DIGITALOCEAN_SSH_KEY_NAME= +export TF_VAR_do_private_key_path="$HOME/.ssh/id_rsa" +``` + +* Go to `API` > `Tokens` > `Personal access tokens` > `Generate New Token` + Enter your token name and click `Generate Token` to get your 64-characters-long API token. + Set these as environment variables: + +``` +export DIGITALOCEAN_TOKEN_NAME="" +export DIGITALOCEAN_TOKEN= +``` + +* Run the following command to get the Digital Ocean ID for your SSH public key (e.g. `1234567`) and set it as an environment variable: + +``` +$ export TF_VAR_do_public_key_id=$(curl -s -X GET -H "Content-Type: application/json" \ +-H "Authorization: Bearer $DIGITALOCEAN_TOKEN" "https://api.digitalocean.com/v2/account/keys" \ +| jq -c --arg key_name "$DIGITALOCEAN_SSH_KEY_NAME" '.ssh_keys | .[] | select(.name==$key_name) | .id') +``` + + or pass it as a Terraform variable: + +``` +$ terraform \ +-var 'do_private_key_path=' \ +-var 'do_public_key_id=' +``` + +### Bash aliases + +You can set the above variables temporarily in your current shell, permanently in your `~/.bashrc` file, or define aliases to activate/deactivate them at will with one single command by adding the below to your `~/.bashrc` file: + +``` +function _do_on() { + export DIGITALOCEAN_TOKEN_NAME="" # Replace with appropriate value. + export DIGITALOCEAN_TOKEN= # Replace with appropriate value. + export DIGITALOCEAN_SSH_KEY_NAME="" # Replace with appropriate value. + export TF_VAR_do_private_key_path="$HOME/.ssh/id_rsa" # Replace with appropriate value. + export TF_VAR_do_public_key_path="$HOME/.ssh/id_rsa.pub" # Replace with appropriate value. + export TF_VAR_do_public_key_id= # Replace with appropriate value. +} +alias _do_on='_do_on' +function _do_off() { + unset DIGITALOCEAN_TOKEN_NAME + unset DIGITALOCEAN_TOKEN + unset DIGITALOCEAN_SSH_KEY_NAME + unset TF_VAR_do_private_key_path + unset TF_VAR_do_public_key_path + unset TF_VAR_do_public_key_id +} +alias _do_off='_do_off' +``` + +N.B.: + +* sourcing `../setup.sh` defines aliases called `do_on` and `do_off`, similarly to the above (however, notice no `_` in front of the name, as opposed to the ones above); +* `../setup.sh`'s `do_on` alias needs the `SECRET_KEY` environment variable to be set in order to decrypt sensitive information. + +## Usage + +* Create the machine: `terraform apply` +* Show the machine's status: `terraform show` +* Stop and destroy the machine: `terraform destroy` +* SSH into the newly-created machine: + +``` +$ ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no `terraform output username`@`terraform output public_ips` +``` + +or + +``` +source ../setup.sh +tf_ssh 1 # Or the nth machine, if multiple VMs are provisioned. +``` + +## Resources + +* [https://www.terraform.io/docs/providers/do/](https://www.terraform.io/docs/providers/do/) +* [https://www.terraform.io/docs/providers/do/r/droplet.html](https://www.terraform.io/docs/providers/do/r/droplet.html) +* [Terraform variables](https://www.terraform.io/intro/getting-started/variables.html) diff --git a/tools/provisioning/do/main.tf b/tools/provisioning/do/main.tf new file mode 100755 index 000000000..834cdf7d6 --- /dev/null +++ b/tools/provisioning/do/main.tf @@ -0,0 +1,42 @@ +provider "digitalocean" { + # See README.md for setup instructions. +} + +# Tags to label and organize droplets: +resource "digitalocean_tag" "name" { + name = "${var.name}" +} + +resource "digitalocean_tag" "app" { + name = "${var.app}" +} + +resource "digitalocean_tag" "terraform" { + name = "terraform" +} + +resource "digitalocean_droplet" "tf_test_vm" { + ssh_keys = ["${var.do_public_key_id}"] + image = "${var.do_os}" + region = "${var.do_dc}" + size = "${var.do_size}" + name = "${var.name}-${count.index}" + count = "${var.num_hosts}" + + tags = [ + "${var.app}", + "${var.name}", + "terraform", + ] + + # Wait for machine to be SSH-able: + provisioner "remote-exec" { + inline = ["exit"] + + connection { + type = "ssh" + user = "${var.do_username}" + private_key = "${file("${var.do_private_key_path}")}" + } + } +} diff --git a/tools/provisioning/do/outputs.tf b/tools/provisioning/do/outputs.tf new file mode 100755 index 000000000..5f0ff455b --- /dev/null +++ b/tools/provisioning/do/outputs.tf @@ -0,0 +1,57 @@ +output "username" { + value = "${var.do_username}" +} + +output "public_ips" { + value = ["${digitalocean_droplet.tf_test_vm.*.ipv4_address}"] +} + +output "hostnames" { + value = "${join("\n", + "${formatlist("%v.%v.%v", + digitalocean_droplet.tf_test_vm.*.name, + digitalocean_droplet.tf_test_vm.*.region, + var.app + )}" + )}" +} + +# /etc/hosts file for the Droplets: +# N.B.: by default Digital Ocean droplets only have public IPs, but in order to +# be consistent with other providers' recipes, we provide an output to generate +# an /etc/hosts file on the Droplets, even though it is using public IPs only. +output "private_etc_hosts" { + value = "${join("\n", + "${formatlist("%v %v.%v.%v", + digitalocean_droplet.tf_test_vm.*.ipv4_address, + digitalocean_droplet.tf_test_vm.*.name, + digitalocean_droplet.tf_test_vm.*.region, + var.app + )}" + )}" +} + +# /etc/hosts file for the client: +output "public_etc_hosts" { + value = "${join("\n", + "${formatlist("%v %v.%v.%v", + digitalocean_droplet.tf_test_vm.*.ipv4_address, + digitalocean_droplet.tf_test_vm.*.name, + digitalocean_droplet.tf_test_vm.*.region, + var.app + )}" + )}" +} + +output "ansible_inventory" { + value = "${format("[all]\n%s", join("\n", + "${formatlist("%v private_ip=%v", + digitalocean_droplet.tf_test_vm.*.ipv4_address, + digitalocean_droplet.tf_test_vm.*.ipv4_address + )}" + ))}" +} + +output "private_key_path" { + value = "${var.do_private_key_path}" +} diff --git a/tools/provisioning/do/variables.tf b/tools/provisioning/do/variables.tf new file mode 100755 index 000000000..6f7f40ed5 --- /dev/null +++ b/tools/provisioning/do/variables.tf @@ -0,0 +1,185 @@ +variable "client_ip" { + description = "IP address of the client machine" +} + +variable "app" { + description = "Name of the application using the created droplet(s)." + default = "default" +} + +variable "name" { + description = "Name of the droplet(s)." + default = "test" +} + +variable "num_hosts" { + description = "Number of droplet(s)." + default = 1 +} + +variable "do_private_key_path" { + description = "Digital Ocean SSH private key path" + default = "~/.ssh/id_rsa" +} + +variable "do_public_key_id" { + description = "Digital Ocean ID for your SSH public key" + + # You can retrieve it and set it as an environment variable this way: + + # $ export TF_VAR_do_public_key_id=$(curl -s -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" "https://api.digitalocean.com/v2/account/keys" | jq -c --arg key_name "$DIGITALOCEAN_SSH_KEY_NAME" '.ssh_keys | .[] | select(.name==$key_name) | .id') +} + +variable "do_username" { + description = "Digital Ocean SSH username" + default = "root" +} + +variable "do_os" { + description = "Digital Ocean OS" + default = "ubuntu-16-04-x64" +} + +# curl -s -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" "https://api.digitalocean.com/v2/images?page=1&per_page=999999" | jq ".images | .[] | .slug" | grep -P "ubuntu|coreos|centos" | grep -v alpha | grep -v beta +# "ubuntu-16-04-x32" +# "ubuntu-16-04-x64" +# "ubuntu-16-10-x32" +# "ubuntu-16-10-x64" +# "ubuntu-14-04-x32" +# "ubuntu-14-04-x64" +# "ubuntu-12-04-x64" +# "ubuntu-12-04-x32" +# "coreos-stable" +# "centos-6-5-x32" +# "centos-6-5-x64" +# "centos-7-0-x64" +# "centos-7-x64" +# "centos-6-x64" +# "centos-6-x32" +# "centos-5-x64" +# "centos-5-x32" + +# Digital Ocean datacenters +# See also: +# $ curl -s -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" "https://api.digitalocean.com/v2/regions" | jq -c ".regions | .[] | .slug" | sort -u + +variable "do_dc_ams2" { + description = "Digital Ocean Amsterdam Datacenter 2" + default = "ams2" +} + +variable "do_dc_ams3" { + description = "Digital Ocean Amsterdam Datacenter 3" + default = "ams3" +} + +variable "do_dc_blr1" { + description = "Digital Ocean Bangalore Datacenter 1" + default = "blr1" +} + +variable "do_dc_fra1" { + description = "Digital Ocean Frankfurt Datacenter 1" + default = "fra1" +} + +variable "do_dc_lon1" { + description = "Digital Ocean London Datacenter 1" + default = "lon1" +} + +variable "do_dc_nyc1" { + description = "Digital Ocean New York Datacenter 1" + default = "nyc1" +} + +variable "do_dc_nyc2" { + description = "Digital Ocean New York Datacenter 2" + default = "nyc2" +} + +variable "do_dc_nyc3" { + description = "Digital Ocean New York Datacenter 3" + default = "nyc3" +} + +variable "do_dc_sfo1" { + description = "Digital Ocean San Francisco Datacenter 1" + default = "sfo1" +} + +variable "do_dc_sfo2" { + description = "Digital Ocean San Francisco Datacenter 2" + default = "sfo2" +} + +variable "do_dc_sgp1" { + description = "Digital Ocean Singapore Datacenter 1" + default = "sgp1" +} + +variable "do_dc_tor1" { + description = "Digital Ocean Toronto Datacenter 1" + default = "tor1" +} + +variable "do_dc" { + description = "Digital Ocean's selected datacenter" + default = "lon1" +} + +variable "do_size" { + description = "Digital Ocean's selected machine size" + default = "4gb" +} + +# Digital Ocean sizes + + +# See also: + + +# $ curl -s -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" "https://api.digitalocean.com/v2/sizes" | jq -c ".sizes | .[] | .slug" + + +# "512mb" + + +# "1gb" + + +# "2gb" + + +# "4gb" + + +# "8gb" + + +# "16gb" + + +# "m-16gb" + + +# "32gb" + + +# "m-32gb" + + +# "48gb" + + +# "m-64gb" + + +# "64gb" + + +# "m-128gb" + + +# "m-224gb" + diff --git a/tools/provisioning/gcp/README.md b/tools/provisioning/gcp/README.md new file mode 100755 index 000000000..b2d6622c8 --- /dev/null +++ b/tools/provisioning/gcp/README.md @@ -0,0 +1,126 @@ +# Google Cloud Platform + +## Introduction + +This project allows you to get hold of some machine on Google Cloud Platform. +You can then use these machines as is or run various Ansible playbooks from `../config_management` to set up Weave Net, Kubernetes, etc. + +## Setup + +* Log in [console.cloud.google.com](https://console.cloud.google.com) with your Google account. + +* Go to `API Manager` > `Credentials` > `Create credentials` > `Service account key`, + in `Service account`, select `Compute Engine default service account`, + in `Key type`, select `JSON`, and then click `Create`. + +* This will download a JSON file to your machine. Place this file wherever you want and then create the following environment variables: + +``` +$ export GOOGLE_CREDENTIALS_FILE="path/to/your.json" +$ export GOOGLE_CREDENTIALS=$(cat "$GOOGLE_CREDENTIALS_FILE") +``` + +* Go to `Compute Engine` > `Metadata` > `SSH keys` and add your username and SSH public key; + or + set it up using `gcloud compute project-info add-metadata --metadata-from-file sshKeys=~/.ssh/id_rsa.pub`. + If you used your default SSH key (i.e. `~/.ssh/id_rsa.pub`), then you do not have anything to do. + Otherwise, you will have to either define the below environment variable: + +``` +$ export TF_VAR_gcp_public_key_path= +$ export TF_VAR_gcp_private_key_path= +``` + + or to pass these as Terraform variables: + +``` +$ terraform \ +-var 'gcp_public_key_path=' \ +-var 'gcp_private_key_path=' +``` + +* Set the username in your public key as an environment variable. + This will be used as the username of the Linux account created on the machine, which you will need to SSH into it later on. + + N.B.: + * GCP already has the username set from the SSH public key you uploaded in the previous step. + * If your username is an email address, e.g. `name@domain.com`, then GCP uses `name` as the username. + +``` +export TF_VAR_gcp_username= +``` + +* Set your current IP address as an environment variable: + +``` +export TF_VAR_client_ip=$(curl -s -X GET http://checkip.amazonaws.com/) +``` + + or pass it as a Terraform variable: + +``` +$ terraform -var 'client_ip=$(curl -s -X GET http://checkip.amazonaws.com/)' +``` + +* Set your project as an environment variable: + +``` +export TF_VAR_gcp_project=weave-net-tests +``` + + or pass it as a Terraform variable: + +``` +$ terraform -var 'gcp_project=weave-net-tests' +``` + +### Bash aliases + +You can set the above variables temporarily in your current shell, permanently in your `~/.bashrc` file, or define aliases to activate/deactivate them at will with one single command by adding the below to your `~/.bashrc` file: + +``` +function _gcp_on() { + export GOOGLE_CREDENTIALS_FILE="&authuser=1 + region = "${var.gcp_region}" + + project = "${var.gcp_project}" +} + +resource "google_compute_instance" "tf_test_vm" { + name = "${var.name}-${count.index}" + machine_type = "${var.gcp_size}" + zone = "${var.gcp_zone}" + count = "${var.num_hosts}" + + disk { + image = "${var.gcp_image}" + } + + tags = [ + "${var.app}", + "${var.name}", + "terraform", + ] + + network_interface { + network = "${var.gcp_network}" + + access_config { + // Ephemeral IP + } + } + + metadata { + ssh-keys = "${var.gcp_username}:${file("${var.gcp_public_key_path}")}" + } + + # Wait for machine to be SSH-able: + provisioner "remote-exec" { + inline = ["exit"] + + connection { + type = "ssh" + user = "${var.gcp_username}" + private_key = "${file("${var.gcp_private_key_path}")}" + } + } +} + +resource "google_compute_firewall" "fw-allow-docker-and-weave" { + name = "${var.name}-allow-docker-and-weave" + network = "${var.gcp_network}" + target_tags = ["${var.name}"] + + allow { + protocol = "tcp" + ports = ["2375", "12375"] + } + + source_ranges = ["${var.client_ip}"] +} + +# Required for FastDP crypto in Weave Net: +resource "google_compute_firewall" "fw-allow-esp" { + name = "${var.name}-allow-esp" + network = "${var.gcp_network}" + target_tags = ["${var.name}"] + + allow { + protocol = "esp" + } + + source_ranges = ["${var.gcp_network_global_cidr}"] +} diff --git a/tools/provisioning/gcp/outputs.tf b/tools/provisioning/gcp/outputs.tf new file mode 100755 index 000000000..9aa1e33e8 --- /dev/null +++ b/tools/provisioning/gcp/outputs.tf @@ -0,0 +1,66 @@ +output "username" { + value = "${var.gcp_username}" +} + +output "public_ips" { + value = ["${google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.assigned_nat_ip}"] +} + +output "hostnames" { + value = "${join("\n", + "${formatlist("%v.%v.%v", + google_compute_instance.tf_test_vm.*.name, + google_compute_instance.tf_test_vm.*.zone, + var.app + )}" + )}" +} + +# /etc/hosts file for the Compute Engine instances: +output "private_etc_hosts" { + value = "${join("\n", + "${formatlist("%v %v.%v.%v", + google_compute_instance.tf_test_vm.*.network_interface.0.address, + google_compute_instance.tf_test_vm.*.name, + google_compute_instance.tf_test_vm.*.zone, + var.app + )}" + )}" +} + +# /etc/hosts file for the client: +output "public_etc_hosts" { + value = "${join("\n", + "${formatlist("%v %v.%v.%v", + google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.assigned_nat_ip, + google_compute_instance.tf_test_vm.*.name, + google_compute_instance.tf_test_vm.*.zone, + var.app + )}" + )}" +} + +output "ansible_inventory" { + value = "${format("[all]\n%s", join("\n", + "${formatlist("%v private_ip=%v", + google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.assigned_nat_ip, + google_compute_instance.tf_test_vm.*.network_interface.0.address + )}" + ))}" +} + +output "private_key_path" { + value = "${var.gcp_private_key_path}" +} + +output "instances_names" { + value = ["${google_compute_instance.tf_test_vm.*.name}"] +} + +output "image" { + value = "${var.gcp_image}" +} + +output "zone" { + value = "${var.gcp_zone}" +} diff --git a/tools/provisioning/gcp/variables.tf b/tools/provisioning/gcp/variables.tf new file mode 100755 index 000000000..1ee48bd08 --- /dev/null +++ b/tools/provisioning/gcp/variables.tf @@ -0,0 +1,71 @@ +variable "gcp_username" { + description = "Google Cloud Platform SSH username" +} + +variable "app" { + description = "Name of the application using the created Compute Engine instance(s)." + default = "default" +} + +variable "name" { + description = "Name of the Compute Engine instance(s)." + default = "test" +} + +variable "num_hosts" { + description = "Number of Compute Engine instance(s)." + default = 1 +} + +variable "client_ip" { + description = "IP address of the client machine" +} + +variable "gcp_public_key_path" { + description = "Path to file containing public key" + default = "~/.ssh/id_rsa.pub" +} + +variable "gcp_private_key_path" { + description = "Path to file containing private key" + default = "~/.ssh/id_rsa" +} + +variable "gcp_project" { + description = "Google Cloud Platform project" + default = "weave-net-tests" +} + +variable "gcp_image" { + # See also: https://cloud.google.com/compute/docs/images + description = "Google Cloud Platform OS" + default = "ubuntu-os-cloud/ubuntu-1604-lts" +} + +variable "gcp_size" { + # See also: + # $ gcloud compute machine-types list + description = "Google Cloud Platform's selected machine size" + + default = "n1-standard-1" +} + +variable "gcp_region" { + description = "Google Cloud Platform's selected region" + default = "us-central1" +} + +variable "gcp_zone" { + description = "Google Cloud Platform's selected zone" + default = "us-central1-a" +} + +variable "gcp_network" { + description = "Google Cloud Platform's selected network" + default = "test" +} + +variable "gcp_network_global_cidr" { + description = "CIDR covering all regions for the selected Google Cloud Platform network" + default = "10.128.0.0/9" +} diff --git a/tools/provisioning/setup.sh b/tools/provisioning/setup.sh new file mode 100755 index 000000000..afe5914d3 --- /dev/null +++ b/tools/provisioning/setup.sh @@ -0,0 +1,353 @@ +#!/bin/bash +# +# Description: +# Helper functions to programmatically provision (e.g. for CIT). +# Aliases on these functions are also created so that this script can be +# sourced in your shell, in your ~/.bashrc file, etc. and directly called. +# +# Usage: +# Source this file and call the relevant functions. +# + +function ssh_public_key() { + echo -e "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDZBgLQts30PYXEMJnCU21QC+1ZE0Sv/Ry48Au3nYXn1KNoW/7C2qQ3KO2ZnpZRHCstFiU8QIlB9edi0cgcAoDWBkCiFBZEORxMvohWtrRQzf+x59o48lVjA/Fn7G+9hmavhLaDf6Qe7OhH8XUshNtnIQIUvNEWXKE75k32wUbuF8ibhJNpOOYKL4tVXK6IIKg6jR88BwGKPY/NZCl/HbhjnDJY0zCU1pZSprN6o/S953y/XXVozkh1772fCNeu4USfbt0oZOEJ57j6EWwEYIJhoeAEMAoD8ELt/bc/5iex8cuarM4Uib2JHO6WPWbBQ0NlrARIOKLrxkjjfGWarOLWBAgvwQn5zLg1pKb7aI4+jbA+ZSrII5B2HuYE9MDlU8NPL4pHrRfapGLkG/Fe9zNPvScXh+9iSWfD6G5ZoISutjiJO/iVYN0QSuj9QEIj9tl20czFz3Dhnq4sPPl5hoLunyQfajY7C/ipv6ilJyrEc0V6Z9FdPhpEI+HOgJr2vDQTFscQuyfWuzGJDZf6zPdZWo2pBql9E7piARuNAjakylGar/ebkCgfy28XQoDbDT0P0VYp+E8W5EYacx+zc5MuNhRTvbsO12fydT8V61MtA78wM/b0059feph+0zTykEHk670mYVoE3erZX+U1/BVBLSV9QzopO6/Pgx2ryriJfQ== weaveworks-cit" +} + +function decrypt() { + if [ -z "$1" ]; then + echo >&2 "Failed to decode and decrypt $2: no secret key was provided." + return 1 + fi + echo "$3" | openssl base64 -d | openssl enc -d -aes256 -pass "pass:$1" +} + +function ssh_private_key() { + # The private key has been AES256-encrypted and then Base64-encoded using the following command: + # $ openssl enc -in /tmp/weaveworks_cit_id_rsa -e -aes256 -pass stdin | openssl base64 > /tmp/weaveworks_cit_id_rsa.aes.b64 + # The below command does the reverse, i.e. base64-decode and AES-decrypt the file, and prints it to stdout. + # N.B.: Ask the password to Marc, or otherwise re-generate the SSH key using: + # $ ssh-keygen -t rsa -b 4096 -C "weaveworks-cit" + decrypt "$1" "SSH private key" "$( + cat <&2 "Failed to decode and decrypt SSH private key: no secret key was provided." + return 1 + fi + local ssh_private_key_path="$HOME/.ssh/weaveworks_cit_id_rsa" + [ -e "$ssh_private_key_path" ] && rm -f "$ssh_private_key_path" + ssh_private_key "$1" >"$ssh_private_key_path" + chmod 400 "$ssh_private_key_path" + echo "$ssh_private_key_path" +} + +function gcp_credentials() { + # The below GCP service account JSON credentials have been AES256-encrypted and then Base64-encoded using the following command: + # $ openssl enc -in ~/.ssh/weaveworks-cit.json -e -aes256 -pass stdin | openssl base64 > /tmp/weaveworks-cit.json.aes.b64 + # The below command does the reverse, i.e. base64-decode and AES-decrypt the file, and prints it to stdout. + # N.B.: Ask the password to Marc, or otherwise re-generate the credentials for GCP, as per ../tools/provisioning/gcp/README.md. + decrypt "$1" "JSON credentials" "$( + cat <&2 "Failed to configure for Digital Ocean: no value for the SECRET_KEY environment variable." + return 1 + fi + + # SSH public key: + export TF_VAR_do_public_key_path="$HOME/.ssh/weaveworks_cit_id_rsa.pub" + ssh_public_key >"$TF_VAR_do_public_key_path" + export DIGITALOCEAN_SSH_KEY_NAME="weaveworks-cit" + export TF_VAR_do_public_key_id=5228799 + + # SSH private key: + export TF_VAR_do_private_key_path=$(set_up_ssh_private_key "$SECRET_KEY") + + # API token: + # The below Digital Ocean token has been AES256-encrypted and then Base64-encoded using the following command: + # $ openssl enc -in /tmp/digital_ocean_token.txt -e -aes256 -pass stdin | openssl base64 > /tmp/digital_ocean_token.txt.aes.b64 + # The below command does the reverse, i.e. base64-decode and AES-decrypt the file, and prints it to stdout. + # N.B.: Ask the password to Marc, or otherwise re-generate the token for Digital Ocean, as per ../tools/provisioning/do/README.md. + export DIGITALOCEAN_TOKEN=$(decrypt "$SECRET_KEY" "Digital Ocean token" "U2FsdGVkX1/Gq5Rj9dDDraME8xK30JOyJ9dhfQzPBaaePJHqDPIG6of71DdJW0UyFUyRtbRflCPaZ8Um1pDJpU5LoNWQk4uCApC8+xciltT73uQtttLBG8FqgFBvYIHS") + export DIGITALOCEAN_TOKEN_NAME="weaveworks-cit" + export TF_VAR_client_ip=$(curl -s -X GET http://checkip.amazonaws.com/) +} +alias do_on='do_on' + +function do_off() { + unset TF_VAR_do_public_key_path + unset DIGITALOCEAN_SSH_KEY_NAME + unset TF_VAR_do_public_key_id + unset TF_VAR_do_private_key_path + unset DIGITALOCEAN_TOKEN + unset DIGITALOCEAN_TOKEN_NAME + unset TF_VAR_client_ip +} +alias do_off='do_off' + +# shellcheck disable=2155 +function gcp_on() { + # Set up everything required to run tests on GCP. + # Steps from ../tools/provisioning/gcp/README.md have been followed. + # All sensitive files have been encrypted, see respective functions. + if [ -z "$SECRET_KEY" ]; then + echo >&2 "Failed to configure for Google Cloud Platform: no value for the SECRET_KEY environment variable." + return 1 + fi + + # SSH public key and SSH username: + export TF_VAR_gcp_public_key_path="$HOME/.ssh/weaveworks_cit_id_rsa.pub" + ssh_public_key >"$TF_VAR_gcp_public_key_path" + export TF_VAR_gcp_username=$(cut -d' ' -f3 "$TF_VAR_gcp_public_key_path" | cut -d'@' -f1) + + # SSH private key: + export TF_VAR_gcp_private_key_path=$(set_up_ssh_private_key "$SECRET_KEY") + + # JSON credentials: + export GOOGLE_CREDENTIALS_FILE="$HOME/.ssh/weaveworks-cit.json" + [ -e "$GOOGLE_CREDENTIALS_FILE" ] && rm -f "$GOOGLE_CREDENTIALS_FILE" + gcp_credentials "$SECRET_KEY" >"$GOOGLE_CREDENTIALS_FILE" + chmod 400 "$GOOGLE_CREDENTIALS_FILE" + export GOOGLE_CREDENTIALS=$(cat "$GOOGLE_CREDENTIALS_FILE") + + export TF_VAR_client_ip=$(curl -s -X GET http://checkip.amazonaws.com/) + export TF_VAR_gcp_project="${PROJECT:-"weave-net-tests"}" + # shellcheck disable=2015 + [ -z "$PROJECT" ] && echo >&2 "WARNING: no value provided for PROJECT environment variable: defaulted it to $TF_VAR_gcp_project." || true +} +alias gcp_on='gcp_on' + +function gcp_off() { + unset TF_VAR_gcp_public_key_path + unset TF_VAR_gcp_username + unset TF_VAR_gcp_private_key_path + unset GOOGLE_CREDENTIALS_FILE + unset GOOGLE_CREDENTIALS + unset TF_VAR_client_ip + unset TF_VAR_gcp_project +} +alias gcp_off='gcp_off' + +# shellcheck disable=2155 +function aws_on() { + # Set up everything required to run tests on Amazon Web Services. + # Steps from ../tools/provisioning/aws/README.md have been followed. + # All sensitive files have been encrypted, see respective functions. + if [ -z "$SECRET_KEY" ]; then + echo >&2 "Failed to configure for Amazon Web Services: no value for the SECRET_KEY environment variable." + return 1 + fi + + # SSH public key: + export TF_VAR_aws_public_key_name="weaveworks_cit_id_rsa" + + # SSH private key: + export TF_VAR_aws_private_key_path=$(set_up_ssh_private_key "$SECRET_KEY") + + # The below AWS access key ID and secret access key have been AES256-encrypted and then Base64-encoded using the following commands: + # $ openssl enc -in /tmp/aws_access_key_id.txt -e -aes256 -pass stdin | openssl base64 > /tmp/aws_access_key_id.txt.aes.b64 + # $ openssl enc -in /tmp/aws_secret_access_key.txt -e -aes256 -pass stdin | openssl base64 > /tmp/aws_secret_access_key.txt.aes.b64 + # The below commands do the reverse, i.e. base64-decode and AES-decrypt the encrypted and encoded strings, and print it to stdout. + # N.B.: Ask the password to Marc, or otherwise re-generate the AWS access key ID and secret access key, as per ../tools/provisioning/aws/README.md. + export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-$(decrypt "$SECRET_KEY" "AWS access key ID" "U2FsdGVkX1+MLsvG53ZVSmFhjvQtWio0pXQpG5Ua+5JaoizuZKtJZFJxrSSyx0jb")} + export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-$(decrypt "$SECRET_KEY" "AWS secret access key" "U2FsdGVkX1+VNjgWv5iGKRqBYP7o8MpOIMnd3BOYPiEho1Mjosx++9CknaZJbeR59vSuz4UdgTS6ezH2dnq2Fw==")} + export TF_VAR_client_ip=$(curl -s -X GET http://checkip.amazonaws.com/) +} +alias aws_on='aws_on' + +function aws_off() { + unset TF_VAR_aws_public_key_name + unset TF_VAR_aws_private_key_path + unset AWS_ACCESS_KEY_ID + unset AWS_SECRET_ACCESS_KEY + unset TF_VAR_client_ip +} +alias aws_off='aws_off' + +function tf_ssh_usage() { + cat >&2 <<-EOF +ERROR: $1 + +Usage: + $ tf_ssh [OPTION]... +Examples: + $ tf_ssh 1 + $ tf_ssh 1 -o LogLevel VERBOSE +Available machines: +EOF + cat -n >&2 <<<"$(terraform output public_etc_hosts)" +} + +# shellcheck disable=SC2155 +function tf_ssh() { + [ -z "$1" ] && tf_ssh_usage "No host ID provided." && return 1 + local ip="$(sed "$1q;d" <<<"$(terraform output public_etc_hosts)" | cut -d ' ' -f 1)" + shift # Drop the first argument, corresponding to the machine ID, to allow passing other arguments to SSH using "$@" -- see below. + [ -z "$ip" ] && tf_ssh_usage "Invalid host ID provided." && return 1 + # shellcheck disable=SC2029 + ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "$@" "$(terraform output username)@$ip" +} +alias tf_ssh='tf_ssh' + +function tf_ansi_usage() { + cat >&2 <<-EOF +ERROR: $1 + +Usage: + $ tf_ansi [OPTION]... +Examples: + $ tf_ansi setup_weave-net_dev + $ tf_ansi 1 + $ tf_ansi 1 -vvv +Available playbooks: +EOF + cat -n >&2 <<<"$(for file in "$(dirname "${BASH_SOURCE[0]}")"/../../config_management/*.yml; do basename "$file" | sed 's/.yml//'; done)" +} + +# shellcheck disable=SC2155,SC2064 +function tf_ansi() { + [ -z "$1" ] && tf_ansi_usage "No Ansible playbook provided." && return 1 + local id="$1" + shift # Drop the first argument to allow passing other arguments to Ansible using "$@" -- see below. + if [[ "$id" =~ ^[0-9]+$ ]]; then + local playbooks=(../../config_management/*.yml) + local path="${playbooks[(($id-1))]}" # Select the ith entry in the list of playbooks (0-based). + else + local path="$(dirname "${BASH_SOURCE[0]}")/../../config_management/$id.yml" + fi + local inventory="$(mktemp /tmp/ansible_inventory_XXX)" + trap 'rm -f $inventory' SIGINT SIGTERM RETURN + echo -e "$(terraform output ansible_inventory)" >"$inventory" + [ ! -r "$path" ] && tf_ansi_usage "Ansible playbook not found: $path" && return 1 + ansible-playbook "$@" -u "$(terraform output username)" -i "$inventory" --ssh-extra-args="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" "$path" +} +alias tf_ansi='tf_ansi' diff --git a/tools/runner/runner.go b/tools/runner/runner.go index c92ac6b5b..38e5a62c9 100644 --- a/tools/runner/runner.go +++ b/tools/runner/runner.go @@ -15,7 +15,7 @@ import ( "time" "github.com/mgutz/ansi" - "github.com/weaveworks/docker/pkg/mflag" + "github.com/weaveworks/common/mflag" ) const ( @@ -148,9 +148,10 @@ func updateScheduler(test string, duration float64) { func getSchedule(tests []string) ([]string, error) { var ( + userName = os.Getenv("CIRCLE_PROJECT_USERNAME") project = os.Getenv("CIRCLE_PROJECT_REPONAME") buildNum = os.Getenv("CIRCLE_BUILD_NUM") - testRun = project + "-integration-" + buildNum + testRun = userName + "-" + project + "-integration-" + buildNum shardCount = os.Getenv("CIRCLE_NODE_TOTAL") shardID = os.Getenv("CIRCLE_NODE_INDEX") requestBody = &bytes.Buffer{} diff --git a/tools/sched b/tools/sched index cf47773e5..72eeee652 100755 --- a/tools/sched +++ b/tools/sched @@ -1,20 +1,20 @@ #!/usr/bin/python -import sys, string, json, urllib +import sys, string, urllib import requests import optparse def test_time(target, test_name, runtime): r = requests.post(target + "/record/%s/%f" % (urllib.quote(test_name, safe=""), runtime)) - print r.text + print r.text.encode('utf-8') assert r.status_code == 204 def test_sched(target, test_run, shard_count, shard_id): - tests = json.dumps({'tests': string.split(sys.stdin.read())}) - r = requests.post(target + "/schedule/%s/%d/%d" % (test_run, shard_count, shard_id), data=tests) + tests = {'tests': string.split(sys.stdin.read())} + r = requests.post(target + "/schedule/%s/%d/%d" % (test_run, shard_count, shard_id), json=tests) assert r.status_code == 200 result = r.json() for test in sorted(result['tests']): - print test + print test.encode('utf-8') def usage(): print "%s (--target=...) " % sys.argv[0] diff --git a/tools/socks/main.go b/tools/socks/main.go index 83a214980..7cd8c7086 100644 --- a/tools/socks/main.go +++ b/tools/socks/main.go @@ -9,8 +9,8 @@ import ( "text/template" socks5 "github.com/armon/go-socks5" - "github.com/weaveworks/docker/pkg/mflag" - "github.com/weaveworks/weave/common/mflagext" + "github.com/weaveworks/common/mflag" + "github.com/weaveworks/common/mflagext" "golang.org/x/net/context" ) diff --git a/tools/test b/tools/test index 6a970884e..1772b1bd6 100755 --- a/tools/test +++ b/tools/test @@ -3,12 +3,14 @@ set -e DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GO_TEST_ARGS=(-tags netgo -cpu 4 -timeout 8m) SLOW= -NO_GO_GET= +NO_GO_GET=true +TAGS= +PARALLEL= +TIMEOUT=1m usage() { - echo "$0 [-slow] [-in-container foo]" + echo "$0 [-slow] [-in-container foo] [-netgo] [-(no-)go-get] [-timeout 1m]" } while [ $# -gt 0 ]; do @@ -21,6 +23,22 @@ while [ $# -gt 0 ]; do NO_GO_GET=true shift 1 ;; + "-go-get") + NO_GO_GET= + shift 1 + ;; + "-netgo") + TAGS="-tags netgo" + shift 1 + ;; + "-p") + PARALLEL=true + shift 1 + ;; + "-timeout") + TIMEOUT=$2 + shift 2 + ;; *) usage exit 2 @@ -28,6 +46,8 @@ while [ $# -gt 0 ]; do esac done +GO_TEST_ARGS=($TAGS -cpu 4 -timeout $TIMEOUT) + if [ -n "$SLOW" ] || [ -n "$CIRCLECI" ]; then SLOW=true fi @@ -60,7 +80,7 @@ fi # If running on circle, use the scheduler to work out what tests to run on what shard if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ] && [ -x "$DIR/sched" ]; then PREFIX=$(go list -e ./ | sed -e 's/\//-/g') - TESTDIRS=($(echo "${TESTDIRS[@]}" | "$DIR/sched" sched "$PREFIX-$CIRCLE_BUILD_NUM" "$CIRCLE_NODE_TOTAL" "$CIRCLE_NODE_INDEX")) + TESTDIRS=($(echo "${TESTDIRS[@]}" | "$DIR/sched" sched "$PREFIX-$CIRCLE_PROJECT_USERNAME-$CIRCLE_PROJECT_REPONAME-$CIRCLE_BUILD_NUM" "$CIRCLE_NODE_TOTAL" "$CIRCLE_NODE_INDEX")) echo "${TESTDIRS[@]}" fi @@ -69,33 +89,49 @@ PACKAGE_BASE=$(go list -e ./) # Speed up the tests by compiling and installing their dependencies first. go test -i "${GO_TEST_ARGS[@]}" "${TESTDIRS[@]}" -for dir in "${TESTDIRS[@]}"; do +run_test() { + local dir=$1 if [ -z "$NO_GO_GET" ]; then - go get -t -tags netgo "$dir" + go get -t "$TAGS" "$dir" fi - GO_TEST_ARGS_RUN=("${GO_TEST_ARGS[@]}") + local GO_TEST_ARGS_RUN=("${GO_TEST_ARGS[@]}") if [ -n "$SLOW" ]; then + local COVERPKGS COVERPKGS=$( ( go list "$dir" go list -f '{{join .Deps "\n"}}' "$dir" | grep -v "vendor" | grep "^$PACKAGE_BASE/" ) | paste -s -d, -) + local output output=$(mktemp "$coverdir/unit.XXXXXXXXXX") - GO_TEST_ARGS_RUN=("${GO_TEST_ARGS[@]}" -coverprofile=$output -coverpkg=$COVERPKGS) + local GO_TEST_ARGS_RUN=("${GO_TEST_ARGS[@]}" -coverprofile=$output -coverpkg=$COVERPKGS) fi + local START START=$(date +%s) if ! go test "${GO_TEST_ARGS_RUN[@]}" "$dir"; then fail=1 fi - RUNTIME=$(($(date +%s) - START)) + local RUNTIME=$(($(date +%s) - START)) # Report test runtime when running on circle, to help scheduler if [ -n "$CIRCLECI" ] && [ -z "$NO_SCHEDULER" ] && [ -x "$DIR/sched" ]; then "$DIR/sched" time "$dir" "$RUNTIME" fi +} + +for dir in "${TESTDIRS[@]}"; do + if [ -n "$PARALLEL" ]; then + run_test "$dir" & + else + run_test "$dir" + fi done +if [ -n "$PARALLEL" ]; then + wait +fi + if [ -n "$SLOW" ] && [ -z "$COVERDIR" ]; then go get github.com/weaveworks/tools/cover cover "$coverdir"/* >profile.cov