From efdc4fcfa9fcbfa0a62a503f9f211ce93e05b6a9 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Wed, 26 Aug 2020 12:36:48 +0200 Subject: [PATCH 01/11] bump versions --- compose/kube-router-k8s-control-plane/docker-compose.yaml | 8 ++++---- compose/simple-k8s-control-plane/docker-compose.yaml | 8 ++++---- prepare-vms/lib/commands.sh | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/compose/kube-router-k8s-control-plane/docker-compose.yaml b/compose/kube-router-k8s-control-plane/docker-compose.yaml index cb207715..791bb13c 100644 --- a/compose/kube-router-k8s-control-plane/docker-compose.yaml +++ b/compose/kube-router-k8s-control-plane/docker-compose.yaml @@ -9,21 +9,21 @@ services: etcd: network_mode: "service:pause" - image: k8s.gcr.io/etcd:3.4.3 + image: k8s.gcr.io/etcd:3.4.9 command: etcd kube-apiserver: network_mode: "service:pause" - image: k8s.gcr.io/hyperkube:v1.17.2 + image: k8s.gcr.io/hyperkube:v1.18.8 command: kube-apiserver --etcd-servers http://127.0.0.1:2379 --address 0.0.0.0 --disable-admission-plugins=ServiceAccount --allow-privileged kube-controller-manager: network_mode: "service:pause" - image: k8s.gcr.io/hyperkube:v1.17.2 + image: k8s.gcr.io/hyperkube:v1.18.8 command: kube-controller-manager --master http://localhost:8080 --allocate-node-cidrs --cluster-cidr=10.CLUSTER.0.0/16 "Edit the CLUSTER placeholder first. Then, remove this line.": kube-scheduler: network_mode: "service:pause" - image: k8s.gcr.io/hyperkube:v1.17.2 + image: k8s.gcr.io/hyperkube:v1.18.8 command: kube-scheduler --master http://localhost:8080 diff --git a/compose/simple-k8s-control-plane/docker-compose.yaml b/compose/simple-k8s-control-plane/docker-compose.yaml index 7c370e7b..aabc0746 100644 --- a/compose/simple-k8s-control-plane/docker-compose.yaml +++ b/compose/simple-k8s-control-plane/docker-compose.yaml @@ -9,20 +9,20 @@ services: etcd: network_mode: "service:pause" - image: k8s.gcr.io/etcd:3.4.3 + image: k8s.gcr.io/etcd:3.4.9 command: etcd kube-apiserver: network_mode: "service:pause" - image: k8s.gcr.io/hyperkube:v1.17.2 + image: k8s.gcr.io/hyperkube:v1.18.8 command: kube-apiserver --etcd-servers http://127.0.0.1:2379 --address 0.0.0.0 --disable-admission-plugins=ServiceAccount kube-controller-manager: network_mode: "service:pause" - image: k8s.gcr.io/hyperkube:v1.17.2 + image: k8s.gcr.io/hyperkube:v1.18.8 command: kube-controller-manager --master http://localhost:8080 kube-scheduler: network_mode: "service:pause" - image: k8s.gcr.io/hyperkube:v1.17.2 + image: k8s.gcr.io/hyperkube:v1.18.8 command: kube-scheduler --master http://localhost:8080 diff --git a/prepare-vms/lib/commands.sh b/prepare-vms/lib/commands.sh index 5c0cc631..61d587b3 100644 --- a/prepare-vms/lib/commands.sh +++ b/prepare-vms/lib/commands.sh @@ -139,19 +139,19 @@ _cmd_kubebins() { cd /usr/local/bin if ! [ -x etcd ]; then ##VERSION## - curl -L https://github.com/etcd-io/etcd/releases/download/v3.4.3/etcd-v3.4.3-linux-amd64.tar.gz \ + curl -L https://github.com/etcd-io/etcd/releases/download/v3.4.9/etcd-v3.4.9-linux-amd64.tar.gz \ | sudo tar --strip-components=1 --wildcards -zx '*/etcd' '*/etcdctl' fi if ! [ -x hyperkube ]; then ##VERSION## - curl -L https://dl.k8s.io/v1.17.2/kubernetes-server-linux-amd64.tar.gz \ + curl -L https://dl.k8s.io/v1.18.8/kubernetes-server-linux-amd64.tar.gz \ | sudo tar --strip-components=3 -zx \ kubernetes/server/bin/kube{ctl,let,-proxy,-apiserver,-scheduler,-controller-manager} fi sudo mkdir -p /opt/cni/bin cd /opt/cni/bin if ! [ -x bridge ]; then - curl -L https://github.com/containernetworking/plugins/releases/download/v0.7.6/cni-plugins-amd64-v0.7.6.tgz \ + curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.6/cni-plugins-linux-amd64-v0.8.6.tgz \ | sudo tar -zx fi " From b4c03781141d459968aac49fa0c4c78d4bf1d512 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Mon, 31 Aug 2020 16:31:59 +0200 Subject: [PATCH 02/11] Add ips command to output tab-separated addresses --- prepare-vms/lib/commands.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/prepare-vms/lib/commands.sh b/prepare-vms/lib/commands.sh index 61d587b3..1a190b7e 100644 --- a/prepare-vms/lib/commands.sh +++ b/prepare-vms/lib/commands.sh @@ -322,6 +322,22 @@ _cmd_ids() { aws_get_instance_ids_by_client_token $TAG } +_cmd ips "Show the IP addresses for a given tag" +_cmd_ips() { + TAG=$1 + need_tag $TAG + + SETTINGS=tags/$TAG/settings.yaml + CLUSTERSIZE=$(awk '/^clustersize:/ {print $2}' $SETTINGS) + while true; do + for I in $(seq $CLUSTERSIZE); do + read ip || return 0 + printf "%s\t" "$ip" + done + printf "\n" + done < tags/$TAG/ips.txt +} + _cmd list "List available groups for a given infrastructure" _cmd_list() { need_infra $1 From d73e597198c5afc2aec40030088eecfeecbb2d4e Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Wed, 2 Sep 2020 13:04:44 +0200 Subject: [PATCH 03/11] Small updates for Kubernetes 1.19 --- slides/k8s/authn-authz.md | 2 +- slides/k8s/daemonset.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/slides/k8s/authn-authz.md b/slides/k8s/authn-authz.md index 94e0bfc1..b6ae85ca 100644 --- a/slides/k8s/authn-authz.md +++ b/slides/k8s/authn-authz.md @@ -118,7 +118,7 @@ - [HTTP basic auth](https://en.wikipedia.org/wiki/Basic_access_authentication) - (carrying user and password in an HTTP header) + (carrying user and password in an HTTP header; [deprecated since Kubernetes 1.19](https://github.com/kubernetes/kubernetes/pull/89069)) - Authentication proxy diff --git a/slides/k8s/daemonset.md b/slides/k8s/daemonset.md index 6c664294..b6dc760f 100644 --- a/slides/k8s/daemonset.md +++ b/slides/k8s/daemonset.md @@ -52,7 +52,7 @@ -- Unfortunately, as of Kubernetes 1.17, the CLI cannot create daemon sets +- Unfortunately, as of Kubernetes 1.19, the CLI cannot create daemon sets -- From 35320837e5b1dbc86ad062cfa23d79266a4b33c9 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Wed, 2 Sep 2020 13:21:21 +0200 Subject: [PATCH 04/11] Add info about immutable configmaps and secrets --- slides/k8s/configuration.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/slides/k8s/configuration.md b/slides/k8s/configuration.md index db5bbc09..3284e4a7 100644 --- a/slides/k8s/configuration.md +++ b/slides/k8s/configuration.md @@ -536,6 +536,8 @@ spec: --- +class: extra-details + ## Differences between configmaps and secrets - Secrets are base64-encoded when shown with `kubectl get secrets -o yaml` @@ -550,6 +552,29 @@ spec: (since they are two different kinds of resources) +--- + +class: extra-details + +## Immutable ConfigMaps and Secrets + +- Since Kubernetes 1.19, it is possible to mark a ConfigMap or Secret as *immutable* + + ```bash + kubectl patch configmap xyz --patch='{"immutable": true}' + ``` + +- This brings performance improvements when using lots of ConfigMaps and Secrets + + (lots = tens of thousands) + +- Once a ConfigMap or Secret has been marked as immutable: + + - its content cannot be changed anymore + - the `immutable` field can't be changed back either + - the only way to change it is to delete and re-create it + - Pods using it will have to be re-created as well + ??? :EN:- Managing application configuration From d98fcbce87ba77a256c0abc6fda7fd1de83de278 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Wed, 2 Sep 2020 13:34:11 +0200 Subject: [PATCH 05/11] Update Ingress to 1.19 --- slides/k8s/ingress.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/slides/k8s/ingress.md b/slides/k8s/ingress.md index 43c6c810..d41da6c5 100644 --- a/slides/k8s/ingress.md +++ b/slides/k8s/ingress.md @@ -485,6 +485,8 @@ spec: --- +class: extra-details + ## Using multiple ingress controllers - You can have multiple ingress controllers active simultaneously @@ -495,11 +497,13 @@ spec: (e.g. one for internal, another for external traffic) -- The `kubernetes.io/ingress.class` annotation can be used to tell which one to use +- To indicate which ingress controller should be used by a given Ingress resouce: -- It's OK if multiple ingress controllers configure the same resource + - before Kubernetes 1.18, use the `kubernetes.io/ingress.class` annotation - (it just means that the service will be accessible through multiple paths) + - since Kubernetes 1.18, use the `ingressClassName` field +
+ (which should refer to an existing `IngressClass` resource) --- @@ -535,9 +539,9 @@ spec: - [ingress.kubernetes.io/rewrite-target: /](https://github.com/kubernetes/contrib/tree/master/ingress/controllers/nginx/examples/rewrite) -- This should eventually stabilize +- The Ingress spec stabilized in Kubernetes 1.19 ... - (remember that ingresses are currently `apiVersion: networking.k8s.io/v1beta1`) + ... without specifying these features! 😭 --- From fee0be7f0964be499a4015535a897c52ed56c07b Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Wed, 2 Sep 2020 16:48:19 +0200 Subject: [PATCH 06/11] Update 'kubectl create deployment' for 1.19 --- slides/k8s/batch-jobs.md | 4 ---- slides/k8s/kubectl-run.md | 15 ++++++++++++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/slides/k8s/batch-jobs.md b/slides/k8s/batch-jobs.md index c8b765ed..569cfd6e 100644 --- a/slides/k8s/batch-jobs.md +++ b/slides/k8s/batch-jobs.md @@ -176,12 +176,8 @@ class: extra-details - can't express parallelism or completions of Jobs - - can't express Pods with multiple containers - - can't express healthchecks, resource limits - - etc. - - `kubectl create` and `kubectl run` are *helpers* that generate YAML manifests - If we write these manifests ourselves, we can use all features and options diff --git a/slides/k8s/kubectl-run.md b/slides/k8s/kubectl-run.md index 7f1ce849..8991cbc0 100644 --- a/slides/k8s/kubectl-run.md +++ b/slides/k8s/kubectl-run.md @@ -296,7 +296,7 @@ class: extra-details - When using `kubectl create deployment`, we cannot indicate the command to execute - (at least, not in Kubernetes 1.18) + (at least, not in Kubernetes 1.18; but that changed in Kubernetes 1.19) - We can: @@ -344,6 +344,19 @@ class: extra-details --- +class: extra-details + +## In Kubernetes 1.19 + +- Since Kubernetes 1.19, we can specify the command to run + +- The command must be passed after two dashes: + ```bash + kubectl create deployment pingpong --image=alpine -- ping 127.1 + ``` + +--- + ## Viewing container output - Let's use the `kubectl logs` command From a02e7429ad1e4b52bf7a09c454ff42109c66cb7b Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Mon, 7 Sep 2020 12:49:08 +0200 Subject: [PATCH 07/11] Add note about httpenv arch --- slides/k8s/kubectlexpose.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/slides/k8s/kubectlexpose.md b/slides/k8s/kubectlexpose.md index 0d3bec75..35b0538d 100644 --- a/slides/k8s/kubectlexpose.md +++ b/slides/k8s/kubectlexpose.md @@ -149,6 +149,28 @@ --- +class: extra-details + +## Supporting other CPU architectures + +- The `jpetazzo/httpenv` image is currently only available for `x86_64` + + (the "classic" Intel 64 bits architecture found on most PCs and Macs) + +- That image won't work on other architectures + + (e.g. Raspberry Pi or other ARM-based machines) + +- Note that Docker supports [multi-arch](https://www.docker.com/blog/multi-arch-build-and-images-the-simple-way/) images + + (so *technically* we could make it work across multiple architectures) + +- If you want to build `httpenv` for your own platform, here is the source: + + https://github.com/jpetazzo/httpenv + +--- + ## Creating a deployment for our HTTP server - We will create a deployment with `kubectl create deployment` From 29c6d2876a8a7db36c31cf7a3199b06add57ba73 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Tue, 8 Sep 2020 11:08:58 +0200 Subject: [PATCH 08/11] Reword sanity check --- slides/k8s/lastwords.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/slides/k8s/lastwords.md b/slides/k8s/lastwords.md index 0d65ca00..d98d9583 100644 --- a/slides/k8s/lastwords.md +++ b/slides/k8s/lastwords.md @@ -191,6 +191,8 @@ are a few tools that can help us.* ## Developer experience +*These questions constitute a quick "smoke test" for our strategy:* + - How do we on-board a new developer? - What do they need to install to get a dev stack? @@ -199,8 +201,6 @@ are a few tools that can help us.* - How does someone add a component to a stack? -*These questions are good "sanity checks" to validate our strategy!* - --- ## Some guidelines From 14679999bedefc4e305d914c0b3427c6ad57473b Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Wed, 9 Sep 2020 19:37:15 +0200 Subject: [PATCH 09/11] Big refactor of deployment script Add support for OVHcloud, Hetzner; refactor Scaleway support --- prepare-vms/infra/example.openstack-cli | 24 +++++ ...example.openstack => example.openstack-tf} | 5 +- prepare-vms/infra/hetzner | 5 + prepare-vms/infra/scaleway | 4 - prepare-vms/lib/commands.sh | 96 +++++-------------- prepare-vms/lib/infra/aws.sh | 43 ++++++++- prepare-vms/lib/infra/hetzner.sh | 50 ++++++++++ prepare-vms/lib/infra/openstack-cli.sh | 48 ++++++++++ .../infra/{openstack.sh => openstack-tf.sh} | 0 prepare-vms/lib/infra/scaleway.sh | 34 +++---- prepare-vms/lib/infra/unimplemented.sh | 23 +++++ prepare-vms/lib/postprep.py | 2 +- prepare-vms/lib/pssh.sh | 8 +- prepare-vms/terraform/machines.tf | 2 +- prepare-vms/workshopctl | 1 - 15 files changed, 235 insertions(+), 110 deletions(-) create mode 100644 prepare-vms/infra/example.openstack-cli rename prepare-vms/infra/{example.openstack => example.openstack-tf} (84%) create mode 100644 prepare-vms/infra/hetzner create mode 100644 prepare-vms/lib/infra/hetzner.sh create mode 100644 prepare-vms/lib/infra/openstack-cli.sh rename prepare-vms/lib/infra/{openstack.sh => openstack-tf.sh} (100%) create mode 100644 prepare-vms/lib/infra/unimplemented.sh diff --git a/prepare-vms/infra/example.openstack-cli b/prepare-vms/infra/example.openstack-cli new file mode 100644 index 00000000..d20f79d6 --- /dev/null +++ b/prepare-vms/infra/example.openstack-cli @@ -0,0 +1,24 @@ +INFRACLASS=openstack-cli + +# Copy that file to e.g. openstack or ovh, then customize it. +# Some Openstack providers (like OVHcloud) will let you download +# a file containing credentials. That's what you need to use. +# The file below contains some example values. +export OS_AUTH_URL=https://auth.cloud.ovh.net/v3/ +export OS_IDENTITY_API_VERSION=3 +export OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME:-"Default"} +export OS_PROJECT_DOMAIN_NAME=${OS_PROJECT_DOMAIN_NAME:-"Default"} +export OS_TENANT_ID=abcd1234 +export OS_TENANT_NAME="0123456" +export OS_USERNAME="user-xyz123" +export OS_PASSWORD=AbCd1234 +export OS_REGION_NAME="GRA7" + +# And then some values to indicate server type, image, etc. +# You can see available flavors with `openstack flavor list` +export OS_FLAVOR=s1-4 +# You can see available images with `openstack image list` +export OS_IMAGE=896c5f54-51dc-44f0-8c22-ce99ba7164df +# You can create a key with `openstack keypair create --public-key ~/.ssh/id_rsa.pub containertraining` +export OS_KEY=containertraining + diff --git a/prepare-vms/infra/example.openstack b/prepare-vms/infra/example.openstack-tf similarity index 84% rename from prepare-vms/infra/example.openstack rename to prepare-vms/infra/example.openstack-tf index a99c43af..bad23dfa 100644 --- a/prepare-vms/infra/example.openstack +++ b/prepare-vms/infra/example.openstack-tf @@ -1,4 +1,5 @@ -INFRACLASS=openstack +INFRACLASS=openstack-tf + # If you are using OpenStack, copy this file (e.g. to "openstack" or "enix") # and customize the variables below. export TF_VAR_user="jpetazzo" @@ -6,4 +7,4 @@ export TF_VAR_tenant="training" export TF_VAR_domain="Default" export TF_VAR_password="..." export TF_VAR_auth_url="https://api.r1.nxs.enix.io/v3" -export TF_VAR_flavor="GP1.S" \ No newline at end of file +export TF_VAR_flavor="GP1.S" diff --git a/prepare-vms/infra/hetzner b/prepare-vms/infra/hetzner new file mode 100644 index 00000000..77547446 --- /dev/null +++ b/prepare-vms/infra/hetzner @@ -0,0 +1,5 @@ +INFRACLASS=hetzner +if ! [ -f ~/.config/hcloud/cli.toml ]; then + warn "~/.config/hcloud/cli.toml not found." + warn "Make sure that the Hetzner CLI (hcloud) is installed and configured." +fi diff --git a/prepare-vms/infra/scaleway b/prepare-vms/infra/scaleway index 7615e024..53f232e7 100644 --- a/prepare-vms/infra/scaleway +++ b/prepare-vms/infra/scaleway @@ -1,5 +1 @@ INFRACLASS=scaleway -if ! [ -f ~/.config/scw/config.yaml ]; then - warn "~/.config/scw/config.yaml not found." - warn "Make sure that the scaleway CLI is installed and configured." -fi diff --git a/prepare-vms/lib/commands.sh b/prepare-vms/lib/commands.sh index 1a190b7e..9f90d656 100644 --- a/prepare-vms/lib/commands.sh +++ b/prepare-vms/lib/commands.sh @@ -73,6 +73,19 @@ _cmd_deploy() { apt-get update && apt-get install sudo -y" fi + # FIXME + # Special case for hetzner since it doesn't have an ubuntu user + #if [ "$INFRACLASS" = "hetzner" ]; then + # pssh -l root " + #[ -d /home/ubuntu ] || + # useradd ubuntu -m -s /bin/bash + #echo 'ubuntu ALL=(ALL:ALL) NOPASSWD:ALL' > /etc/sudoers.d/ubuntu + #[ -d /home/ubuntu/.ssh ] || + # install --owner=ubuntu --mode=700 --directory /home/ubuntu/.ssh + #[ -f /home/ubuntu/.ssh/authorized_keys ] || + # install --owner=ubuntu --mode=600 /root/.ssh/authorized_keys --target-directory /home/ubuntu/.ssh" + #fi + # Copy settings and install Python YAML parser pssh -I tee /tmp/settings.yaml /tmp/token && - sudo kubeadm init $EXTRA_KUBEADM --token \$(cat /tmp/token) --apiserver-cert-extra-sans \$(cat /tmp/ipv4) + sudo kubeadm init $EXTRA_KUBEADM --token \$(cat /tmp/token) --apiserver-cert-extra-sans \$(cat /tmp/ipv4) --ignore-preflight-errors=NumCPU fi" # Put kubeconfig in ubuntu's and docker's accounts @@ -224,13 +237,13 @@ _cmd_kube() { # Install kubectx and kubens pssh " [ -d kubectx ] || git clone https://github.com/ahmetb/kubectx && - sudo ln -sf /home/ubuntu/kubectx/kubectx /usr/local/bin/kctx && - sudo ln -sf /home/ubuntu/kubectx/kubens /usr/local/bin/kns && - sudo cp /home/ubuntu/kubectx/completion/*.bash /etc/bash_completion.d && + sudo ln -sf \$HOME/kubectx/kubectx /usr/local/bin/kctx && + sudo ln -sf \$HOME/kubectx/kubens /usr/local/bin/kns && + sudo cp \$HOME/kubectx/completion/*.bash /etc/bash_completion.d && [ -d kube-ps1 ] || git clone https://github.com/jonmosco/kube-ps1 && sudo -u docker sed -i s/docker-prompt/kube_ps1/ /home/docker/.bashrc && sudo -u docker tee -a /home/docker/.bashrc < webssh/known_hosts" + done | sudo tee /opt/webssh/known_hosts" pssh "cat >webssh.service </dev/null; then - aws ec2 import-key-pair --key-name $AWS_KEY_NAME \ - --public-key-material "$(ssh-add -L \ - | grep -i RSA \ - | head -n1 \ - | cut -d " " -f 1-2)" &>/dev/null - - if ! aws ec2 describe-key-pairs --key-name "$AWS_KEY_NAME" &>/dev/null; then - die "Somehow, importing the key didn't work. Make sure that 'ssh-add -l | grep RSA | head -n1' returns an RSA key?" - else - info "Imported new key $AWS_KEY_NAME." - fi - else - info "Using existing key $AWS_KEY_NAME." - fi -} diff --git a/prepare-vms/lib/infra/aws.sh b/prepare-vms/lib/infra/aws.sh index f7721003..a01459d1 100644 --- a/prepare-vms/lib/infra/aws.sh +++ b/prepare-vms/lib/infra/aws.sh @@ -1,9 +1,13 @@ +if ! command -v aws >/dev/null; then + warn "AWS CLI (aws) not found." +fi + infra_list() { aws_display_tags } infra_quotas() { - greet + aws_greet max_instances=$(aws ec2 describe-account-attributes \ --attribute-names max-instances \ @@ -21,10 +25,10 @@ infra_start() { COUNT=$1 # Print our AWS username, to ease the pain of credential-juggling - greet + aws_greet # Upload our SSH keys to AWS if needed, to be added to each VM's authorized_keys - key_name=$(sync_keys) + key_name=$(aws_sync_keys) AMI=$(aws_get_ami) # Retrieve the AWS image ID if [ -z "$AMI" ]; then @@ -61,7 +65,7 @@ infra_start() { aws_tag_instances $TAG $TAG # Wait until EC2 API tells us that the instances are running - wait_until_tag_is_running $TAG $COUNT + aws_wait_until_tag_is_running $TAG $COUNT aws_get_instance_ips_by_tag $TAG > tags/$TAG/ips.txt } @@ -98,7 +102,7 @@ infra_disableaddrchecks() { done } -wait_until_tag_is_running() { +aws_wait_until_tag_is_running() { max_retry=100 i=0 done_count=0 @@ -214,3 +218,32 @@ aws_get_ami() { ##VERSION## find_ubuntu_ami -r $AWS_DEFAULT_REGION -a amd64 -v 18.04 -t hvm:ebs -N -q } + +aws_greet() { + IAMUSER=$(aws iam get-user --query 'User.UserName') + info "Hello! You seem to be UNIX user $USER, and IAM user $IAMUSER." +} + +aws_sync_keys() { + # make sure ssh-add -l contains "RSA" + ssh-add -l | grep -q RSA \ + || die "The output of \`ssh-add -l\` doesn't contain 'RSA'. Start the agent, add your keys?" + + AWS_KEY_NAME=$(make_key_name) + info "Syncing keys... " + if ! aws ec2 describe-key-pairs --key-name "$AWS_KEY_NAME" &>/dev/null; then + aws ec2 import-key-pair --key-name $AWS_KEY_NAME \ + --public-key-material "$(ssh-add -L \ + | grep -i RSA \ + | head -n1 \ + | cut -d " " -f 1-2)" &>/dev/null + + if ! aws ec2 describe-key-pairs --key-name "$AWS_KEY_NAME" &>/dev/null; then + die "Somehow, importing the key didn't work. Make sure that 'ssh-add -l | grep RSA | head -n1' returns an RSA key?" + else + info "Imported new key $AWS_KEY_NAME." + fi + else + info "Using existing key $AWS_KEY_NAME." + fi +} diff --git a/prepare-vms/lib/infra/hetzner.sh b/prepare-vms/lib/infra/hetzner.sh new file mode 100644 index 00000000..7d61ff5a --- /dev/null +++ b/prepare-vms/lib/infra/hetzner.sh @@ -0,0 +1,50 @@ +if ! command -v hcloud >/dev/null; then + warn "Hetzner CLI (hcloud) not found." +fi +if ! [ -f ~/.config/hcloud/cli.toml ]; then + warn "~/.config/hcloud/cli.toml not found." +fi + +infra_start() { + COUNT=$1 + + HETZNER_INSTANCE_TYPE=${HETZNER_INSTANCE_TYPE-cx21} + HETZNER_DATACENTER=${HETZNER_DATACENTER-nbg1-dc3} + HETZNER_IMAGE=${HETZNER_IMAGE-168855} + + for I in $(seq 1 $COUNT); do + NAME=$(printf "%s-%03d" $TAG $I) + sep "Starting instance $I/$COUNT" + info " Datacenter: $HETZNER_DATACENTER" + info " Name: $NAME" + info " Instance type: $HETZNER_INSTANCE_TYPE" + hcloud server create \ + --type=${HETZNER_INSTANCE_TYPE} \ + --datacenter=${HETZNER_DATACENTER} \ + --image=${HETZNER_IMAGE} \ + --name=$NAME \ + --label=tag=$TAG \ + --ssh-key ~/.ssh/id_rsa.pub + done + + hetzner_get_ips_by_tag $TAG > tags/$TAG/ips.txt +} + +infra_stop() { + for ID in $(hetzner_get_ids_by_tag $TAG); do + info "Scheduling deletion of instance $ID..." + hcloud server delete $ID & + done + info "Waiting for deletion to complete..." + wait +} + +hetzner_get_ids_by_tag() { + TAG=$1 + hcloud server list --selector=tag=$TAG -o json | jq -r .[].name +} + +hetzner_get_ips_by_tag() { + TAG=$1 + hcloud server list --selector=tag=$TAG -o json | jq -r .[].public_net.ipv4.ip +} diff --git a/prepare-vms/lib/infra/openstack-cli.sh b/prepare-vms/lib/infra/openstack-cli.sh new file mode 100644 index 00000000..aa4fe7cd --- /dev/null +++ b/prepare-vms/lib/infra/openstack-cli.sh @@ -0,0 +1,48 @@ +infra_start() { + COUNT=$1 + + sep "Starting $COUNT instances" + info " Region: $OS_REGION_NAME" + info " User: $OS_USERNAME" + info " Flavor: $OS_FLAVOR" + info " Image: $OS_IMAGE" + openstack server create \ + --flavor $OS_FLAVOR \ + --image $OS_IMAGE \ + --key-name $OS_KEY \ + --min $COUNT --max $COUNT \ + --property workshopctl=$TAG \ + $TAG + + sep "Waiting for IP addresses to be available" + GOT=0 + while [ "$GOT" != "$COUNT" ]; do + echo "Got $GOT/$COUNT IP addresses." + oscli_get_ips_by_tag $TAG > tags/$TAG/ips.txt + GOT="$(wc -l < tags/$TAG/ips.txt)" + done + +} + +infra_stop() { + info "Counting instances..." + oscli_get_instances_json $TAG | + jq -r .[].Name | + wc -l + info "Deleting instances..." + oscli_get_instances_json $TAG | + jq -r .[].Name | + xargs -P10 -n1 openstack server delete + info "Done." +} + +oscli_get_instances_json() { + TAG=$1 + openstack server list -f json --name "${TAG}-[0-9]*" +} + +oscli_get_ips_by_tag() { + TAG=$1 + oscli_get_instances_json $TAG | + jq -r .[].Networks | cut -d= -f2 | cut -d, -f1 | grep . || true +} diff --git a/prepare-vms/lib/infra/openstack.sh b/prepare-vms/lib/infra/openstack-tf.sh similarity index 100% rename from prepare-vms/lib/infra/openstack.sh rename to prepare-vms/lib/infra/openstack-tf.sh diff --git a/prepare-vms/lib/infra/scaleway.sh b/prepare-vms/lib/infra/scaleway.sh index 9bd07ede..70b7f127 100644 --- a/prepare-vms/lib/infra/scaleway.sh +++ b/prepare-vms/lib/infra/scaleway.sh @@ -1,15 +1,13 @@ -infra_list() { - die "unimplemented" -} - -infra_quotas() { - die "unimplemented" -} +if ! command -v scw >/dev/null; then + warn "Scaleway CLI (scw) not found." +fi +if ! [ -f ~/.config/scw/config.yaml ]; then + warn "~/.config/scw/config.yaml not found." +fi infra_start() { COUNT=$1 - AWS_KEY_NAME=$(make_key_name) SCW_INSTANCE_TYPE=${SCW_INSTANCE_TYPE-DEV1-M} SCW_ZONE=${SCW_ZONE-fr-par-1} @@ -29,12 +27,12 @@ infra_start() { } infra_stop() { - for ID in $(scw_get_ids_by_tag $TAG); do - info "Scheduling deletion of instance $ID..." - scw instance server delete force-shutdown=true server-id=$ID & - done - info "Waiting for deletion to complete..." - wait + info "Counting instances..." + scw_get_ids_by_tag $TAG | wc -l + info "Deleting instances..." + scw_get_ids_by_tag $TAG | + xargs -n1 -P10 -I@@ \ + scw instance server delete force-shutdown=true server-id=@@ } scw_get_ids_by_tag() { @@ -46,11 +44,3 @@ scw_get_ips_by_tag() { TAG=$1 scw instance server list name=$TAG -o json | jq -r .[].public_ip.address } - -infra_opensg() { - die "unimplemented" -} - -infra_disableaddrchecks() { - die "unimplemented" -} diff --git a/prepare-vms/lib/infra/unimplemented.sh b/prepare-vms/lib/infra/unimplemented.sh new file mode 100644 index 00000000..c32e2356 --- /dev/null +++ b/prepare-vms/lib/infra/unimplemented.sh @@ -0,0 +1,23 @@ +infra_disableaddrchecks() { + die "unimplemented" +} + +infra_list() { + die "unimplemented" +} + +infra_opensg() { + die "unimplemented" +} + +infra_quotas() { + die "unimplemented" +} + +infra_start() { + die "unimplemented" +} + +infra_stop() { + die "unimplemented" +} diff --git a/prepare-vms/lib/postprep.py b/prepare-vms/lib/postprep.py index e60079bf..e14bd9c4 100755 --- a/prepare-vms/lib/postprep.py +++ b/prepare-vms/lib/postprep.py @@ -37,7 +37,7 @@ def system(cmd): td = str(t2-t1)[:5] f.write(bold("[{}] in {}s\n".format(retcode, td))) STEP += 1 - with open("/home/ubuntu/.bash_history", "a") as f: + with open(os.environ["HOME"] + "/.bash_history", "a") as f: f.write("{}\n".format(cmd)) if retcode != 0: msg = "The following command failed with exit code {}:\n".format(retcode) diff --git a/prepare-vms/lib/pssh.sh b/prepare-vms/lib/pssh.sh index abb94539..ca3bc639 100644 --- a/prepare-vms/lib/pssh.sh +++ b/prepare-vms/lib/pssh.sh @@ -18,7 +18,13 @@ pssh() { echo "[parallel-ssh] $@" export PSSH=$(which pssh || which parallel-ssh) - $PSSH -h $HOSTFILE -l ubuntu \ + if [ "$INFRACLASS" = hetzner ]; then + LOGIN=root + else + LOGIN=ubuntu + fi + + $PSSH -h $HOSTFILE -l $LOGIN \ --par 100 \ -O LogLevel=ERROR \ -O UserKnownHostsFile=/dev/null \ diff --git a/prepare-vms/terraform/machines.tf b/prepare-vms/terraform/machines.tf index 41ff96e7..78b9da6a 100644 --- a/prepare-vms/terraform/machines.tf +++ b/prepare-vms/terraform/machines.tf @@ -1,7 +1,7 @@ resource "openstack_compute_instance_v2" "machine" { count = "${var.count}" name = "${format("%s-%04d", "${var.prefix}", count.index+1)}" - image_name = "Ubuntu 16.04.5 (Xenial Xerus)" + image_name = "Ubuntu 18.04.4 20200324" flavor_name = "${var.flavor}" security_groups = ["${openstack_networking_secgroup_v2.full_access.name}"] key_pair = "${openstack_compute_keypair_v2.ssh_deploy_key.name}" diff --git a/prepare-vms/workshopctl b/prepare-vms/workshopctl index 870d3e06..9ebd6ceb 100755 --- a/prepare-vms/workshopctl +++ b/prepare-vms/workshopctl @@ -15,7 +15,6 @@ for lib in lib/*.sh; do done DEPENDENCIES=" - aws ssh curl jq From 2d4961fbd3940c1bbde007c8f59e4f33f40f030d Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Fri, 11 Sep 2020 15:13:24 +0200 Subject: [PATCH 10/11] Add fwdays slides --- slides/index.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/slides/index.yaml b/slides/index.yaml index aaa83c44..0f68e9e1 100644 --- a/slides/index.yaml +++ b/slides/index.yaml @@ -58,6 +58,7 @@ speaker: jpetazzo title: Intensive Docker Online Workshop attend: https://fwdays.com/en/event/intensive-docker-workshop + slides: https://2020-08-fwdays.container.training/ - date: [2020-09-12, 2020-09-13] country: www @@ -66,6 +67,7 @@ speaker: jpetazzo title: Kubernetes Intensive Online Workshop attend: https://fwdays.com/en/event/kubernetes-intensive-workshop + slides: https://2020-09-fwdays.container.training/ - date: [2020-07-07, 2020-07-09] country: www From 07457af6f7473f96409bb301181a1cbfd9c71c3a Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Fri, 11 Sep 2020 22:30:18 +0200 Subject: [PATCH 11/11] Update Consul section --- k8s/consul-1.yaml | 77 ++++++++++++++++++ k8s/{consul.yaml => consul-2.yaml} | 20 +++-- k8s/consul-3.yaml | 104 +++++++++++++++++++++++++ slides/k8s/local-persistent-volumes.md | 18 ++--- slides/k8s/statefulsets.md | 102 +++++++++++++++++++++--- 5 files changed, 292 insertions(+), 29 deletions(-) create mode 100644 k8s/consul-1.yaml rename k8s/{consul.yaml => consul-2.yaml} (78%) create mode 100644 k8s/consul-3.yaml diff --git a/k8s/consul-1.yaml b/k8s/consul-1.yaml new file mode 100644 index 00000000..f913201f --- /dev/null +++ b/k8s/consul-1.yaml @@ -0,0 +1,77 @@ +# Basic Consul cluster using Cloud Auto-Join. +# Caveats: +# - no actual persistence +# - scaling down to 1 will break the cluster +# - pods may be colocated +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: consul +rules: + - apiGroups: [""] + resources: + - pods + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: consul +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: consul +subjects: + - kind: ServiceAccount + name: consul +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: consul +--- +apiVersion: v1 +kind: Service +metadata: + name: consul +spec: + ports: + - port: 8500 + name: http + selector: + app: consul +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: consul +spec: + serviceName: consul + replicas: 3 + selector: + matchLabels: + app: consul + template: + metadata: + labels: + app: consul + spec: + serviceAccountName: consul + containers: + - name: consul + image: "consul:1.8" + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + args: + - "agent" + - "-bootstrap-expect=3" + - "-retry-join=provider=k8s label_selector=\"app=consul\" namespace=\"$(NAMESPACE)\"" + - "-client=0.0.0.0" + - "-data-dir=/consul/data" + - "-server" + - "-ui" diff --git a/k8s/consul.yaml b/k8s/consul-2.yaml similarity index 78% rename from k8s/consul.yaml rename to k8s/consul-2.yaml index d8452a0c..e683aacd 100644 --- a/k8s/consul.yaml +++ b/k8s/consul-2.yaml @@ -1,5 +1,9 @@ +# Better Consul cluster. +# There is still no actual persistence, but: +# - podAntiaffinity prevents pod colocation +# - clusters works when scaling down to 1 (thanks to lifecycle hook) apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole +kind: Role metadata: name: consul rules: @@ -11,17 +15,16 @@ rules: - list --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding +kind: RoleBinding metadata: name: consul roleRef: apiGroup: rbac.authorization.k8s.io - kind: ClusterRole + kind: Role name: consul subjects: - kind: ServiceAccount name: consul - namespace: default --- apiVersion: v1 kind: ServiceAccount @@ -68,11 +71,16 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: consul - image: "consul:1.6" + image: "consul:1.8" + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace args: - "agent" - "-bootstrap-expect=3" - - "-retry-join=provider=k8s label_selector=\"app=consul\"" + - "-retry-join=provider=k8s label_selector=\"app=consul\" namespace=\"$(NAMESPACE)\"" - "-client=0.0.0.0" - "-data-dir=/consul/data" - "-server" diff --git a/k8s/consul-3.yaml b/k8s/consul-3.yaml new file mode 100644 index 00000000..af62fe0e --- /dev/null +++ b/k8s/consul-3.yaml @@ -0,0 +1,104 @@ +# Even better Consul cluster. +# That one uses a volumeClaimTemplate to achieve true persistence. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: consul +rules: + - apiGroups: [""] + resources: + - pods + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: consul +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: consul +subjects: + - kind: ServiceAccount + name: consul +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: consul +--- +apiVersion: v1 +kind: Service +metadata: + name: consul +spec: + ports: + - port: 8500 + name: http + selector: + app: consul +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: consul +spec: + serviceName: consul + replicas: 3 + selector: + matchLabels: + app: consul + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + template: + metadata: + labels: + app: consul + spec: + serviceAccountName: consul + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - persistentconsul + topologyKey: kubernetes.io/hostname + terminationGracePeriodSeconds: 10 + containers: + - name: consul + image: "consul:1.8" + volumeMounts: + - name: data + mountPath: /consul/data + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + args: + - "agent" + - "-bootstrap-expect=3" + - "-retry-join=provider=k8s label_selector=\"app=consul\" namespace=\"$(NAMESPACE)\"" + - "-client=0.0.0.0" + - "-data-dir=/consul/data" + - "-server" + - "-ui" + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - consul leave diff --git a/slides/k8s/local-persistent-volumes.md b/slides/k8s/local-persistent-volumes.md index fda28eae..5910b575 100644 --- a/slides/k8s/local-persistent-volumes.md +++ b/slides/k8s/local-persistent-volumes.md @@ -58,7 +58,7 @@ ## Deploying Consul -- We will use a slightly different YAML file +- Let's use a new manifest for our Consul cluster - The only differences between that file and the previous one are: @@ -66,15 +66,11 @@ - the corresponding `volumeMounts` in the Pod spec - - the label `consul` has been changed to `persistentconsul` -
- (to avoid conflicts with the other Stateful Set) - .exercise[ - Apply the persistent Consul YAML file: ```bash - kubectl apply -f ~/container.training/k8s/persistent-consul.yaml + kubectl apply -f ~/container.training/k8s/consul-3.yaml ``` ] @@ -97,7 +93,7 @@ kubectl get pv ``` -- The Pod `persistentconsul-0` is not scheduled yet: +- The Pod `consul-0` is not scheduled yet: ```bash kubectl get pods -o wide ``` @@ -112,9 +108,9 @@ - In a Stateful Set, the Pods are started one by one -- `persistentconsul-1` won't be created until `persistentconsul-0` is running +- `consul-1` won't be created until `consul-0` is running -- `persistentconsul-0` has a dependency on an unbound Persistent Volume Claim +- `consul-0` has a dependency on an unbound Persistent Volume Claim - The scheduler won't schedule the Pod until the PVC is bound @@ -152,7 +148,7 @@ - Once a PVC is bound, its pod can start normally -- Once the pod `persistentconsul-0` has started, `persistentconsul-1` can be created, etc. +- Once the pod `consul-0` has started, `consul-1` can be created, etc. - Eventually, our Consul cluster is up, and backend by "persistent" volumes @@ -160,7 +156,7 @@ - Check that our Consul clusters has 3 members indeed: ```bash - kubectl exec persistentconsul-0 -- consul members + kubectl exec consul-0 -- consul members ``` ] diff --git a/slides/k8s/statefulsets.md b/slides/k8s/statefulsets.md index e1a1feb1..84f35018 100644 --- a/slides/k8s/statefulsets.md +++ b/slides/k8s/statefulsets.md @@ -218,7 +218,9 @@ consul agent -data-dir=/consul/data -client=0.0.0.0 -server -ui \ - Replace X.X.X.X and Y.Y.Y.Y with the addresses of other nodes -- The same command-line can be used on all nodes (convenient!) +- A node can add its own address (it will work fine) + +- ... Which means that we can use the same command-line on all nodes (convenient!) --- @@ -258,19 +260,13 @@ consul agent -data-dir=/consul/data -client=0.0.0.0 -server -ui \ ## Putting it all together -- The file `k8s/consul.yaml` defines the required resources +- The file `k8s/consul-1.yaml` defines the required resources - (service account, cluster role, cluster role binding, service, stateful set) + (service account, role, role binding, service, stateful set) -- It has a few extra touches: +- Inspired by this [excellent tutorial](https://github.com/kelseyhightower/consul-on-kubernetes) by Kelsey Hightower - - a `podAntiAffinity` prevents two pods from running on the same node - - - a `preStop` hook makes the pod leave the cluster when shutdown gracefully - -This was inspired by this [excellent tutorial](https://github.com/kelseyhightower/consul-on-kubernetes) by Kelsey Hightower. -Some features from the original tutorial (TLS authentication between -nodes and encryption of gossip traffic) were removed for simplicity. + (many features from the original tutorial were removed for simplicity) --- @@ -282,7 +278,7 @@ nodes and encryption of gossip traffic) were removed for simplicity. - Create the stateful set and associated service: ```bash - kubectl apply -f ~/container.training/k8s/consul.yaml + kubectl apply -f ~/container.training/k8s/consul-1.yaml ``` - Check the logs as the pods come up one after another: @@ -306,6 +302,88 @@ nodes and encryption of gossip traffic) were removed for simplicity. ## Caveats +- The scheduler may place two Consul pods on the same node + + - if that node fails, we lose two Consul pods at the same time + - this will cause the cluster to fail + +- Scaling down the cluster will cause it to fail + + - when a Consul member leaves the cluster, it needs to inform the others + - otherwise, the last remaining node doesn't have quorum and stops functioning + +- This Consul cluster doesn't use real persistence yet + + - data is stored in the containers' ephemeral filesystem + - if a pod fails, its replacement starts from a blank slate + +--- + +## Improving pod placement + +- We need to tell the scheduler: + + *do not put two of these pods on the same node!* + +- This is done with an `affinity` section like the following one: + ```yaml + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - consul + topologyKey: kubernetes.io/hostname + ``` + +--- + +## Using a lifecycle hook + +- When a Consul member leaves the cluster, it needs to execute: + ```bash + consul leave + ``` + +- This is done with a `lifecycle` section like the following one: + ```yaml + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - consul leave + ``` + +--- + +## Running a better Consul cluster + +- Let's try to add the scheduling constraint and lifecycle hook + +- We can do that in the same namespace or another one (as we like) + +- If we do that in the same namespace, we will see a rolling update + + (pods will be replaced one by one) + +.exercise[ + +- Deploy a better Consul cluster: + ```bash + kubectl apply -f ~/container.training/k8s/consul-2.yaml + ``` + +] + +--- + +## Still no persistence, though + - We aren't using actual persistence yet (no `volumeClaimTemplate`, Persistent Volume, etc.)