From 36d1199b70937c053a6bd803d6ec871f9ecf44c3 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Sun, 9 Feb 2020 11:29:05 -0600 Subject: [PATCH] Upgrade from 1.15 to illustrate version skipping + retry logic Instead of upgrading from 1.16 to we upgrade from 1.15 to 1.16, because upgrading from is a special case and it is better to show the general case. Also, the script that sets up admin clusters now has some retry logic to accommodate hiccups in pssh or in the cloud provider. --- prepare-vms/setup-admin-clusters.sh | 40 +++++++++++++------ slides/k8s/cluster-upgrade.md | 61 ++++++++++++++++++++++------- 2 files changed, 74 insertions(+), 27 deletions(-) diff --git a/prepare-vms/setup-admin-clusters.sh b/prepare-vms/setup-admin-clusters.sh index b50783d4..c4635740 100755 --- a/prepare-vms/setup-admin-clusters.sh +++ b/prepare-vms/setup-admin-clusters.sh @@ -1,9 +1,24 @@ #!/bin/sh set -e +retry () { + N=$1 + I=0 + shift + + while ! "$@"; do + I=$(($I+1)) + if [ $I -gt $N ]; then + echo "FAILED, ABORTING" + exit 1 + fi + echo "FAILED, RETRYING ($I/$N)" + done +} + export AWS_INSTANCE_TYPE=t3a.small -INFRA=infra/aws-us-west-2 +INFRA=infra/aws-eu-west-3 STUDENTS=2 @@ -17,9 +32,9 @@ TAG=$PREFIX-$SETTINGS --settings settings/$SETTINGS.yaml \ --count $STUDENTS -./workshopctl deploy $TAG -./workshopctl disabledocker $TAG -./workshopctl kubebins $TAG +retry 5 ./workshopctl deploy $TAG +retry 5 ./workshopctl disabledocker $TAG +retry 5 ./workshopctl kubebins $TAG ./workshopctl cards $TAG SETTINGS=admin-kubenet @@ -30,9 +45,9 @@ TAG=$PREFIX-$SETTINGS --settings settings/$SETTINGS.yaml \ --count $((3*$STUDENTS)) -./workshopctl disableaddrchecks $TAG -./workshopctl deploy $TAG -./workshopctl kubebins $TAG +retry 5 ./workshopctl disableaddrchecks $TAG +retry 5 ./workshopctl deploy $TAG +retry 5 ./workshopctl kubebins $TAG ./workshopctl cards $TAG SETTINGS=admin-kuberouter @@ -43,9 +58,9 @@ TAG=$PREFIX-$SETTINGS --settings settings/$SETTINGS.yaml \ --count $((3*$STUDENTS)) -./workshopctl disableaddrchecks $TAG -./workshopctl deploy $TAG -./workshopctl kubebins $TAG +retry 5 ./workshopctl disableaddrchecks $TAG +retry 5 ./workshopctl deploy $TAG +retry 5 ./workshopctl kubebins $TAG ./workshopctl cards $TAG #INFRA=infra/aws-us-west-1 @@ -60,7 +75,6 @@ TAG=$PREFIX-$SETTINGS --settings settings/$SETTINGS.yaml \ --count $((3*$STUDENTS)) -./workshopctl deploy $TAG -./workshopctl kube $TAG 1.16.6 +retry 5 ./workshopctl deploy $TAG +retry 5 ./workshopctl kube $TAG 1.15.9 ./workshopctl cards $TAG - diff --git a/slides/k8s/cluster-upgrade.md b/slides/k8s/cluster-upgrade.md index 20a22a22..bfae7535 100644 --- a/slides/k8s/cluster-upgrade.md +++ b/slides/k8s/cluster-upgrade.md @@ -81,7 +81,7 @@ ## What version are we running anyway? -- When I say, "I'm running Kubernetes 1.16", is that the version of: +- When I say, "I'm running Kubernetes 1.15", is that the version of: - kubectl @@ -254,7 +254,7 @@ and kubectl, which can be one MINOR ahead or behind API server.] sudo vim /etc/kubernetes/manifests/kube-apiserver.yaml ``` -- Look for the `image:` line, and update it to e.g. `v1.17.0` +- Look for the `image:` line, and update it to e.g. `v1.16.0` ] @@ -308,11 +308,11 @@ and kubectl, which can be one MINOR ahead or behind API server.] ] -Note 1: kubeadm thinks that our cluster is running 1.17.0. +Note 1: kubeadm thinks that our cluster is running 1.16.0.
It is confused by our manual upgrade of the API server! -Note 2: kubeadm itself is still version 1.16.6. -
It doesn't know how to upgrade do 1.17.X. +Note 2: kubeadm itself is still version 1.15.9. +
It doesn't know how to upgrade do 1.16.X. --- @@ -334,8 +334,39 @@ Note 2: kubeadm itself is still version 1.16.6. ] -Note: kubeadm still thinks that our cluster is running 1.17.0. -
But at least it knows about version 1.17.X now. +Problem: kubeadm doesn't know know how to handle +upgrades from version 1.15. + +This is because we installed version 1.17 (or even later). + +We need to install kubeadm version 1.16.X. + +--- + +## Downgrading kubeadm + +- We need to go back to version 1.16.X (e.g. 1.16.6) + +.exercise[ + +- View available versions for package `kubeadm`: + ```bash + apt show kubeadm -a | grep ^Version | grep 1.16 + ``` + +- Downgrade kubeadm: + ``` + sudo apt install kubeadm=1.16.6-00 + ``` + +- Check what kubeadm tells us: + ``` + sudo kubeadm upgrade plan + ``` + +] + +kubeadm should now agree to upgrade to 1.16.6. --- @@ -351,7 +382,7 @@ Note: kubeadm still thinks that our cluster is running 1.17.0. - Perform the upgrade: ```bash - sudo kubeadm upgrade apply v1.17.2 + sudo kubeadm upgrade apply v1.16.6 ``` ] @@ -375,7 +406,7 @@ Note: kubeadm still thinks that our cluster is running 1.17.0. - Upgrade kubelet: ```bash - sudo apt install kubelet=1.17.2-00 + sudo apt install kubelet=1.16.6-00 ``` ] @@ -423,7 +454,7 @@ Note: kubeadm still thinks that our cluster is running 1.17.0. ## Upgrading kubelet the right way -- The command that we need to run was shown by kubeadm +- We need to upgrade kubeadm, upgrade kubelet config, then upgrade kubelet (after upgrading the control plane) @@ -432,8 +463,10 @@ Note: kubeadm still thinks that our cluster is running 1.17.0. - Download the configuration on each node, and upgrade kubelet: ```bash for N in 1 2 3; do - ssh test$N sudo kubeadm upgrade node config --kubelet-version v1.17.2 - ssh test$N sudo apt install kubelet=1.17.2-00 + ssh test$N " + sudo apt install kubeadm=1.16.6-00 && + sudo kubeadm upgrade node && + sudo apt install kubelet=1.16.6-00" done ``` ] @@ -442,7 +475,7 @@ Note: kubeadm still thinks that our cluster is running 1.17.0. ## Checking what we've done -- All our nodes should now be updated to version 1.17.2 +- All our nodes should now be updated to version 1.16.6 .exercise[ @@ -459,7 +492,7 @@ class: extra-details ## Skipping versions -- This example worked because we went from 1.16 to 1.17 +- This example worked because we went from 1.15 to 1.16 - If you are upgrading from e.g. 1.14, you will have to go through 1.15 first