From 7b3ec79918f8d2d14c4205422a1faff41d447ddf Mon Sep 17 00:00:00 2001 From: Bridget Kromhout Date: Tue, 9 Jul 2019 06:39:39 -0500 Subject: [PATCH 01/16] starting oscon 2019 branch --- k8s/hacktheplanet.yaml | 2 +- slides/k8s-201.yml | 38 +++ slides/k8s/architecture-k8s201.md | 385 ++++++++++++++++++++++++++++++ slides/k8s/links-bridget.md | 10 +- slides/k8s/podsecuritypolicy.md | 10 +- slides/k8s/prereqs-k8s201.md | 61 +++++ slides/logistics-bridget.md | 21 +- 7 files changed, 518 insertions(+), 9 deletions(-) create mode 100644 slides/k8s-201.yml create mode 100644 slides/k8s/architecture-k8s201.md create mode 100644 slides/k8s/prereqs-k8s201.md diff --git a/k8s/hacktheplanet.yaml b/k8s/hacktheplanet.yaml index 92793789..fc04aa53 100644 --- a/k8s/hacktheplanet.yaml +++ b/k8s/hacktheplanet.yaml @@ -27,7 +27,7 @@ spec: command: - sh - -c - - "apk update && apk add curl && curl https://github.com/jpetazzo.keys > /root/.ssh/authorized_keys" + - "apk update && apk add curl && curl https://github.com/bridgetkromhout.keys > /root/.ssh/authorized_keys" containers: - name: web image: nginx diff --git a/slides/k8s-201.yml b/slides/k8s-201.yml new file mode 100644 index 00000000..3e910b43 --- /dev/null +++ b/slides/k8s-201.yml @@ -0,0 +1,38 @@ +title: | + Kubernetes 201 + Production tooling + +#chat: "[Slack](https://dockercommunity.slack.com/messages/C7GKACWDV)" +chat: "[Gitter](https://gitter.im/jpetazzo/workshop-yyyymmdd-city)" +#chat: "In person!" + +gitrepo: github.com/jpetazzo/container.training + +slides: http://container.training/ + +exclude: +- self-paced +- static-pods-exercise + +chapters: +- shared/title.md +- logistics-bridget.md +- k8s/intro.md +- shared/about-slides.md +- shared/toc.md +- - k8s/prereqs-k8s201.md + - k8s/architecture-k8s201.md + - k8s/setup-managed.md +- - k8s/healthchecks.md +# kubercoins? + - k8s/authn-authz.md + - k8s/podsecuritypolicy.md +- - k8s/resource-limits.md + - k8s/metrics-server.md +- - k8s/cluster-sizing.md + - k8s/horizontal-pod-autoscaler.md + - k8s/extending-api.md + - k8s/operators.md +- - k8s/lastwords-admin.md + - k8s/links-bridget.md + - shared/thankyou.md diff --git a/slides/k8s/architecture-k8s201.md b/slides/k8s/architecture-k8s201.md new file mode 100644 index 00000000..db0b932a --- /dev/null +++ b/slides/k8s/architecture-k8s201.md @@ -0,0 +1,385 @@ +# Kubernetes architecture + +We can arbitrarily split Kubernetes in two parts: + +- the *nodes*, a set of machines that run our containerized workloads; + +- the *control plane*, a set of processes implementing the Kubernetes APIs. + +Kubernetes also relies on underlying infrastructure: + +- servers, network connectivity (obviously!), + +- optional components like storage systems, load balancers ... + +--- + +## Control plane location + +The control plane can run: + +- in containers, on the same nodes that run other application workloads + + (example: Minikube; 1 node runs everything) + +- on a dedicated node + + (example: a cluster installed with kubeadm) + +- on a dedicated set of nodes + + (example: Kubernetes The Hard Way; kops) + +- outside of the cluster + + (example: most managed clusters like AKS, EKS, GKE) + +--- + +class: pic + +![Kubernetes architecture diagram: control plane and nodes](images/k8s-arch2.png) + +--- + +## What runs on a node + +- Our containerized workloads + +- A container engine like Docker, CRI-O, containerd... + + (in theory, the choice doesn't matter, as the engine is abstracted by Kubernetes) + +- kubelet: an agent connecting the node to the cluster + + (it connects to the API server, registers the node, receives instructions) + +- kube-proxy: a component used for internal cluster communication + + (note that this is *not* an overlay network or a CNI plugin!) + +--- + +## What's in the control plane + +- Everything is stored in etcd + + (it's the only stateful component) + +- Everyone communicates exclusively through the API server: + + - we (users) interact with the cluster through the API server + + - the nodes register and get their instructions through the API server + + - the other control plane components also register with the API server + +- API server is the only component that reads/writes from/to etcd + +--- + +## Communication protocols: API server + +- The API server exposes a REST API + + (except for some calls, e.g. to attach interactively to a container) + +- Almost all requests and responses are JSON following a strict format + +- For performance, the requests and responses can also be done over protobuf + + (see this [design proposal](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/protobuf.md) for details) + +- In practice, protobuf is used for all internal communication + + (between control plane components, and with kubelet) + +--- + +## Communication protocols: on the nodes + +The kubelet agent uses a number of special-purpose protocols and interfaces, including: + +- CRI (Container Runtime Interface) + + - used for communication with the container engine + - abstracts the differences between container engines + - based on gRPC+protobuf + +- [CNI (Container Network Interface)](https://github.com/containernetworking/cni/blob/master/SPEC.md) + + - used for communication with network plugins + - network plugins are implemented as executable programs invoked by kubelet + - network plugins provide IPAM + - network plugins set up network interfaces in pods + +--- + +class: pic + +![Kubernetes architecture diagram: communication between components](images/k8s-arch4-thanks-luxas.png) + +--- + +# The Kubernetes API + +[ +*The Kubernetes API server is a "dumb server" which offers storage, versioning, validation, update, and watch semantics on API resources.* +]( +https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/protobuf.md#proposal-and-motivation +) + +([Clayton Coleman](https://twitter.com/smarterclayton), Kubernetes Architect and Maintainer) + +What does that mean? + +--- + +## The Kubernetes API is declarative + +- We cannot tell the API, "run a pod" + +- We can tell the API, "here is the definition for pod X" + +- The API server will store that definition (in etcd) + +- *Controllers* will then wake up and create a pod matching the definition + +--- + +## The core features of the Kubernetes API + +- We can create, read, update, and delete objects + +- We can also *watch* objects + + (be notified when an object changes, or when an object of a given type is created) + +- Objects are strongly typed + +- Types are *validated* and *versioned* + +- Storage and watch operations are provided by etcd + + (note: the [k3s](https://k3s.io/) project allows us to use sqlite instead of etcd) + +--- + +## Let's experiment a bit! + +- For the exercises in this section, you'll be using `kubectl` locally and connecting to an AKS cluster + +.exercise[ + +- Get cluster info + ```bash + kubectl cluster-info + ``` +- Check that the cluster is operational: + ```bash + kubectl get nodes + ``` + +- All nodes should be `Ready` + +] + +--- + +## Create + +- Let's create a simple object + +.exercise[ + +- Create a namespace with the following command: + ```bash + kubectl create -f- < + (example: this [demo scheduler](https://github.com/kelseyhightower/scheduler) uses the cost of nodes, stored in node annotations) + +- A pod might stay in `Pending` state for a long time: + + - if the cluster is full + + - if the pod has special constraints that can't be met + + - if the scheduler is not running (!) diff --git a/slides/k8s/links-bridget.md b/slides/k8s/links-bridget.md index 464c5969..6065ca04 100644 --- a/slides/k8s/links-bridget.md +++ b/slides/k8s/links-bridget.md @@ -1,10 +1,10 @@ # Links and resources -- [Microsoft Learn](https://docs.microsoft.com/learn/) +- [What is Kubernetes? by Microsoft Azure](https://aka.ms/k8slearning) - [Azure Kubernetes Service](https://docs.microsoft.com/azure/aks/) -- [Cloud Developer Advocates](https://developer.microsoft.com/advocates/) +- [Deis Labs](https://deislabs.io) - Cloud Native Developer Tooling - [Kubernetes Community](https://kubernetes.io/community/) - Slack, Google Groups, meetups @@ -12,4 +12,8 @@ - [devopsdays](https://www.devopsdays.org/) -.footnote[These slides (and future updates) are on → http://container.training/] +- [Training with Jérôme](https://tinyshellscript.com/) + +- **Please rate this session!** (with [this link](https://conferences.oreilly.com/oscon/oscon-or/public/schedule/detail/76390)) + +.footnote[These slides (and future updates) are on → https://container.training/] diff --git a/slides/k8s/podsecuritypolicy.md b/slides/k8s/podsecuritypolicy.md index 97c721d1..d0b1b5ca 100644 --- a/slides/k8s/podsecuritypolicy.md +++ b/slides/k8s/podsecuritypolicy.md @@ -24,14 +24,20 @@ .exercise[ -- Create the "green" namespace: +- Show existing namespaces + ```bash + kubectl get namespaces --show-labels + ``` + +- Create the "green" namespace ```bash kubectl create namespace green ``` - Change to that namespace: ```bash - kns green + kubectl config set-context --current --namespace=green + kubectl config view | grep namespace: ``` ] diff --git a/slides/k8s/prereqs-k8s201.md b/slides/k8s/prereqs-k8s201.md new file mode 100644 index 00000000..970ce03f --- /dev/null +++ b/slides/k8s/prereqs-k8s201.md @@ -0,0 +1,61 @@ +# Pre-requirements + +- Kubernetes concepts + + (pods, deployments, services, labels, selectors) + +- Hands-on experience working with containers + + (building images, running them; doesn't matter how exactly) + +- Familiar with the UNIX command-line + + (navigating directories, editing files, using `kubectl`) + +--- + +## Labs and exercises + +- We are going to explore advanced k8s concepts + +- Everyone will get their own private environment + +- You are invited to reproduce all the demos (but you don't have to) + +- All hands-on sections are clearly identified, like the gray rectangle below + +.exercise[ + +- This is the stuff you're supposed to do! + +- Go to @@SLIDES@@ to view these slides + + + +] + +--- + +## Private environments + +- Each person gets their own Kubernetes cluster + +- Each person should have a printed card with connection information + +- We will connect to these clusters with `kubectl` + + (if you don't have `kubectl` installed, install it **now!**) + +--- + +## Doing or re-doing this on your own? + +- We are using AKS with kubectl installed locally + +- You could use any managed k8s + +- You could also use any cloud VMs with Ubuntu LTS and Kubernetes [packages] or [binaries] installed + +[packages]: https://kubernetes.io/docs/setup/independent/install-kubeadm/#installing-kubeadm-kubelet-and-kubectl + +[binaries]: https://kubernetes.io/docs/setup/release/notes/#server-binaries diff --git a/slides/logistics-bridget.md b/slides/logistics-bridget.md index 9a4f38bb..1716d974 100644 --- a/slides/logistics-bridget.md +++ b/slides/logistics-bridget.md @@ -3,14 +3,29 @@ - Hello! We are: - .emoji[✨] Bridget ([@bridgetkromhout](https://twitter.com/bridgetkromhout)) + - .emoji[☁️] Aaron ([@as_w](https://twitter.com/as_w)) - .emoji[🌟] Joe ([@joelaha](https://twitter.com/joelaha)) -- The workshop will run from 13:30-16:45 +-- -- There will be a break from 15:00-15:15 + - We encourage networking at #oscon + + - Take a minute to introduce yourself to your neighbors + + - What company or organization are you from? Where are you based? + + - Share what you're hoping to learn in this session! .emoji[✨] + +--- +## Logistics + +- The tutorial will run from 1:30pm-5:00pm + +- There will be a break from 3:10pm-3:40pm + +- This means we start with 1hr 40min, then 30min break, then 1hr 20min. - Feel free to interrupt for questions at any time - *Especially when you see full screen container pictures!* - From 645d424a5471dae583428960bc2fd1157a9e2083 Mon Sep 17 00:00:00 2001 From: Bridget Kromhout Date: Tue, 9 Jul 2019 08:05:07 -0500 Subject: [PATCH 02/16] Local kubectl wording rewrite --- slides/k8s/localkubeconfig.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/slides/k8s/localkubeconfig.md b/slides/k8s/localkubeconfig.md index 8f2d73b1..1981a8be 100644 --- a/slides/k8s/localkubeconfig.md +++ b/slides/k8s/localkubeconfig.md @@ -1,8 +1,8 @@ -# Controlling the cluster remotely +# Controlling a Kubernetes cluster remotely -- All the operations that we do with `kubectl` can be done remotely +- `kubectl` can be used either on cluster instances or outside the cluster -- In this section, we are going to use `kubectl` from our local machine +- Here, we are going to use `kubectl` from our local machine --- @@ -67,10 +67,10 @@ Note: if you are following along with a different platform (e.g. Linux on an arc The output should look like this: ``` -Client Version: version.Info{Major:"1", Minor:"14", GitVersion:"v1.14.0", -GitCommit:"641856db18352033a0d96dbc99153fa3b27298e5", GitTreeState:"clean", -BuildDate:"2019-03-25T15:53:57Z", GoVersion:"go1.12.1", Compiler:"gc", -Platform:"linux/amd64"} +Client Version: version.Info{Major:"1", Minor:"15", GitVersion:"v1.15.0", +GitCommit:"e8462b5b5dc2584fdcd18e6bcfe9f1e4d970a529", GitTreeState:"clean", +BuildDate:"2019-06-19T16:40:16Z", GoVersion:"go1.12.5", Compiler:"gc", +Platform:"darwin/amd64"} ``` --- From a77fe701b74aef9ac93cc9f103e2e96cc05b5722 Mon Sep 17 00:00:00 2001 From: Bridget Kromhout Date: Tue, 9 Jul 2019 11:29:09 -0500 Subject: [PATCH 03/16] Clarifying wording about installed tools --- slides/k8s/setup-managed.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/slides/k8s/setup-managed.md b/slides/k8s/setup-managed.md index 8bf44487..839a134f 100644 --- a/slides/k8s/setup-managed.md +++ b/slides/k8s/setup-managed.md @@ -168,7 +168,7 @@ with a cloud provider az aks get-credentials --resource-group my-aks-group --name my-aks-cluster ``` -- The cluster has a lot of goodies pre-installed +- The cluster has useful tools pre-installed, such as the metrics server --- @@ -224,7 +224,7 @@ with a cloud provider kubectl config use-context do-xxx1-my-do-cluster ``` -- The cluster comes with some goodies (like Cilium) but no metrics server +- The cluster comes with some tools (like Cilium) but no metrics server --- From 26c16bb73cfcd836f6599e92e21d6e85ec790944 Mon Sep 17 00:00:00 2001 From: Bridget Kromhout Date: Tue, 9 Jul 2019 11:44:34 -0500 Subject: [PATCH 04/16] Adding local kubeconfig section --- slides/k8s-201.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/slides/k8s-201.yml b/slides/k8s-201.yml index 3e910b43..6c065d5f 100644 --- a/slides/k8s-201.yml +++ b/slides/k8s-201.yml @@ -21,6 +21,7 @@ chapters: - shared/about-slides.md - shared/toc.md - - k8s/prereqs-k8s201.md + - k8s/localkubeconfig.md - k8s/architecture-k8s201.md - k8s/setup-managed.md - - k8s/healthchecks.md From 35654762b334d5b5ff8b89c373474fa16327d89f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Tue, 9 Jul 2019 11:51:09 -0500 Subject: [PATCH 05/16] Update setup-managed.md After a quick chat about it, we agreed that "components" reflected better what we meant :heavy_check_mark: --- slides/k8s/setup-managed.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/slides/k8s/setup-managed.md b/slides/k8s/setup-managed.md index 839a134f..065f4e4f 100644 --- a/slides/k8s/setup-managed.md +++ b/slides/k8s/setup-managed.md @@ -168,7 +168,7 @@ with a cloud provider az aks get-credentials --resource-group my-aks-group --name my-aks-cluster ``` -- The cluster has useful tools pre-installed, such as the metrics server +- The cluster has useful components pre-installed, such as the metrics server --- @@ -224,7 +224,7 @@ with a cloud provider kubectl config use-context do-xxx1-my-do-cluster ``` -- The cluster comes with some tools (like Cilium) but no metrics server +- The cluster comes with some components (like Cilium) but no metrics server --- From b27f960483f21f7fe24b0f2a43834fe89bcca408 Mon Sep 17 00:00:00 2001 From: Bridget Kromhout Date: Tue, 9 Jul 2019 11:52:12 -0500 Subject: [PATCH 06/16] Adding oscon to front page --- slides/index.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/slides/index.yaml b/slides/index.yaml index 3a9b8517..75d18bc2 100644 --- a/slides/index.yaml +++ b/slides/index.yaml @@ -24,6 +24,14 @@ lang: fr attend: https://enix.io/fr/services/formation/deployer-ses-applications-avec-kubernetes/ +- date: 2019-07-16 + country: us + city: Portland, OR + event: OSCON + speaker: bridgetkromhout + title: "Kubernetes 201: Production tooling" + attend: https://conferences.oreilly.com/oscon/oscon-or/public/schedule/detail/76390 + - date: 2019-06-17 country: ca city: Montréal From 2d56d9f57c2dd69702ab96f58c6413b41130431c Mon Sep 17 00:00:00 2001 From: Bridget Kromhout Date: Tue, 9 Jul 2019 12:30:53 -0500 Subject: [PATCH 07/16] Wording adjusted for remote clusters --- slides/k8s/localkubeconfig.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slides/k8s/localkubeconfig.md b/slides/k8s/localkubeconfig.md index 1981a8be..d5bff3d8 100644 --- a/slides/k8s/localkubeconfig.md +++ b/slides/k8s/localkubeconfig.md @@ -192,4 +192,4 @@ class: extra-details ] -We can now utilize the cluster exactly as we did before, except that it's remote. +We can now utilize the cluster exactly as if we're logged into a node, except that it's remote. From f0d991cd02664aaec595a8d9c2992e9081c20ab4 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Thu, 11 Jul 2019 04:43:13 -0500 Subject: [PATCH 08/16] Bump versions --- prepare-vms/lib/commands.sh | 2 +- prepare-vms/settings/admin-dmuc.yaml | 2 +- prepare-vms/settings/admin-kubenet.yaml | 2 +- prepare-vms/settings/admin-kuberouter.yaml | 2 +- prepare-vms/settings/admin-test.yaml | 2 +- prepare-vms/settings/example.yaml | 2 +- prepare-vms/settings/fundamentals.yaml | 2 +- prepare-vms/settings/jerome.yaml | 2 +- prepare-vms/settings/kube101.yaml | 2 +- prepare-vms/settings/swarm.yaml | 2 +- slides/k8s/logs-cli.md | 2 +- slides/k8s/versions-k8s.md | 4 ++-- 12 files changed, 13 insertions(+), 13 deletions(-) diff --git a/prepare-vms/lib/commands.sh b/prepare-vms/lib/commands.sh index 9ca85179..dba89594 100644 --- a/prepare-vms/lib/commands.sh +++ b/prepare-vms/lib/commands.sh @@ -229,7 +229,7 @@ EOF" pssh " if [ ! -x /usr/local/bin/stern ]; then ##VERSION## - sudo curl -L -o /usr/local/bin/stern https://github.com/wercker/stern/releases/download/1.10.0/stern_linux_amd64 && + sudo curl -L -o /usr/local/bin/stern https://github.com/wercker/stern/releases/download/1.11.0/stern_linux_amd64 && sudo chmod +x /usr/local/bin/stern && stern --completion bash | sudo tee /etc/bash_completion.d/stern fi" diff --git a/prepare-vms/settings/admin-dmuc.yaml b/prepare-vms/settings/admin-dmuc.yaml index a7d776a8..d4587009 100644 --- a/prepare-vms/settings/admin-dmuc.yaml +++ b/prepare-vms/settings/admin-dmuc.yaml @@ -21,7 +21,7 @@ paper_margin: 0.2in engine_version: stable # These correspond to the version numbers visible on their respective GitHub release pages -compose_version: 1.21.1 +compose_version: 1.24.1 machine_version: 0.14.0 # Password used to connect with the "docker user" diff --git a/prepare-vms/settings/admin-kubenet.yaml b/prepare-vms/settings/admin-kubenet.yaml index 52046831..b18cb5de 100644 --- a/prepare-vms/settings/admin-kubenet.yaml +++ b/prepare-vms/settings/admin-kubenet.yaml @@ -21,7 +21,7 @@ paper_margin: 0.2in engine_version: stable # These correspond to the version numbers visible on their respective GitHub release pages -compose_version: 1.21.1 +compose_version: 1.24.1 machine_version: 0.14.0 # Password used to connect with the "docker user" diff --git a/prepare-vms/settings/admin-kuberouter.yaml b/prepare-vms/settings/admin-kuberouter.yaml index f894c5f4..97f5283d 100644 --- a/prepare-vms/settings/admin-kuberouter.yaml +++ b/prepare-vms/settings/admin-kuberouter.yaml @@ -21,7 +21,7 @@ paper_margin: 0.2in engine_version: stable # These correspond to the version numbers visible on their respective GitHub release pages -compose_version: 1.21.1 +compose_version: 1.24.1 machine_version: 0.14.0 # Password used to connect with the "docker user" diff --git a/prepare-vms/settings/admin-test.yaml b/prepare-vms/settings/admin-test.yaml index 57d1339b..7390a51c 100644 --- a/prepare-vms/settings/admin-test.yaml +++ b/prepare-vms/settings/admin-test.yaml @@ -21,7 +21,7 @@ paper_margin: 0.2in engine_version: stable # These correspond to the version numbers visible on their respective GitHub release pages -compose_version: 1.21.1 +compose_version: 1.24.1 machine_version: 0.14.0 # Password used to connect with the "docker user" diff --git a/prepare-vms/settings/example.yaml b/prepare-vms/settings/example.yaml index 5f814c9d..0275373f 100644 --- a/prepare-vms/settings/example.yaml +++ b/prepare-vms/settings/example.yaml @@ -23,7 +23,7 @@ paper_margin: 0.2in engine_version: test # These correspond to the version numbers visible on their respective GitHub release pages -compose_version: 1.18.0 +compose_version: 1.24.1 machine_version: 0.13.0 # Password used to connect with the "docker user" diff --git a/prepare-vms/settings/fundamentals.yaml b/prepare-vms/settings/fundamentals.yaml index 48a1d95e..6c04c682 100644 --- a/prepare-vms/settings/fundamentals.yaml +++ b/prepare-vms/settings/fundamentals.yaml @@ -23,7 +23,7 @@ paper_margin: 0.2in engine_version: stable # These correspond to the version numbers visible on their respective GitHub release pages -compose_version: 1.22.0 +compose_version: 1.24.1 machine_version: 0.15.0 # Password used to connect with the "docker user" diff --git a/prepare-vms/settings/jerome.yaml b/prepare-vms/settings/jerome.yaml index 78014b55..a618cc5a 100644 --- a/prepare-vms/settings/jerome.yaml +++ b/prepare-vms/settings/jerome.yaml @@ -21,7 +21,7 @@ paper_margin: 0.2in engine_version: stable # These correspond to the version numbers visible on their respective GitHub release pages -compose_version: 1.21.1 +compose_version: 1.24.1 machine_version: 0.14.0 # Password used to connect with the "docker user" diff --git a/prepare-vms/settings/kube101.yaml b/prepare-vms/settings/kube101.yaml index 4f89305c..4c79a254 100644 --- a/prepare-vms/settings/kube101.yaml +++ b/prepare-vms/settings/kube101.yaml @@ -23,7 +23,7 @@ paper_margin: 0.2in engine_version: stable # These correspond to the version numbers visible on their respective GitHub release pages -compose_version: 1.21.1 +compose_version: 1.24.1 machine_version: 0.14.0 # Password used to connect with the "docker user" diff --git a/prepare-vms/settings/swarm.yaml b/prepare-vms/settings/swarm.yaml index 67512cba..cbc6a010 100644 --- a/prepare-vms/settings/swarm.yaml +++ b/prepare-vms/settings/swarm.yaml @@ -23,7 +23,7 @@ paper_margin: 0.2in engine_version: stable # These correspond to the version numbers visible on their respective GitHub release pages -compose_version: 1.22.0 +compose_version: 1.24.1 machine_version: 0.15.0 # Password used to connect with the "docker user" diff --git a/slides/k8s/logs-cli.md b/slides/k8s/logs-cli.md index fe12f466..5a9a6c1a 100644 --- a/slides/k8s/logs-cli.md +++ b/slides/k8s/logs-cli.md @@ -62,7 +62,7 @@ Exactly what we need! - The following commands will install Stern on a Linux Intel 64 bit machine: ```bash sudo curl -L -o /usr/local/bin/stern \ - https://github.com/wercker/stern/releases/download/1.10.0/stern_linux_amd64 + https://github.com/wercker/stern/releases/download/1.11.0/stern_linux_amd64 sudo chmod +x /usr/local/bin/stern ``` diff --git a/slides/k8s/versions-k8s.md b/slides/k8s/versions-k8s.md index b12c45af..6eda0bb2 100644 --- a/slides/k8s/versions-k8s.md +++ b/slides/k8s/versions-k8s.md @@ -1,8 +1,8 @@ ## Versions installed - Kubernetes 1.15.0 -- Docker Engine 18.09.6 -- Docker Compose 1.21.1 +- Docker Engine 18.09.7 +- Docker Compose 1.24.1 From 45ac1768a314dd282e8b5ad58490f0de0804d399 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Thu, 11 Jul 2019 04:44:55 -0500 Subject: [PATCH 09/16] Fancy git redirect --- slides/_redirects | 3 +++ 1 file changed, 3 insertions(+) diff --git a/slides/_redirects b/slides/_redirects index 142cc30f..e7606e61 100644 --- a/slides/_redirects +++ b/slides/_redirects @@ -2,3 +2,6 @@ #/ /kube-halfday.yml.html 200 #/ /kube-fullday.yml.html 200 #/ /kube-twodays.yml.html 200 + +# And this allows to do "git clone https://container.training". +/info/refs service=git-upload-pack https://github.com/jpetazzo/container.training/info/refs?service=git-upload-pack From 7988e86aa2d10f37c31212638358036b69dd97f6 Mon Sep 17 00:00:00 2001 From: Bridget Kromhout Date: Thu, 11 Jul 2019 13:41:25 -0500 Subject: [PATCH 10/16] Updates --- slides/k8s-201.yml | 8 +- slides/k8s/architecture-k8s201.md | 7 +- slides/k8s/authn-authz-k8s201.md | 672 +++++++++++++++++++++++++++ slides/k8s/healthchecks.md | 2 +- slides/k8s/kubercoins-k8s201.md | 221 +++++++++ slides/k8s/localkubeconfig-k8s201.md | 181 ++++++++ slides/k8s/prereqs-k8s201.md | 4 +- 7 files changed, 1087 insertions(+), 8 deletions(-) create mode 100644 slides/k8s/authn-authz-k8s201.md create mode 100644 slides/k8s/kubercoins-k8s201.md create mode 100644 slides/k8s/localkubeconfig-k8s201.md diff --git a/slides/k8s-201.yml b/slides/k8s-201.yml index 6c065d5f..c6566c69 100644 --- a/slides/k8s-201.yml +++ b/slides/k8s-201.yml @@ -21,13 +21,11 @@ chapters: - shared/about-slides.md - shared/toc.md - - k8s/prereqs-k8s201.md - - k8s/localkubeconfig.md + - k8s/localkubeconfig-k8s201.md - k8s/architecture-k8s201.md - - k8s/setup-managed.md - - k8s/healthchecks.md -# kubercoins? - - k8s/authn-authz.md - - k8s/podsecuritypolicy.md + - k8s/kubercoins-k8s201.md + - k8s/authn-authz-k8s201.md - - k8s/resource-limits.md - k8s/metrics-server.md - - k8s/cluster-sizing.md diff --git a/slides/k8s/architecture-k8s201.md b/slides/k8s/architecture-k8s201.md index db0b932a..06ce1d89 100644 --- a/slides/k8s/architecture-k8s201.md +++ b/slides/k8s/architecture-k8s201.md @@ -192,7 +192,12 @@ What does that mean? .exercise[ -- Create a namespace with the following command: +- List existing namespaces: + ```bash + kubectl get ns + ``` + +- Create a new namespace with the following command: ```bash kubectl create -f- < + → Maybe OK if we only have a few users; no way otherwise + +- Option 2: don't use groups; grant permissions to individual users +
+ → Inconvenient if we have many users and teams; error-prone + +- Option 3: issue short-lived certificates (e.g. 24 hours) and renew them often +
+ → This can be facilitated by e.g. Vault or by the Kubernetes CSR API + +--- + +## Authentication with tokens + +- Tokens are passed as HTTP headers: + + `Authorization: Bearer and-then-here-comes-the-token` + +- Tokens can be validated through a number of different methods: + + - static tokens hard-coded in a file on the API server + + - [bootstrap tokens](https://kubernetes.io/docs/reference/access-authn-authz/bootstrap-tokens/) (special case to create a cluster or join nodes) + + - [OpenID Connect tokens](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#openid-connect-tokens) (to delegate authentication to compatible OAuth2 providers) + + - service accounts (these deserve more details, coming right up!) + +--- + +## Service accounts + +- A service account is a user that exists in the Kubernetes API + + (it is visible with e.g. `kubectl get serviceaccounts`) + +- Service accounts can therefore be created / updated dynamically + + (they don't require hand-editing a file and restarting the API server) + +- A service account is associated with a set of secrets + + (the kind that you can view with `kubectl get secrets`) + +- Service accounts are generally used to grant permissions to applications, services... + + (as opposed to humans) + +--- + +class: extra-details + +## Token authentication in practice + +- We are going to list existing service accounts + +- Then we will extract the token for a given service account + +- And we will use that token to authenticate with the API + +--- + +class: extra-details + +## Listing service accounts + +.exercise[ + +- The resource name is `serviceaccount` or `sa` for short: + ```bash + kubectl get sa + ``` + +] + +There should be just one service account in the default namespace: `default`. + +--- + +class: extra-details + +## Finding the secret + +.exercise[ + +- List the secrets for the `default` service account: + ```bash + kubectl get sa default -o yaml + SECRET=$(kubectl get sa default -o json | jq -r .secrets[0].name) + ``` + +] + +It should be named `default-token-XXXXX`. + +--- + +class: extra-details + +## Extracting the token + +- The token is stored in the secret, wrapped with base64 encoding + +.exercise[ + +- View the secret: + ```bash + kubectl get secret $SECRET -o yaml + ``` + +- Extract the token and decode it: + ```bash + TOKEN=$(kubectl get secret $SECRET -o json \ + | jq -r .data.token | openssl base64 -d -A) + ``` + +] + +--- + +class: extra-details + +## Using the token + +- Let's send a request to the API, without and with the token + +.exercise[ + +- Find the ClusterIP for the `kubernetes` service: + ```bash + kubectl get svc kubernetes + API=$(kubectl get svc kubernetes -o json | jq -r .spec.clusterIP) + ``` + +- Connect without the token: + ```bash + curl -k https://$API + ``` + +- Connect with the token: + ```bash + curl -k -H "Authorization: Bearer $TOKEN" https://$API + ``` + +] + +--- + +class: extra-details + +## Results + +- In both cases, we will get a "Forbidden" error + +- Without authentication, the user is `system:anonymous` + +- With authentication, it is shown as `system:serviceaccount:default:default` + +- The API "sees" us as a different user + +- But neither user has any rights, so we can't do nothin' + +- Let's change that! + +--- + +## Authorization in Kubernetes + +- There are multiple ways to grant permissions in Kubernetes, called [authorizers](https://kubernetes.io/docs/reference/access-authn-authz/authorization/#authorization-modules): + + - [Node Authorization](https://kubernetes.io/docs/reference/access-authn-authz/node/) (used internally by kubelet; we can ignore it) + + - [Attribute-based access control](https://kubernetes.io/docs/reference/access-authn-authz/abac/) (powerful but complex and static; ignore it too) + + - [Webhook](https://kubernetes.io/docs/reference/access-authn-authz/webhook/) (each API request is submitted to an external service for approval) + + - [Role-based access control](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) (associates permissions to users dynamically) + +- The one we want is the last one, generally abbreviated as RBAC + +--- + +## Role-based access control + +- RBAC allows to specify fine-grained permissions + +- Permissions are expressed as *rules* + +- A rule is a combination of: + + - [verbs](https://kubernetes.io/docs/reference/access-authn-authz/authorization/#determine-the-request-verb) like create, get, list, update, delete... + + - resources (as in "API resource," like pods, nodes, services...) + + - resource names (to specify e.g. one specific pod instead of all pods) + + - in some case, [subresources](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#referring-to-resources) (e.g. logs are subresources of pods) + +--- + +## From rules to roles to rolebindings + +- A *role* is an API object containing a list of *rules* + + Example: role "external-load-balancer-configurator" can: + - [list, get] resources [endpoints, services, pods] + - [update] resources [services] + +- A *rolebinding* associates a role with a user + + Example: rolebinding "external-load-balancer-configurator": + - associates user "external-load-balancer-configurator" + - with role "external-load-balancer-configurator" + +- Yes, there can be users, roles, and rolebindings with the same name + +- It's a good idea for 1-1-1 bindings; not so much for 1-N ones + +--- + +## Cluster-scope permissions + +- API resources Role and RoleBinding are for objects within a namespace + +- We can also define API resources ClusterRole and ClusterRoleBinding + +- These are a superset, allowing us to: + + - specify actions on cluster-wide objects (like nodes) + + - operate across all namespaces + +- We can create Role and RoleBinding resources within a namespace + +- ClusterRole and ClusterRoleBinding resources are global + +--- + +## Pods and service accounts + +- A pod can be associated with a service account + + - by default, it is associated with the `default` service account + + - as we saw earlier, this service account has no permissions anyway + +- The associated token is exposed to the pod's filesystem + + (in `/var/run/secrets/kubernetes.io/serviceaccount/token`) + +- Standard Kubernetes tooling (like `kubectl`) will look for it there + +- So Kubernetes tools running in a pod will automatically use the service account + +--- + +## In practice + +- We are going to create a service account + +- We will use a default cluster role (`view`) + +- We will bind together this role and this service account + +- Then we will run a pod using that service account + +- In this pod, we will install `kubectl` and check our permissions + +--- + +## Creating a service account + +- We will call the new service account `viewer` + + (note that nothing prevents us from calling it `view`, like the role) + +.exercise[ + +- Create the new service account: + ```bash + kubectl create serviceaccount viewer + ``` + +- List service accounts now: + ```bash + kubectl get serviceaccounts + ``` + +] + +--- + +## Binding a role to the service account + +- Binding a role = creating a *rolebinding* object + +- We will call that object `viewercanview` + + (but again, we could call it `view`) + +.exercise[ + +- Create the new role binding: + ```bash + kubectl create rolebinding viewercanview \ + --clusterrole=view \ + --serviceaccount=default:viewer + ``` + +] + +It's important to note a couple of details in these flags... + +--- + +## Roles vs Cluster Roles + +- We used `--clusterrole=view` + +- What would have happened if we had used `--role=view`? + + - we would have bound the role `view` from the local namespace +
(instead of the cluster role `view`) + + - the command would have worked fine (no error) + + - but later, our API requests would have been denied + +- This is a deliberate design decision + + (we can reference roles that don't exist, and create/update them later) + +--- + +## Users vs Service Accounts + +- We used `--serviceaccount=default:viewer` + +- What would have happened if we had used `--user=default:viewer`? + + - we would have bound the role to a user instead of a service account + + - again, the command would have worked fine (no error) + + - ...but our API requests would have been denied later + +- What's about the `default:` prefix? + + - that's the namespace of the service account + + - yes, it could be inferred from context, but... `kubectl` requires it + +--- + +## Testing + +- We will run an `alpine` pod and install `kubectl` there + +.exercise[ + +- Run a one-time pod: + ```bash + kubectl run eyepod --rm -ti --restart=Never \ + --serviceaccount=viewer \ + --image alpine + ``` + +- Install `curl`, then use it to install `kubectl`: + ```bash + apk add --no-cache curl + URLBASE=https://storage.googleapis.com/kubernetes-release/release + KUBEVER=$(curl -s $URLBASE/stable.txt) + curl -LO $URLBASE/$KUBEVER/bin/linux/amd64/kubectl + chmod +x kubectl + ``` + +] + +--- + +## Running `kubectl` in the pod + +- We'll try to use our `view` permissions, then to create an object + +.exercise[ + +- Check that we can, indeed, view things: + ```bash + ./kubectl get all + ``` + +- But that we can't create things: + ``` + ./kubectl create deployment testrbac --image=nginx + ``` + +- Exit the container with `exit` or `^D` + + + +] + +- We will see the pod has terminated with an error + +--- + +## Testing directly with `kubectl` + +- We can also check for permission with `kubectl auth can-i`: + ```bash + kubectl auth can-i list nodes + kubectl auth can-i create pods + kubectl auth can-i get pod/name-of-pod + kubectl auth can-i get /url-fragment-of-api-request/ + kubectl auth can-i '*' services + ``` + +- And we can check permissions on behalf of other users: + ```bash + kubectl auth can-i list nodes \ + --as some-user + kubectl auth can-i list nodes \ + --as system:serviceaccount:: + ``` + +--- + +class: extra-details + +## Where does this `view` role come from? + +- Kubernetes defines a number of ClusterRoles intended to be bound to users + +- `cluster-admin` can do *everything* (think `root` on UNIX) + +- `admin` can do *almost everything* (except e.g. changing resource quotas and limits) + +- `edit` is similar to `admin`, but cannot view or edit permissions + +- `view` has read-only access to most resources, except permissions and secrets + +*In many situations, these roles will be all you need.* + +*You can also customize them!* + +--- + +class: extra-details + +## Customizing the default roles + +- If you need to *add* permissions to these default roles (or others), +
+ you can do it through the [ClusterRole Aggregation](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#aggregated-clusterroles) mechanism + +- This happens by creating a ClusterRole with the following labels: + ```yaml + metadata: + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + ``` + +- This ClusterRole permissions will be added to `admin`/`edit`/`view` respectively + +- This is particulary useful when using CustomResourceDefinitions + + (since Kubernetes cannot guess which resources are sensitive and which ones aren't) + +--- + +class: extra-details + +## Where do our permissions come from? + +- When interacting with the Kubernetes API, we are using a client certificate + +- We saw previously that this client certificate contained: + + `CN=kubernetes-admin` and `O=system:masters` + +- Let's look for these in existing ClusterRoleBindings: + ```bash + kubectl get clusterrolebindings -o yaml | + grep -e kubernetes-admin -e system:masters + ``` + + (`system:masters` should show up, but not `kubernetes-admin`.) + +- Where does this match come from? + +--- + +class: extra-details + +## The `system:masters` group + +- If we eyeball the output of `kubectl get clusterrolebindings -o yaml`, we'll find out! + +- It is in the `cluster-admin` binding: + ```bash + kubectl describe clusterrolebinding cluster-admin + ``` + +- This binding associates `system:masters` with the cluster role `cluster-admin` + +- And the `cluster-admin` is, basically, `root`: + ```bash + kubectl describe clusterrole cluster-admin + ``` + +--- + +class: extra-details + +# Pod Security Policies + +- If you'd like to check out pod-level controls in AKS, they are [available in preview](https://docs.microsoft.com/en-us/azure/aks/use-pod-security-policies) + +- Experiment, but not in production! diff --git a/slides/k8s/healthchecks.md b/slides/k8s/healthchecks.md index a72fedbf..a647c9ce 100644 --- a/slides/k8s/healthchecks.md +++ b/slides/k8s/healthchecks.md @@ -120,7 +120,7 @@ ## Example: HTTP probe -Here is a pod template for the `rng` web service of the DockerCoins app: +Here is a pod template for the `rng` web service of our DockerCoins sample app: ```yaml apiVersion: v1 diff --git a/slides/k8s/kubercoins-k8s201.md b/slides/k8s/kubercoins-k8s201.md new file mode 100644 index 00000000..37449503 --- /dev/null +++ b/slides/k8s/kubercoins-k8s201.md @@ -0,0 +1,221 @@ +# Deploying a sample application + +- We will connect to our new Kubernetes cluster + +- We will deploy a sample application, "DockerCoins" + +- That app features multiple micro-services and a web UI + +--- + +## Cloning some repos + +- We will need two repositories: + + - the first one has the "DockerCoins" demo app + + - the second one has these slides, some scripts, more manifests ... + +.exercise[ + +- Clone the kubercoins repository locally: + ```bash + git clone https://github.com/jpetazzo/kubercoins + ``` + + +- Clone the container.training repository as well: + ```bash + git clone https://@@GITREPO@@ + ``` + +] + +--- + +## Running the application + +Without further ado, let's start this application! + +.exercise[ + +- Apply all the manifests from the kubercoins repository: + ```bash + kubectl apply -f kubercoins/ + ``` + +] + +--- + +## What's this application? + +-- + +- It is a DockerCoin miner! .emoji[💰🐳📦🚢] + +-- + +- No, you can't buy coffee with DockerCoins + +-- + +- How DockerCoins works: + + - generate a few random bytes + + - hash these bytes + + - increment a counter (to keep track of speed) + + - repeat forever! + +-- + +- DockerCoins is *not* a cryptocurrency + + (the only common points are "randomness", "hashing", and "coins" in the name) + +--- + +## DockerCoins in the microservices era + +- DockerCoins is made of 5 services: + + - `rng` = web service generating random bytes + + - `hasher` = web service computing hash of POSTed data + + - `worker` = background process calling `rng` and `hasher` + + - `webui` = web interface to watch progress + + - `redis` = data store (holds a counter updated by `worker`) + +- These 5 services are visible in the application's Compose file, + [docker-compose.yml]( + https://@@GITREPO@@/blob/master/dockercoins/docker-compose.yml) + +--- + +## How DockerCoins works + +- `worker` invokes web service `rng` to generate random bytes + +- `worker` invokes web service `hasher` to hash these bytes + +- `worker` does this in an infinite loop + +- every second, `worker` updates `redis` to indicate how many loops were done + +- `webui` queries `redis`, and computes and exposes "hashing speed" in our browser + +*(See diagram on next slide!)* + +--- + +class: pic + +![Diagram showing the 5 containers of the applications](images/dockercoins-diagram.svg) + +--- + +## Service discovery in container-land + +How does each service find out the address of the other ones? + +-- + +- We do not hard-code IP addresses in the code + +- We do not hard-code FQDNs in the code, either + +- We just connect to a service name, and container-magic does the rest + + (And by container-magic, we mean "a crafty, dynamic, embedded DNS server") + +--- + +## Example in `worker/worker.py` + +```python +redis = Redis("`redis`") + + +def get_random_bytes(): + r = requests.get("http://`rng`/32") + return r.content + + +def hash_bytes(data): + r = requests.post("http://`hasher`/", + data=data, + headers={"Content-Type": "application/octet-stream"}) +``` + +(Full source code available [here]( +https://@@GITREPO@@/blob/8279a3bce9398f7c1a53bdd95187c53eda4e6435/dockercoins/worker/worker.py#L17 +)) + +--- + +## Show me the code! + +- You can check the GitHub repository with all the materials of this workshop: +
https://@@GITREPO@@ + +- The application is in the [dockercoins]( + https://@@GITREPO@@/tree/master/dockercoins) + subdirectory + +- The Compose file ([docker-compose.yml]( + https://@@GITREPO@@/blob/master/dockercoins/docker-compose.yml)) + lists all 5 services + +- `redis` is using an official image from the Docker Hub + +- `hasher`, `rng`, `worker`, `webui` are each built from a Dockerfile + +- Each service's Dockerfile and source code is in its own directory + + (`hasher` is in the [hasher](https://@@GITREPO@@/blob/master/dockercoins/hasher/) directory, + `rng` is in the [rng](https://@@GITREPO@@/blob/master/dockercoins/rng/) + directory, etc.) + +--- + +## Our application at work + +- We can check the logs of our application's pods + +.exercise[ + +- Check the logs of the various components: + ```bash + kubectl logs deploy/worker + kubectl logs deploy/hasher + ``` + +] + +--- + +## Connecting to the web UI + +- "Logs are exciting and fun!" (No-one, ever) + +- The `webui` container exposes a web dashboard; let's view it + +.exercise[ + +- Check the NodePort allocated to the web UI: + ```bash + kubectl get svc webui + ``` + +- Open that in a web browser + +] + +A drawing area should show up, and after a few seconds, a blue +graph will appear. diff --git a/slides/k8s/localkubeconfig-k8s201.md b/slides/k8s/localkubeconfig-k8s201.md new file mode 100644 index 00000000..bc167bab --- /dev/null +++ b/slides/k8s/localkubeconfig-k8s201.md @@ -0,0 +1,181 @@ +# Controlling a Kubernetes cluster remotely + +- `kubectl` can be used either on cluster instances or outside the cluster + +- Here, we are going to use `kubectl` from our local machine + +--- + +## Requirements + +.warning[The exercises in this chapter should be done *on your local machine*.] + +- `kubectl` is officially available on Linux, macOS, Windows + + (and unofficially anywhere we can build and run Go binaries) + +- You may want to try Azure cloud shell if you are following along from: + + - a tablet or phone + + - a web-based terminal + + - an environment where you can't install and run new binaries + +--- + +## Installing `kubectl` + +- If you already have `kubectl` on your local machine, you can skip this + +.exercise[ + + + +- Download the `kubectl` binary from one of these links: + + [Linux](https://storage.googleapis.com/kubernetes-release/release/v1.15.0/bin/linux/amd64/kubectl) + | + [macOS](https://storage.googleapis.com/kubernetes-release/release/v1.15.0/bin/darwin/amd64/kubectl) + | + [Windows](https://storage.googleapis.com/kubernetes-release/release/v1.15.0/bin/windows/amd64/kubectl.exe) + +- On Linux and macOS, make the binary executable with `chmod +x kubectl` + + (And remember to run it with `./kubectl` or move it to your `$PATH`) + +] + +Note: if you are following along with a different platform (e.g. Linux on an architecture different from amd64, or with a phone or tablet), installing `kubectl` might be more complicated (or even impossible) so check with us about cloud shell. + +--- + +## Testing `kubectl` + +- Check that `kubectl` works correctly + + (before even trying to connect to a remote cluster!) + +.exercise[ + +- Ask `kubectl` to show its version number: + ```bash + kubectl version --client + ``` + +] + +The output should look like this: +``` +Client Version: version.Info{Major:"1", Minor:"15", GitVersion:"v1.15.0", +GitCommit:"e8462b5b5dc2584fdcd18e6bcfe9f1e4d970a529", GitTreeState:"clean", +BuildDate:"2019-06-19T16:40:16Z", GoVersion:"go1.12.5", Compiler:"gc", +Platform:"darwin/amd64"} +``` + +--- + +## Preserving the existing `~/.kube/config` + +- If you already have a `~/.kube/config` file, rename it + + (we are going to overwrite it in the following slides!) + +- If you never used `kubectl` on your machine before: nothing to do! + +.exercise[ + +- Make a copy of `~/.kube/config`; if you are using macOS or Linux, you can do: + ```bash + cp ~/.kube/config ~/.kube/config.before.training + ``` + +- If you are using Windows, you will need to adapt this command + +] + +--- + +## Connecting to your AKS cluster + +[fill in] + +--- + +## Let's look at your cluster! + + +- First, inspect the config + ```bash + kubectl config view + ``` + +- Look for the `server:` address that matches your new cluster + +``` +- cluster: + certificate-authority-data: DATA+OMITTED + server: https://aks-test-c-aks-test-group-0d35f7-28c7d691.hcp.eastus.azmk8s.io:443 + name: aks-test-cluster +``` + +--- + +class: extra-details + +## What if we get a certificate error? + +- Generally, the Kubernetes API uses a certificate that is valid for: + + - `kubernetes` + - `kubernetes.default` + - `kubernetes.default.svc` + - `kubernetes.default.svc.cluster.local` + - the ClusterIP address of the `kubernetes` service + - the hostname of the node hosting the control plane + - the IP address of the node hosting the control plane + +- On most clouds, the IP address of the node is an internal IP address + +- ... And we are going to connect over the external IP address + +- ... And that external IP address was not used when creating the certificate! + +--- + +class: extra-details + +## Working around the certificate error + +- We need to tell `kubectl` to skip TLS verification + + (only do this with testing clusters, never in production!) + +- The following command will do the trick: + ```bash + kubectl config set-cluster --insecure-skip-tls-verify + ``` + +--- + +## Checking that we can connect to the cluster + +- We can now run a couple of trivial commands to check that all is well + +.exercise[ + +- Check the versions of the local client and remote server: + ```bash + kubectl version + ``` + +It is okay if you have a newer client than what is available on the server. + +- View the nodes of the cluster: + ```bash + kubectl get nodes + ``` + +] + +We can now utilize the cluster exactly as if we're logged into a node, except that it's remote. diff --git a/slides/k8s/prereqs-k8s201.md b/slides/k8s/prereqs-k8s201.md index 970ce03f..b000c7af 100644 --- a/slides/k8s/prereqs-k8s201.md +++ b/slides/k8s/prereqs-k8s201.md @@ -44,7 +44,9 @@ - We will connect to these clusters with `kubectl` - (if you don't have `kubectl` installed, install it **now!**) +- If you don't have `kubectl` installed, we'll explain how to install it shortly + +- You will also want to install `jq` --- From 15c8fe5e39e7d45ad48cdaf7cb664b0a52df81a5 Mon Sep 17 00:00:00 2001 From: Bridget Kromhout Date: Thu, 11 Jul 2019 13:55:27 -0500 Subject: [PATCH 11/16] Updates --- slides/k8s-201.yml | 2 +- slides/k8s/resource-limits-k8s201.md | 522 +++++++++++++++++++++++++++ 2 files changed, 523 insertions(+), 1 deletion(-) create mode 100644 slides/k8s/resource-limits-k8s201.md diff --git a/slides/k8s-201.yml b/slides/k8s-201.yml index c6566c69..ed43023d 100644 --- a/slides/k8s-201.yml +++ b/slides/k8s-201.yml @@ -26,7 +26,7 @@ chapters: - - k8s/healthchecks.md - k8s/kubercoins-k8s201.md - k8s/authn-authz-k8s201.md -- - k8s/resource-limits.md +- - k8s/resource-limits-k8s201.md - k8s/metrics-server.md - - k8s/cluster-sizing.md - k8s/horizontal-pod-autoscaler.md diff --git a/slides/k8s/resource-limits-k8s201.md b/slides/k8s/resource-limits-k8s201.md new file mode 100644 index 00000000..e394a8c8 --- /dev/null +++ b/slides/k8s/resource-limits-k8s201.md @@ -0,0 +1,522 @@ +# Resource Limits + +- We can attach resource indications to our pods + + (or rather: to the *containers* in our pods) + +- We can specify *limits* and/or *requests* + +- We can specify quantities of CPU and/or memory + +--- + +## CPU vs memory + +- CPU is a *compressible resource* + + (it can be preempted immediately without adverse effect) + +- Memory is an *incompressible resource* + + (it needs to be swapped out to be reclaimed; and this is costly) + +- As a result, exceeding limits will have different consequences for CPU and memory + +--- + +## Exceeding CPU limits + +- CPU can be reclaimed instantaneously + + (in fact, it is preempted hundreds of times per second, at each context switch) + +- If a container uses too much CPU, it can be throttled + + (it will be scheduled less often) + +- The processes in that container will run slower + + (or rather: they will not run faster) + +--- + +## Exceeding memory limits + +- Memory needs to be swapped out before being reclaimed + +- "Swapping" means writing memory pages to disk, which is very slow + +- On a classic system, a process that swaps can get 1000x slower + + (because disk I/O is 1000x slower than memory I/O) + +- Exceeding the memory limit (even by a small amount) can reduce performance *a lot* + +- Kubernetes *does not support swap* (more on that later!) + +- Exceeding the memory limit will cause the container to be killed + + + +--- + +## Limits vs requests + +- Limits are "hard limits" (they can't be exceeded) + + - a container exceeding its memory limit is killed + + - a container exceeding its CPU limit is throttled + +- Requests are used for scheduling purposes + + - a container using *less* than what it requested will never be killed or throttled + + - the scheduler uses the requested sizes to determine placement + + - the resources requested by all pods on a node will never exceed the node size + +--- + +## Pod quality of service + +Each pod is assigned a QoS class (visible in `status.qosClass`). + +- If limits = requests: + + - as long as the container uses less than the limit, it won't be affected + + - if all containers in a pod have *(limits=requests)*, QoS is considered "Guaranteed" + +- If requests < limits: + + - as long as the container uses less than the request, it won't be affected + + - otherwise, it might be killed/evicted if the node gets overloaded + + - if at least one container has *(requests<limits)*, QoS is considered "Burstable" + +- If a pod doesn't have any request nor limit, QoS is considered "BestEffort" + +--- + +## Quality of service impact + +- When a node is overloaded, BestEffort pods are killed first + +- Then, Burstable pods that exceed their limits + +- Burstable and Guaranteed pods below their limits are never killed + + (except if their node fails) + +- If we only use Guaranteed pods, no pod should ever be killed + + (as long as they stay within their limits) + +(Pod QoS is also explained in [this page](https://kubernetes.io/docs/tasks/configure-pod-container/quality-service-pod/) of the Kubernetes documentation and in [this blog post](https://medium.com/google-cloud/quality-of-service-class-qos-in-kubernetes-bb76a89eb2c6).) + +--- + +## Where is my swap? + +- The semantics of memory and swap limits on Linux cgroups are complex + +- In particular, it's not possible to disable swap for a cgroup + + (the closest option is to [reduce "swappiness"](https://unix.stackexchange.com/questions/77939/turning-off-swapping-for-only-one-process-with-cgroups)) + +- The architects of Kubernetes wanted to ensure that Guaranteed pods never swap + +- The only solution was to disable swap entirely + +--- + +## Alternative point of view + +- Swap enables paging¹ of anonymous² memory + +- Even when swap is disabled, Linux will still page memory for: + + - executables, libraries + + - mapped files + +- Disabling swap *will reduce performance and available resources* + +- For a good time, read [kubernetes/kubernetes#53533](https://github.com/kubernetes/kubernetes/issues/53533) + +- Also read this [excellent blog post about swap](https://jvns.ca/blog/2017/02/17/mystery-swap/) + +¹Paging: reading/writing memory pages from/to disk to reclaim physical memory + +²Anonymous memory: memory that is not backed by files or blocks + +--- + +## Enabling swap anyway + +- If you don't care that pods are swapping, you can enable swap + +- You will need to add the flag `--fail-swap-on=false` to kubelet + + (otherwise, it won't start!) + +--- + +## Specifying resources + +- Resource requests are expressed at the *container* level + +- CPU is expressed in "virtual CPUs" + + (corresponding to the virtual CPUs offered by some cloud providers) + +- CPU can be expressed with a decimal value, or even a "milli" suffix + + (so 100m = 0.1) + +- Memory is expressed in bytes + +- Memory can be expressed with k, M, G, T, ki, Mi, Gi, Ti suffixes + + (corresponding to 10^3, 10^6, 10^9, 10^12, 2^10, 2^20, 2^30, 2^40) + +--- + +## Specifying resources in practice + +This is what the spec of a Pod with resources will look like: + +```yaml +containers: +- name: httpenv + image: jpetazzo/httpenv + resources: + limits: + memory: "100Mi" + cpu: "100m" + requests: + memory: "100Mi" + cpu: "10m" +``` + +This set of resources makes sure that this service won't be killed (as long as it stays below 100 MB of RAM), but allows its CPU usage to be throttled if necessary. + +--- + +## Default values + +- If we specify a limit without a request: + + the request is set to the limit + +- If we specify a request without a limit: + + there will be no limit + + (which means that the limit will be the size of the node) + +- If we don't specify anything: + + the request is zero and the limit is the size of the node + +*Unless there are default values defined for our namespace!* + +--- + +## We need default resource values + +- If we do not set resource values at all: + + - the limit is "the size of the node" + + - the request is zero + +- This is generally *not* what we want + + - a container without a limit can use up all the resources of a node + + - if the request is zero, the scheduler can't make a smart placement decision + +- To address this, we can set default values for resources + +- This is done with a LimitRange object + +--- + +# Defining min, max, and default resources + +- We can create LimitRange objects to indicate any combination of: + + - min and/or max resources allowed per pod + + - default resource *limits* + + - default resource *requests* + + - maximal burst ratio (*limit/request*) + +- LimitRange objects are namespaced + +- They apply to their namespace only + +--- + +## LimitRange example + +```yaml +apiVersion: v1 +kind: LimitRange +metadata: + name: my-very-detailed-limitrange +spec: + limits: + - type: Container + min: + cpu: "100m" + max: + cpu: "2000m" + memory: "1Gi" + default: + cpu: "500m" + memory: "250Mi" + defaultRequest: + cpu: "500m" +``` + +--- + +## Example explanation + +The YAML on the previous slide shows an example LimitRange object specifying very detailed limits on CPU usage, +and providing defaults on RAM usage. + +Note the `type: Container` line: in the future, +it might also be possible to specify limits +per Pod, but it's not [officially documented yet](https://github.com/kubernetes/website/issues/9585). + +--- + +## LimitRange details + +- LimitRange restrictions are enforced only when a Pod is created + + (they don't apply retroactively) + +- They don't prevent creation of e.g. an invalid Deployment or DaemonSet + + (but the pods will not be created as long as the LimitRange is in effect) + +- If there are multiple LimitRange restrictions, they all apply together + + (which means that it's possible to specify conflicting LimitRanges, +
preventing any Pod from being created) + +- If a LimitRange specifies a `max` for a resource but no `default`, +
that `max` value becomes the `default` limit too + +--- + +# Namespace quotas + +- We can also set quotas per namespace + +- Quotas apply to the total usage in a namespace + + (e.g. total CPU limits of all pods in a given namespace) + +- Quotas can apply to resource limits and/or requests + + (like the CPU and memory limits that we saw earlier) + +- Quotas can also apply to other resources: + + - "extended" resources (like GPUs) + + - storage size + + - number of objects (number of pods, services...) + +--- + +## Creating a quota for a namespace + +- Quotas are enforced by creating a ResourceQuota object + +- ResourceQuota objects are namespaced, and apply to their namespace only + +- We can have multiple ResourceQuota objects in the same namespace + +- The most restrictive values are used + +--- + +## Limiting total CPU/memory usage + +- The following YAML specifies an upper bound for *limits* and *requests*: + ```yaml + apiVersion: v1 + kind: ResourceQuota + metadata: + name: a-little-bit-of-compute + spec: + hard: + requests.cpu: "10" + requests.memory: 10Gi + limits.cpu: "20" + limits.memory: 20Gi + ``` + +These quotas will apply to the namespace where the ResourceQuota is created. + +--- + +## Limiting number of objects + +- The following YAML specifies how many objects of specific types can be created: + ```yaml + apiVersion: v1 + kind: ResourceQuota + metadata: + name: quota-for-objects + spec: + hard: + pods: 100 + services: 10 + secrets: 10 + configmaps: 10 + persistentvolumeclaims: 20 + services.nodeports: 0 + services.loadbalancers: 0 + count/roles.rbac.authorization.k8s.io: 10 + ``` + +(The `count/` syntax allows limiting arbitrary objects, including CRDs.) + +--- + +## YAML vs CLI + +- Quotas can be created with a YAML definition + +- ...Or with the `kubectl create quota` command + +.exercise[ + +- Create the following quota: + ```bash + kubectl create quota my-resource-quota \ + --hard=pods=300,limits.memory=300Gi + + ``` + +] + +- With both YAML and CLI form, the values are always under the `hard` section + + (there is no `soft` quota) + +--- + +## Viewing current usage + +When a ResourceQuota is created, we can see how much of it is used: + + +.exercise[ + +- Check how much of the ResourceQuota is used: + ```bash + kubectl describe resourcequota my-resource-quota + + ``` + +] + + +--- + +## Advanced quotas and PriorityClass + +- Since Kubernetes 1.12, it is possible to create PriorityClass objects + +- Pods can be assigned a PriorityClass + +- Quotas can be linked to a PriorityClass + +- This allows us to reserve resources for pods within a namespace + +- For more details, check [this documentation page](https://kubernetes.io/docs/concepts/policy/resource-quotas/#resource-quota-per-priorityclass) + +--- + +# Limiting resources in practice + +- We have at least three mechanisms: + + - requests and limits per Pod + + - LimitRange per namespace + + - ResourceQuota per namespace + +- Let's see a simple recommendation to get started with resource limits + +--- + +## Set a LimitRange + +- In each namespace, create a LimitRange object + +- Set a small default CPU request and CPU limit + + (e.g. "100m") + +- Set a default memory request and limit depending on your most common workload + + - for Java, Ruby: start with "1G" + + - for Go, Python, PHP, Node: start with "250M" + +- Set upper bounds slightly below your expected node size + + (80-90% of your node size, with at least a 500M memory buffer) + +--- + +## Set a ResourceQuota + +- In each namespace, create a ResourceQuota object + +- Set generous CPU and memory limits + + (e.g. half the cluster size if the cluster hosts multiple apps) + +- Set generous objects limits + + - these limits should not be here to constrain your users + + - they should catch a runaway process creating many resources + + - example: a custom controller creating many pods + +--- + +## Observe, refine, iterate + +- Observe the resource usage of your pods + + (we will see how in the next chapter) + +- Adjust individual pod limits + +- If you see trends: adjust the LimitRange + + (rather than adjusting every individual set of pod limits) + +- Observe the resource usage of your namespaces + + (with `kubectl describe resourcequota ...`) + +- Rinse and repeat regularly From 0d4b7d6c7ec7f21bf5266b43232d6bea747f6aa1 Mon Sep 17 00:00:00 2001 From: Bridget Kromhout Date: Thu, 11 Jul 2019 13:56:28 -0500 Subject: [PATCH 12/16] Fix typo --- slides/k8s/metrics-server.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slides/k8s/metrics-server.md b/slides/k8s/metrics-server.md index e7e4189a..b0023ef6 100644 --- a/slides/k8s/metrics-server.md +++ b/slides/k8s/metrics-server.md @@ -59,7 +59,7 @@ If it shows our nodes and their CPU and memory load, we're good! - Show resource usage across all containers: ```bash - kuebectl top pods --containers --all-namespaces + kubectl top pods --containers --all-namespaces ``` ] From 15023bd30a81002a88ba6452ed1ebf6ea2b9bf7f Mon Sep 17 00:00:00 2001 From: Bridget Kromhout Date: Thu, 11 Jul 2019 14:22:06 -0500 Subject: [PATCH 13/16] Remove quota so it doesn't cause confusion later --- slides/k8s/resource-limits-k8s201.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/slides/k8s/resource-limits-k8s201.md b/slides/k8s/resource-limits-k8s201.md index e394a8c8..fed14f22 100644 --- a/slides/k8s/resource-limits-k8s201.md +++ b/slides/k8s/resource-limits-k8s201.md @@ -434,6 +434,15 @@ When a ResourceQuota is created, we can see how much of it is used: ] +.exercise[ + +- Remove quota: + ```bash + kubectl delete quota my-resource-quota + + ``` + +] --- From a53a384aed196ec60fbc223d05355eab753fc67d Mon Sep 17 00:00:00 2001 From: Bridget Kromhout Date: Thu, 11 Jul 2019 20:39:14 -0500 Subject: [PATCH 14/16] jq --- slides/k8s/prereqs-k8s201.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slides/k8s/prereqs-k8s201.md b/slides/k8s/prereqs-k8s201.md index b000c7af..f92d88f1 100644 --- a/slides/k8s/prereqs-k8s201.md +++ b/slides/k8s/prereqs-k8s201.md @@ -46,7 +46,7 @@ - If you don't have `kubectl` installed, we'll explain how to install it shortly -- You will also want to install `jq` +- You will also want to install [jq](https://stedolan.github.io/jq/) --- From 4d6f336c7e0108cc932c27d1b01d71e7062585ac Mon Sep 17 00:00:00 2001 From: Bridget Kromhout Date: Thu, 11 Jul 2019 21:12:01 -0500 Subject: [PATCH 15/16] cloud shell and cli --- slides/k8s/localkubeconfig-k8s201.md | 51 ++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/slides/k8s/localkubeconfig-k8s201.md b/slides/k8s/localkubeconfig-k8s201.md index bc167bab..d11b983a 100644 --- a/slides/k8s/localkubeconfig-k8s201.md +++ b/slides/k8s/localkubeconfig-k8s201.md @@ -96,9 +96,56 @@ Platform:"darwin/amd64"} --- -## Connecting to your AKS cluster +## Connecting to your AKS cluster via Azure Cloud Shell -[fill in] +- open portal.azure.com in a browser +- auth with the info on your card + +- click `[>_]` in the top menu bar to open cloud shell + +.exercise[ + +- get your cluster credentials: + ```bash + az aks get-credentials -g $RESOURCE_GROUP -n $CLUSTER_NAME + ``` + +- check out your cluster: + ```bash + kubectl get nodes + ``` + +] + +--- + +## Connecting to your AKS cluster via local tools + +.exercise[ + +- install the [az CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest) + +- log in to azure: + ```bash + az login + ``` + +- get your cluster credentials: + ```bash + az aks get-credentials -g $RESOURCE_GROUP -n $CLUSTER_NAME + ``` + +- optionally, if you don't have kubectl: + ```bash + az aks install-cli + ``` + +- check out your cluster: + ```bash + kubectl get nodes + ``` + +] --- From b11221d33d8a9c62ac6dc294941f1aaebce3eb08 Mon Sep 17 00:00:00 2001 From: Bridget Kromhout Date: Thu, 11 Jul 2019 21:28:28 -0500 Subject: [PATCH 16/16] redirect for branch --- slides/_redirects | 1 + 1 file changed, 1 insertion(+) diff --git a/slides/_redirects b/slides/_redirects index e7606e61..8cdf4a88 100644 --- a/slides/_redirects +++ b/slides/_redirects @@ -2,6 +2,7 @@ #/ /kube-halfday.yml.html 200 #/ /kube-fullday.yml.html 200 #/ /kube-twodays.yml.html 200 +/ /k8s-201.yml.html 200 # And this allows to do "git clone https://container.training". /info/refs service=git-upload-pack https://github.com/jpetazzo/container.training/info/refs?service=git-upload-pack