diff --git a/prepare-vms/lib/infra/aws.sh b/prepare-vms/lib/infra/aws.sh index cb5f8fd0..f7721003 100644 --- a/prepare-vms/lib/infra/aws.sh +++ b/prepare-vms/lib/infra/aws.sh @@ -31,6 +31,7 @@ infra_start() { die "I could not find which AMI to use in this region. Try another region?" fi AWS_KEY_NAME=$(make_key_name) + AWS_INSTANCE_TYPE=${AWS_INSTANCE_TYPE-t3a.medium} sep "Starting instances" info " Count: $COUNT" @@ -38,10 +39,11 @@ infra_start() { info " Token/tag: $TAG" info " AMI: $AMI" info " Key name: $AWS_KEY_NAME" + info " Instance type: $AWS_INSTANCE_TYPE" result=$(aws ec2 run-instances \ --key-name $AWS_KEY_NAME \ --count $COUNT \ - --instance-type ${AWS_INSTANCE_TYPE-t2.medium} \ + --instance-type $AWS_INSTANCE_TYPE \ --client-token $TAG \ --block-device-mapping 'DeviceName=/dev/sda1,Ebs={VolumeSize=20}' \ --image-id $AMI) @@ -97,7 +99,7 @@ infra_disableaddrchecks() { } wait_until_tag_is_running() { - max_retry=50 + max_retry=100 i=0 done_count=0 while [[ $done_count -lt $COUNT ]]; do diff --git a/prepare-vms/setup-admin-clusters.sh b/prepare-vms/setup-admin-clusters.sh index a4719d7a..75717e52 100755 --- a/prepare-vms/setup-admin-clusters.sh +++ b/prepare-vms/setup-admin-clusters.sh @@ -30,9 +30,9 @@ TAG=$PREFIX-$SETTINGS --settings settings/$SETTINGS.yaml \ --count $((3*$STUDENTS)) +./workshopctl disableaddrchecks $TAG ./workshopctl deploy $TAG ./workshopctl kubebins $TAG -./workshopctl disableaddrchecks $TAG ./workshopctl cards $TAG SETTINGS=admin-kuberouter @@ -43,11 +43,15 @@ TAG=$PREFIX-$SETTINGS --settings settings/$SETTINGS.yaml \ --count $((3*$STUDENTS)) +./workshopctl disableaddrchecks $TAG ./workshopctl deploy $TAG ./workshopctl kubebins $TAG -./workshopctl disableaddrchecks $TAG ./workshopctl cards $TAG +#INFRA=infra/aws-us-west-1 + +export AWS_INSTANCE_TYPE=t3a.medium + SETTINGS=admin-test TAG=$PREFIX-$SETTINGS ./workshopctl start \ @@ -59,3 +63,4 @@ TAG=$PREFIX-$SETTINGS ./workshopctl deploy $TAG ./workshopctl kube $TAG 1.13.5 ./workshopctl cards $TAG + diff --git a/slides/alfun-2.yml b/slides/alfun-2.yml index e1caaeb7..9c974341 100644 --- a/slides/alfun-2.yml +++ b/slides/alfun-2.yml @@ -45,6 +45,7 @@ chapters: #- k8s/create-chart.md - k8s/rollout.md - - k8s/healthchecks.md + #- k8s/healthchecks-more.md - k8s/kubectlproxy.md - k8s/localkubeconfig.md - k8s/accessinternal.md diff --git a/slides/index.yaml b/slides/index.yaml index 668fdbe2..3a9b8517 100644 --- a/slides/index.yaml +++ b/slides/index.yaml @@ -1,3 +1,11 @@ +- date: [2019-11-04, 2019-11-05] + country: de + city: Berlin + event: Velocity + speaker: jpetazzo + title: Deploying and scaling applications with Kubernetes + attend: https://conferences.oreilly.com/velocity/vl-eu/public/schedule/detail/79109 + - date: 2019-11-13 country: fr city: Marseille diff --git a/slides/k8s/cloud-controller-manager.md b/slides/k8s/cloud-controller-manager.md index d0975278..0e6dd06f 100644 --- a/slides/k8s/cloud-controller-manager.md +++ b/slides/k8s/cloud-controller-manager.md @@ -20,15 +20,15 @@ - Configuring routing tables in the cloud network (specific to GCE) -- Updating node labels to indicate region, zone, instance type ... +- Updating node labels to indicate region, zone, instance type... - Obtain node name, internal and external addresses from cloud metadata service - Deleting nodes from Kubernetes when they're deleted in the cloud -- Managing *some* volumes (e.g. ELBs, AzureDisks ...) +- Managing *some* volumes (e.g. ELBs, AzureDisks...) - (Eventually, volumes will be managed by the CSI) + (Eventually, volumes will be managed by the Container Storage Interface) --- @@ -83,7 +83,7 @@ The list includes the following providers: ## Audience questions -- What kind of clouds are you using / planning to use? +- What kind of clouds are you using/planning to use? - What kind of details would you like to see in this section? @@ -105,7 +105,7 @@ The list includes the following providers: - When using managed clusters, this is done automatically -- There is very little documentation to write the configuration file +- There is very little documentation on writing the configuration file (except for OpenStack) @@ -123,7 +123,7 @@ The list includes the following providers: - To get these addresses, the node needs to communicate with the control plane -- ... Which means joining the cluster +- ...Which means joining the cluster (The problem didn't occur when cloud-specific code was running in kubelet: kubelet could obtain the required information directly from the cloud provider's metadata service.) diff --git a/slides/k8s/cluster-backup.md b/slides/k8s/cluster-backup.md index eb34a58c..048d09ee 100644 --- a/slides/k8s/cluster-backup.md +++ b/slides/k8s/cluster-backup.md @@ -6,7 +6,7 @@ - error recovery (human or process has altered or corrupted data) - - cloning environments (for testing, validation ...) + - cloning environments (for testing, validation...) - Let's see the strategies and tools available with Kubernetes! @@ -18,13 +18,13 @@ (it gives us replication primitives) -- Kubernetes helps us to clone / replicate environments +- Kubernetes helps us clone / replicate environments (all resources can be described with manifests) - Kubernetes *does not* help us with error recovery -- We still need to backup / snapshot our data: +- We still need to back up/snapshot our data: - with database backups (mysqldump, pgdump, etc.) @@ -58,7 +58,7 @@ - If our deployment system isn't fully automated, it should at least be documented -- Litmus test: how long does it take to deploy a cluster ... +- Litmus test: how long does it take to deploy a cluster... - for a senior engineer? @@ -66,7 +66,7 @@ - Does it require external intervention? - (e.g. provisioning servers, signing TLS certs ...) + (e.g. provisioning servers, signing TLS certs...) --- @@ -108,7 +108,7 @@ - For real applications: add resources (as YAML files) -- For applications deployed multiple times: Helm, Kustomize ... +- For applications deployed multiple times: Helm, Kustomize... (staging and production count as "multiple times") diff --git a/slides/k8s/cluster-upgrade.md b/slides/k8s/cluster-upgrade.md index 48197842..0b477ce9 100644 --- a/slides/k8s/cluster-upgrade.md +++ b/slides/k8s/cluster-upgrade.md @@ -287,8 +287,8 @@ - Download the configuration on each node, and upgrade kubelet: ```bash for N in 1 2 3; do - ssh node$N sudo kubeadm upgrade node config --kubelet-version v1.14.2 - ssh node $N sudo apt install kubelet=1.14.2-00 + ssh test$N sudo kubeadm upgrade node config --kubelet-version v1.14.2 + ssh test$N sudo apt install kubelet=1.14.2-00 done ``` ] diff --git a/slides/k8s/cni.md b/slides/k8s/cni.md index 3970f119..26a736b9 100644 --- a/slides/k8s/cni.md +++ b/slides/k8s/cni.md @@ -66,6 +66,8 @@ Look in each plugin's directory for its documentation. --- +class: extra-details + ## Conf vs conflist - There are two slightly different configuration formats @@ -276,7 +278,7 @@ class: extra-details - The address of the API server will be `http://A.B.C.D:8080` - (where `A.B.C.D` is the address of `kuberouter1`, running the control plane) + (where `A.B.C.D` is the public address of `kuberouter1`, running the control plane) .exercise[ @@ -453,7 +455,7 @@ We should see the local pod CIDR connected to `kube-bridge`, and the other nodes - Or try to exec into one of the kube-router pods: ```bash - kubectl -n kube-system exec kuber-router-xxxxx bash + kubectl -n kube-system exec kube-router-xxxxx bash ``` ] @@ -575,7 +577,7 @@ done ## Starting the route reflector -- Only do this if you are doing this on your own +- Only do this slide if you are doing this on your own - There is a Compose file in the `compose/frr-route-reflector` directory diff --git a/slides/k8s/csr-api.md b/slides/k8s/csr-api.md index 2673c4a1..f5bc4976 100644 --- a/slides/k8s/csr-api.md +++ b/slides/k8s/csr-api.md @@ -62,7 +62,7 @@ This is what I do if I want to obtain a certificate. 2. Create a Certificate Signing Request (CSR). - (The CSR contains the identity that I claim and an expiration date.) + (The CSR contains the identity that I claim and a public key.) 3. Send that CSR to the Certificate Authority (CA). @@ -231,7 +231,7 @@ For a user named `jean.doe`, we will have: - Let's use OpenSSL; it's not the best one, but it's installed everywhere (many people prefer cfssl, easyrsa, or other tools; that's fine too!) - + .exercise[ - Generate the key and certificate signing request: @@ -244,7 +244,7 @@ For a user named `jean.doe`, we will have: The command above generates: -- a 2048-bit RSA key, without DES encryption, stored in key.pem +- a 2048-bit RSA key, without encryption, stored in key.pem - a CSR for the name `jean.doe` in group `devs` --- @@ -345,7 +345,7 @@ The command above generates: kctx - ``` -- Retrieve the certificate from the CSR: +- Retrieve the updated CSR object and extract the certificate: ```bash kubectl get csr users:jean.doe \ -o jsonpath={.status.certificate} \ diff --git a/slides/k8s/dmuc.md b/slides/k8s/dmuc.md index d70972aa..17f60543 100644 --- a/slides/k8s/dmuc.md +++ b/slides/k8s/dmuc.md @@ -175,7 +175,7 @@ Success! ] -So far, so good. +We should get `No resources found.` and the `kubernetes` service, respectively. Note: the API server automatically created the `kubernetes` service entry. @@ -225,7 +225,7 @@ Success? ] -Our Deployment is in a bad shape: +Our Deployment is in bad shape: ``` NAME READY UP-TO-DATE AVAILABLE AGE deployment.apps/web 0/1 0 0 2m26s diff --git a/slides/k8s/healthchecks-more.md b/slides/k8s/healthchecks-more.md new file mode 100644 index 00000000..887ce5b3 --- /dev/null +++ b/slides/k8s/healthchecks-more.md @@ -0,0 +1,393 @@ +## Questions to ask before adding healthchecks + +- Do we want liveness, readiness, both? + + (sometimes, we can use the same check, but with different failure thresholds) + +- Do we have existing HTTP endpoints that we can use? + +- Do we need to add new endpoints, or perhaps use something else? + +- Are our healthchecks likely to use resources and/or slow down the app? + +- Do they depend on additional services? + + (this can be particularly tricky, see next slide) + +--- + +## Healthchecks and dependencies + +- A good healthcheck should always indicate the health of the service itself + +- It should not be affected by the state of the service's dependencies + +- Example: a web server requiring a database connection to operate + + (make sure that the healthcheck can report "OK" even if the database is down; +
+ because it won't help us to restart the web server if the issue is with the DB!) + +- Example: a microservice calling other microservices + +- Example: a worker process + + (these will generally require minor code changes to report health) + +--- + +## Adding healthchecks to an app + +- Let's add healthchecks to DockerCoins! + +- We will examine the questions of the previous slide + +- Then we will review each component individually to add healthchecks + +--- + +## Liveness, readiness, or both? + +- To answer that question, we need to see the app run for a while + +- Do we get temporary, recoverable glitches? + + → then use readiness + +- Or do we get hard lock-ups requiring a restart? + + → then use liveness + +- In the case of DockerCoins, we don't know yet! + +- Let's pick liveness + +--- + +## Do we have HTTP endpoints that we can use? + +- Each of the 3 web services (hasher, rng, webui) has a trivial route on `/` + +- These routes: + + - don't seem to perform anything complex or expensive + + - don't seem to call other services + +- Perfect! + + (See next slides for individual details) + +--- + +- [hasher.rb](https://github.com/jpetazzo/container.training/blob/master/dockercoins/hasher/hasher.rb) + ```ruby + get '/' do + "HASHER running on #{Socket.gethostname}\n" + end + ``` + +- [rng.py](https://github.com/jpetazzo/container.training/blob/master/dockercoins/rng/rng.py) + ```python + @app.route("/") + def index(): + return "RNG running on {}\n".format(hostname) + ``` + +- [webui.js](https://github.com/jpetazzo/container.training/blob/master/dockercoins/webui/webui.js) + ```javascript + app.get('/', function (req, res) { + res.redirect('/index.html'); + }); + ``` + +--- + +## Running DockerCoins + +- We will run DockerCoins in a new, separate namespace + +- We will use a set of YAML manifests and pre-built images + +- We will add our new liveness probe to the YAML of the `rng` DaemonSet + +- Then, we will deploy the application + +--- + +## Creating a new namespace + +- This will make sure that we don't collide / conflict with previous exercises + +.exercise[ + +- Create the yellow namespace: + ```bash + kubectl create namespace yellow + ``` + +- Switch to that namespace: + ```bash + kns yellow + ``` + +] + +--- + +## Retrieving DockerCoins manifests + +- All the manifests that we need are on a convenient repository: + + https://github.com/jpetazzo/kubercoins + +.exercise[ + +- Clone that repository: + ```bash + cd ~ + git clone https://github.com/jpetazzo/kubercoins + ``` + +- Change directory to the repository: + ```bash + cd kubercoins + ``` + +] + +--- + +## A simple HTTP liveness probe + +This is what our liveness probe should look like: + +```yaml +containers: +- name: ... +image: ... +livenessProbe: + httpGet: + path: / + port: 80 + initialDelaySeconds: 30 + periodSeconds: 5 +``` + +This will give 30 seconds to the service to start. (Way more than necessary!) +
+It will run the probe every 5 seconds. +
+It will use the default timeout (1 second). +
+It will use the default failure threshold (3 failed attempts = dead). +
+It will use the default success threshold (1 successful attempt = alive). + +--- + +## Adding the liveness probe + +- Let's add the liveness probe, then deploy DockerCoins + +.exercise[ + +- Edit `rng-daemonset.yaml` and add the liveness probe + ```bash + vim rng-daemonset.yaml + ``` + +- Load the YAML for all the resources of DockerCoins: + ```bash + kubectl apply -f . + ``` + +] + +--- + +## Testing the liveness probe + +- The rng service needs 100ms to process a request + + (because it is single-threaded and sleeps 0.1s in each request) + +- The probe timeout is set to 1 second + +- If we send more than 10 requests per second per backend, it will break + +- Let's generate traffic and see what happens! + +.exercise[ + +- Get the ClusterIP address of the rng service: + ```bash + kubectl get svc rng + ``` + +] + +--- + +## Monitoring the rng service + +- Each command below will show us what's happening on a different level + +.exercise[ + +- In one window, monitor cluster events: + ```bash + kubectl get events -w + ``` + +- In another window, monitor the response time of rng: + ```bash + httping `` + ``` + +- In another window, monitor pods status: + ```bash + kubectl get pods -w + ``` + +] + +--- + +## Generating traffic + +- Let's use `ab` to send concurrent requests to rng + +.exercise[ + +- In yet another window, generate traffic: + ```bash + ab -c 10 -n 1000 http://``/1 + ``` + +- Experiment with higher values of `-c` and see what happens + +] + +- The `-c` parameter indicates the number of concurrent requests + +- The final `/1` is important to generate actual traffic + + (otherwise we would use the ping endpoint, which doesn't sleep 0.1s per request) + +--- + +## Discussion + +- Above a given threshold, the liveness probe starts failing + + (about 10 concurrent requests per backend should be plenty enough) + +- When the liveness probe fails 3 times in a row, the container is restarted + +- During the restart, there is *less* capacity available + +- ... Meaning that the other backends are likely to timeout as well + +- ... Eventually causing all backends to be restarted + +- ... And each fresh backend gets restarted, too + +- This goes on until the load goes down, or we add capacity + +*This wouldn't be a good healthcheck in a real application!* + +--- + +## Better healthchecks + +- We need to make sure that the healthcheck doesn't trip when + performance degrades due to external pressure + +- Using a readiness check would have fewer effects + + (but it would still be an imperfect solution) + +- A possible combination: + + - readiness check with a short timeout / low failure threshold + + - liveness check with a longer timeout / higher failure treshold + +--- + +## Healthchecks for redis + +- A liveness probe is enough + + (it's not useful to remove a backend from rotation when it's the only one) + +- We could use an exec probe running `redis-cli ping` + +--- + +class: extra-details + +## Exec probes and zombies + +- When using exec probes, we should make sure that we have a *zombie reaper* + + 🤔🧐🧟 Wait, what? + +- When a process terminates, its parent must call `wait()`/`waitpid()` + + (this is how the parent process retrieves the child's exit status) + +- In the meantime, the process is in *zombie* state + + (the process state will show as `Z` in `ps`, `top` ...) + +- When a process is killed, its children are *orphaned* and attached to PID 1 + +- PID 1 has the responsibility of *reaping* these processes when they terminate + +- OK, but how does that affect us? + +--- + +class: extra-details + +## PID 1 in containers + +- On ordinary systems, PID 1 (`/sbin/init`) has logic to reap processes + +- In containers, PID 1 is typically our application process + + (e.g. Apache, the JVM, NGINX, Redis ...) + +- These *do not* take care of reaping orphans + +- If we use exec probes, we need to add a process reaper + +- We can add [tini](https://github.com/krallin/tini) to our images + +- Or [share the PID namespace between containers of a pod](https://kubernetes.io/docs/tasks/configure-pod-container/share-process-namespace/) + + (and have gcr.io/pause take care of the reaping) + +--- + +## Healthchecks for worker + +- Readiness isn't useful + + (because worker isn't a backend for a service) + +- Liveness may help us restart a broken worker, but how can we check it? + +- Embedding an HTTP server is an option + + (but it has a high potential for unwanted side effects and false positives) + +- Using a "lease" file can be relatively easy: + + - touch a file during each iteration of the main loop + + - check the timestamp of that file from an exec probe + +- Writing logs (and checking them from the probe) also works diff --git a/slides/k8s/lastwords-admin.md b/slides/k8s/lastwords-admin.md index 4fbde579..df9bab7a 100644 --- a/slides/k8s/lastwords-admin.md +++ b/slides/k8s/lastwords-admin.md @@ -48,7 +48,7 @@ - Acknowledge that a lot of tasks are outsourced - (e.g. if we add "buy / rack / provision machines" in that list) + (e.g. if we add "buy/rack/provision machines" in that list) --- @@ -122,7 +122,7 @@ (YAML, Helm charts, Kustomize ...) -- Team "run" adjusts some parameters and monitors the application +- Team "run" adjusts some parameters and monitors the application ✔️ parity between dev and prod environments @@ -150,7 +150,7 @@ - do we reward on-call duty without encouraging hero syndrome? - - do we give resources (time, money) to people to learn? + - do we give people resources (time, money) to learn? --- @@ -183,9 +183,9 @@ are a few tools that can help us.* - If cloud: public vs. private -- Which vendor / distribution to pick? +- Which vendor/distribution to pick? -- Which versions / features to enable? +- Which versions/features to enable? --- @@ -205,6 +205,6 @@ are a few tools that can help us.* - Transfer knowledge - (make sure everyone is on the same page / same level) + (make sure everyone is on the same page/level) - Iterate! diff --git a/slides/k8s/multinode.md b/slides/k8s/multinode.md index 0efb7dc5..af6b6194 100644 --- a/slides/k8s/multinode.md +++ b/slides/k8s/multinode.md @@ -96,7 +96,7 @@ class: extra-details - We need to generate a `kubeconfig` file for kubelet -- This time, we need to put the IP address of `kubenet1` +- This time, we need to put the public IP address of `kubenet1` (instead of `localhost` or `127.0.0.1`) @@ -195,7 +195,7 @@ class: extra-details ## Check our pods -- The pods will be scheduled to the nodes +- The pods will be scheduled on the nodes - The nodes will pull the `nginx` image, and start the pods @@ -325,7 +325,7 @@ class: extra-details - We will add the `--network-plugin` and `--pod-cidr` flags -- We all have a "cluster number" (let's call that `C`) +- We all have a "cluster number" (let's call that `C`) printed on your VM info card - We will use pod CIDR `10.C.N.0/24` (where `N` is the node number: 1, 2, 3) @@ -480,6 +480,23 @@ Sometimes it works, sometimes it doesn't. Why? ```bash kubectl get nodes -o wide ``` + +--- + +## Firewalling + +- By default, Docker prevents containers from using arbitrary IP addresses + + (by setting up iptables rules) + +- We need to allow our containers to use our pod CIDR + +- For simplicity, we will insert a blanket iptables rule allowing all traffic: + + `iptables -I FORWARD -j ACCEPT` + +- This has to be done on every node + --- ## Setting up routing @@ -488,6 +505,8 @@ Sometimes it works, sometimes it doesn't. Why? - Create all the routes on all the nodes +- Insert the iptables rule allowing traffic + - Check that you can ping all the pods from one of the nodes - Check that you can `curl` the ClusterIP of the Service successfully diff --git a/slides/k8s/operators-design.md b/slides/k8s/operators-design.md new file mode 100644 index 00000000..c7b31d18 --- /dev/null +++ b/slides/k8s/operators-design.md @@ -0,0 +1,356 @@ +## What does it take to write an operator? + +- Writing a quick-and-dirty operator, or a POC/MVP, is easy + +- Writing a robust operator is hard + +- We will describe the general idea + +- We will identify some of the associated challenges + +- We will list a few tools that can help us + +--- + +## Top-down vs. bottom-up + +- Both approaches are possible + +- Let's see what they entail, and their respective pros and cons + +--- + +## Top-down approach + +- Start with high-level design (see next slide) + +- Pros: + + - can yield cleaner design that will be more robust + +- Cons: + + - must be able to anticipate all the events that might happen + + - design will be better only to the extend of what we anticipated + + - hard to anticipate if we don't have production experience + +--- + +## High-level design + +- What are we solving? + + (e.g.: geographic databases backed by PostGIS with Redis caches) + +- What are our use-cases, stories? + + (e.g.: adding/resizing caches and read replicas; load balancing queries) + +- What kind of outage do we want to address? + + (e.g.: loss of individual node, pod, volume) + +- What are our *non-features*, the things we don't want to address? + + (e.g.: loss of datacenter/zone; differentiating between read and write queries; +
+ cache invalidation; upgrading to newer major versions of Redis, PostGIS, PostgreSQL) + +--- + +## Low-level design + +- What Custom Resource Definitions do we need? + + (one, many?) + +- How will we store configuration information? + + (part of the CRD spec fields, annotations, other?) + +- Do we need to store state? If so, where? + + - state that is small and doesn't change much can be stored via the Kubernetes API +
+ (e.g.: leader information, configuration, credentials) + + - things that are big and/or change a lot should go elsewhere +
+ (e.g.: metrics, bigger configuration file like GeoIP) + +--- + +class: extra-details + +## What can we store via the Kubernetes API? + +- The API server stores most Kubernetes resources in etcd + +- Etcd is designed for reliability, not for performance + +- If our storage needs exceed what etcd can offer, we need to use something else: + + - either directly + + - or by extending the API server +
(for instance by using the agregation layer, like [metrics server](https://github.com/kubernetes-incubator/metrics-server) does) + +--- + +## Bottom-up approach + +- Start with existing Kubernetes resources (Deployment, Stateful Set...) + +- Run the system in production + +- Add scripts, automation, to facilitate day-to-day operations + +- Turn the scripts into an operator + +- Pros: simpler to get started; reflects actual use-cases + +- Cons: can result in convoluted designs requiring extensive refactor + +--- + +## General idea + +- Our operator will watch its CRDs *and associated resources* + +- Drawing state diagrams and finite state automata helps a lot + +- It's OK if some transitions lead to a big catch-all "human intervention" + +- Over time, we will learn about new failure modes and add to these diagrams + +- It's OK to start with CRD creation / deletion and prevent any modification + + (that's the easy POC/MVP we were talking about) + +- *Presentation* and *validation* will help our users + + (more on that later) + +--- + +## Challenges + +- Reacting to infrastructure disruption can seem hard at first + +- Kubernetes gives us a lot of primitives to help: + + - Pods and Persistent Volumes will *eventually* recover + + - Stateful Sets give us easy ways to "add N copies" of a thing + +- The real challenges come with configuration changes + + (i.e., what to do when our users update our CRDs) + +- Keep in mind that [some] of the [largest] cloud [outages] haven't been caused by [natural catastrophes], or even code bugs, but by configuration changes + +[some]: https://www.datacenterdynamics.com/news/gcp-outage-mainone-leaked-google-cloudflare-ip-addresses-china-telecom/ +[largest]: https://aws.amazon.com/message/41926/ +[outages]: https://aws.amazon.com/message/65648/ +[natural catastrophes]: https://www.datacenterknowledge.com/amazon/aws-says-it-s-never-seen-whole-data-center-go-down + +--- + +## Configuration changes + +- It is helpful to analyze and understand how Kubernetes controllers work: + + - watch resource for modifications + + - compare desired state (CRD) and current state + + - issue actions to converge state + +- Configuration changes will probably require *another* state diagram or FSA + +- Again, it's OK to have transitions labeled as "unsupported" + + (i.e. reject some modifications because we can't execute them) + +--- + +## Tools + +- CoreOS / RedHat Operator Framework + + [GitHub](https://github.com/operator-framework) + | + [Blog](https://developers.redhat.com/blog/2018/12/18/introduction-to-the-kubernetes-operator-framework/) + | + [Intro talk](https://www.youtube.com/watch?v=8k_ayO1VRXE) + | + [Deep dive talk](https://www.youtube.com/watch?v=fu7ecA2rXmc) + +- Zalando Kubernetes Operator Pythonic Framework (KOPF) + + [GitHub](https://github.com/zalando-incubator/kopf) + | + [Docs](https://kopf.readthedocs.io/) + | + [Step-by-step tutorial](https://kopf.readthedocs.io/en/stable/walkthrough/problem/) + +- Mesosphere Kubernetes Universal Declarative Operator (KUDO) + + [GitHub](https://github.com/kudobuilder/kudo) + | + [Blog](https://mesosphere.com/blog/announcing-maestro-a-declarative-no-code-approach-to-kubernetes-day-2-operators/) + | + [Docs](https://kudo.dev/) + | + [Zookeeper example](https://github.com/kudobuilder/frameworks/tree/master/repo/stable/zookeeper) + +--- + +## Validation + +- By default, a CRD is "free form" + + (we can put pretty much anything we want in it) + +- When creating a CRD, we can provide an OpenAPI v3 schema + ([Example](https://github.com/amaizfinance/redis-operator/blob/master/deploy/crds/k8s_v1alpha1_redis_crd.yaml#L34)) + +- The API server will then validate resources created/edited with this schema + +- If we need a stronger validation, we can use a Validating Admission Webhook: + + - run an [admission webhook server](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#write-an-admission-webhook-server) to receive validation requests + + - register the webhook by creating a [ValidatingWebhookConfiguration](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#configure-admission-webhooks-on-the-fly) + + - each time the API server receives a request matching the configuration, +
the request is sent to our server for validation + +--- + +## Presentation + +- By default, `kubectl get mycustomresource` won't display much information + + (just the name and age of each resource) + +- When creating a CRD, we can specify additional columns to print + ([Example](https://github.com/amaizfinance/redis-operator/blob/master/deploy/crds/k8s_v1alpha1_redis_crd.yaml#L6), + [Docs](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#additional-printer-columns)) + +- By default, `kubectl describe mycustomresource` will also be generic + +- `kubectl describe` can show events related to our custom resources + + (for that, we need to create Event resources, and fill the `involvedObject` field) + +- For scalable resources, we can define a `scale` sub-resource + +- This will enable the use of `kubectl scale` and other scaling-related operations + +--- + +## About scaling + +- It is possible to use the HPA (Horizontal Pod Autoscaler) with CRDs + +- But it is not always desirable + +- The HPA works very well for homogenous, stateless workloads + +- For other workloads, your mileage may vary + +- Some systems can scale across multiple dimensions + + (for instance: increase number of replicas, or number of shards?) + +- If autoscaling is desired, the operator will have to take complex decisions + + (example: Zalando's Elasticsearch Operator ([Video](https://www.youtube.com/watch?v=lprE0J0kAq0))) + +--- + +## Versioning + +- As our operator evolves over time, we may have to change the CRD + + (add, remove, change fields) + +- Like every other resource in Kubernetes, [custom resources are versioned](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definition-versioning/ +) + +- When creating a CRD, we need to specify a *list* of versions + +- Versions can be marked as `stored` and/or `served` + +--- + +## Stored version + +- Exactly one version has to be marked as the `stored` version + +- As the name implies, it is the one that will be stored in etcd + +- Resources in storage are never converted automatically + + (we need to read and re-write them ourselves) + +- Yes, this means that we can have different versions in etcd at any time + +- Our code needs to handle all the versions that still exist in storage + +--- + +## Served versions + +- By default, the Kubernetes API will serve resources "as-is" + + (using their stored version) + +- It will assume that all versions are compatible storage-wise + + (i.e. that the spec and fields are compatible between versions) + +- We can provide [conversion webhooks](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definition-versioning/#webhook-conversion) to "translate" requests + + (the alternative is to upgrade all stored resources and stop serving old versions) + +--- + +## Operator reliability + +- Remember that the operator itself must be resilient + + (e.g.: the node running it can fail) + +- Our operator must be able to restart and recover gracefully + +- Do not store state locally + + (unless we can reconstruct that state when we restart) + +- As indicated earlier, we can use the Kubernetes API to store data: + + - in the custom resources themselves + + - in other resources' annotations + +--- + +## Beyond CRDs + +- CRDs cannot use custom storage (e.g. for time series data) + +- CRDs cannot support arbitrary subresources (like logs or exec for Pods) + +- CRDs cannot support protobuf (for faster, more efficient communication) + +- If we need these things, we can use the [aggregation layer](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/apiserver-aggregation/) instead + +- The aggregation layer proxies all requests below a specific path to another server + + (this is used e.g. by the metrics server) + +- [This documentation page](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#choosing-a-method-for-adding-custom-resources) compares the features of CRDs and API aggregation diff --git a/slides/k8s/operators.md b/slides/k8s/operators.md new file mode 100644 index 00000000..fdf6bbf6 --- /dev/null +++ b/slides/k8s/operators.md @@ -0,0 +1,389 @@ +# Operators + +- Operators are one of the many ways to extend Kubernetes + +- We will define operators + +- We will see how they work + +- We will install a specific operator (for ElasticSearch) + +- We will use it to provision an ElasticSearch cluster + +--- + +## What are operators? + +*An operator represents **human operational knowledge in software,** +
+to reliably manage an application. +— [CoreOS](https://coreos.com/blog/introducing-operators.html)* + +Examples: + +- Deploying and configuring replication with MySQL, PostgreSQL ... + +- Setting up Elasticsearch, Kafka, RabbitMQ, Zookeeper ... + +- Reacting to failures when intervention is needed + +- Scaling up and down these systems + +--- + +## What are they made from? + +- Operators combine two things: + + - Custom Resource Definitions + + - controller code watching the corresponding resources and acting upon them + +- A given operator can define one or multiple CRDs + +- The controller code (control loop) typically runs within the cluster + + (running as a Deployment with 1 replica is a common scenario) + +- But it could also run elsewhere + + (nothing mandates that the code run on the cluster, as long as it has API access) + +--- + +## Why use operators? + +- Kubernetes gives us Deployments, StatefulSets, Services ... + +- These mechanisms give us building blocks to deploy applications + +- They work great for services that are made of *N* identical containers + + (like stateless ones) + +- They also work great for some stateful applications like Consul, etcd ... + + (with the help of highly persistent volumes) + +- They're not enough for complex services: + + - where different containers have different roles + + - where extra steps have to be taken when scaling or replacing containers + +--- + +## Use-cases for operators + +- Systems with primary/secondary replication + + Examples: MariaDB, MySQL, PostgreSQL, Redis ... + +- Systems where different groups of nodes have different roles + + Examples: ElasticSearch, MongoDB ... + +- Systems with complex dependencies (that are themselves managed with operators) + + Examples: Flink or Kafka, which both depend on Zookeeper + +--- + +## More use-cases + +- Representing and managing external resources + + (Example: [AWS Service Operator](https://operatorhub.io/operator/alpha/aws-service-operator.v0.0.1)) + +- Managing complex cluster add-ons + + (Example: [Istio operator](https://operatorhub.io/operator/beta/istio-operator.0.1.6)) + +- Deploying and managing our applications' lifecycles + + (more on that later) + +--- + +## How operators work + +- An operator creates one or more CRDs + + (i.e., it creates new "Kinds" of resources on our cluster) + +- The operator also runs a *controller* that will watch its resources + +- Each time we create/update/delete a resource, the controller is notified + + (we could write our own cheap controller with `kubectl get --watch`) + +--- + +## One operator in action + +- We will install the UPMC Enterprises ElasticSearch operator + +- This operator requires PersistentVolumes + +- We will install Rancher's [local path storage provisioner](https://github.com/rancher/local-path-provisioner) to automatically create these + +- Then, we will create an ElasticSearch resource + +- The operator will detect that resource and provision the cluster + +--- + +## Installing a Persistent Volume provisioner + +(This step can be skipped if you already have a dynamic volume provisioner.) + +- This provisioner creates Persistent Volumes backed by `hostPath` + + (local directories on our nodes) + +- It doesn't require anything special ... + +- ... But losing a node = losing the volumes on that node! + +.exercise[ + +- Install the local path storage provisioner: + ```bash + kubectl apply -f ~/container.training/k8s/local-path-storage.yaml + ``` + +] + +--- + +## Making sure we have a default StorageClass + +- The ElasticSearch operator will create StatefulSets + +- These StatefulSets will instantiate PersistentVolumeClaims + +- These PVCs need to be explicitly associated with a StorageClass + +- Or we need to tag a StorageClass to be used as the default one + +.exercise[ + +- List StorageClasses: + ```bash + kubectl get storageclasses + ``` + +] + +We should see the `local-path` StorageClass. + +--- + +## Setting a default StorageClass + +- This is done by adding an annotation to the StorageClass: + + `storageclass.kubernetes.io/is-default-class: true` + +.exercise[ + +- Tag the StorageClass so that it's the default one: + ```bash + kubectl annotate storageclass local-path \ + storageclass.kubernetes.io/is-default-class=true + ``` + +- Check the result: + ```bash + kubectl get storageclasses + ``` + +] + +Now, the StorageClass should have `(default)` next to its name. + +--- + +## Install the ElasticSearch operator + +- The operator needs: + + - a Deployment for its controller + - a ServiceAccount, ClusterRole, ClusterRoleBinding for permissions + - a Namespace + +- We have grouped all the definitions for these resources in a YAML file + +.exercise[ + +- Install the operator: + ```bash + kubectl apply -f ~/container.training/k8s/elasticsearch-operator.yaml + ``` + +] + +--- + +## Wait for the operator to be ready + +- Some operators require to create their CRDs separately + +- This operator will create its CRD itself + + (i.e. the CRD is not listed in the YAML that we applied earlier) + +.exercise[ + +- Wait until the `elasticsearchclusters` CRD shows up: + ```bash + kubectl get crds + ``` + +] + +--- + +## Create an ElasticSearch resource + +- We can now create a resource with `kind: ElasticsearchCluster` + +- The YAML for that resource will specify all the desired parameters: + + - how many nodes do we want of each type (client, master, data) + - image to use + - add-ons (kibana, cerebro, ...) + - whether to use TLS or not + - etc. + +.exercise[ + +- Create our ElasticSearch cluster: + ```bash + kubectl apply -f ~/container.training/k8s/elasticsearch-cluster.yaml + ``` + +] + +--- + +## Operator in action + +- Over the next minutes, the operator will create: + + - StatefulSets (one for master nodes, one for data nodes) + + - Deployments (for client nodes; and for add-ons like cerebro and kibana) + + - Services (for all these pods) + +.exercise[ + +- Wait for all the StatefulSets to be fully up and running: + ```bash + kubectl get statefulsets -w + ``` + +] + +--- + +## Connecting to our cluster + +- Since connecting directly to the ElasticSearch API is a bit raw, +
we'll connect to the cerebro frontend instead + +.exercise[ + +- Edit the cerebro service to change its type from ClusterIP to NodePort: + ```bash + kubectl patch svc cerebro-es -p "spec: { type: NodePort }" + ``` + +- Retrieve the NodePort that was allocated: + ```bash + kubectl get svc cerebreo-es + ``` + +- Connect to that port with a browser + +] + +--- + +## (Bonus) Setup filebeat + +- Let's send some data to our brand new ElasticSearch cluster! + +- We'll deploy a filebeat DaemonSet to collect node logs + +.exercise[ + +- Deploy filebeat: + ```bash + kubectl apply -f ~/container.training/k8s/filebeat.yaml + ``` + +] + +We should see at least one index being created in cerebro. + +--- + +## (Bonus) Access log data with kibana + +- Let's expose kibana (by making kibana-es a NodePort too) + +- Then access kibana + +- We'll need to configure kibana indexes + +--- + +## Deploying our apps with operators + +- It is very simple to deploy with `kubectl run` / `kubectl expose` + +- We can unlock more features by writing YAML and using `kubectl apply` + +- Kustomize or Helm let us deploy in multiple environments + + (and adjust/tweak parameters in each environment) + +- We can also use an operator to deploy our application + +--- + +## Pros and cons of deploying with operators + +- The app definition and configuration is persisted in the Kubernetes API + +- Multiple instances of the app can be manipulated with `kubectl get` + +- We can add labels, annotations to the app instances + +- Our controller can execute custom code for any lifecycle event + +- However, we need to write this controller + +- We need to be careful about changes + + (what happens when the resource `spec` is updated?) + +--- + +## Operators are not magic + +- Look at the ElasticSearch resource definition + + (`~/container.training/k8s/elasticsearch-cluster.yaml`) + +- What should happen if we flip the `use-tls` flag? Twice? + +- What should happen if we remove / re-add the kibana or cerebro sections? + +- What should happen if we change the number of nodes? + +- What if we want different images or parameters for the different nodes? + +*Operators can be very powerful, iff we know exactly the scenarios that they can handle.* diff --git a/slides/k8s/podsecuritypolicy.md b/slides/k8s/podsecuritypolicy.md index fd333919..97c721d1 100644 --- a/slides/k8s/podsecuritypolicy.md +++ b/slides/k8s/podsecuritypolicy.md @@ -212,7 +212,7 @@ - Have a look at the static pods: ```bash - ls -l /etc/kubernetes/manifest + ls -l /etc/kubernetes/manifests ``` - Edit the one corresponding to the API server: @@ -236,7 +236,7 @@ - Add `PodSecurityPolicy` - (It should read `--enable-admission-plugins=NodeRestriction,PodSecurityPolicy`) + It should read: `--enable-admission-plugins=NodeRestriction,PodSecurityPolicy` - Save, quit diff --git a/slides/k8s/setup-k8s.md b/slides/k8s/setup-k8s.md index 3800e78e..6b704ef0 100644 --- a/slides/k8s/setup-k8s.md +++ b/slides/k8s/setup-k8s.md @@ -90,4 +90,4 @@ - For a longer list, check the Kubernetes documentation:
- it has a great guide to [pick the right solution](https://kubernetes.io/docs/setup/pick-right-solution/) to set up Kubernetes. + it has a great guide to [pick the right solution](https://kubernetes.io/docs/setup/#production-environment) to set up Kubernetes. diff --git a/slides/k8s/staticpods.md b/slides/k8s/staticpods.md index e95b0840..31ad23ea 100644 --- a/slides/k8s/staticpods.md +++ b/slides/k8s/staticpods.md @@ -18,7 +18,7 @@ ## A possible approach -- Since each component of the control plane can be replicated ... +- Since each component of the control plane can be replicated... - We could set up the control plane outside of the cluster @@ -39,9 +39,9 @@ - Worst case scenario, we might need to: - set up a new control plane (outside of the cluster) - + - restore a backup from the old control plane - + - move the new control plane to the cluster (again) - This doesn't sound like a great experience @@ -57,7 +57,7 @@ - The kubelet can also get a list of *static pods* from: - a directory containing one (or multiple) *manifests*, and/or - + - a URL (serving a *manifest*) - These "manifests" are basically YAML definitions @@ -100,11 +100,11 @@ ## Static pods vs normal pods -- The API only gives us a read-only access to static pods +- The API only gives us read-only access to static pods -- We can `kubectl delete` a static pod ... +- We can `kubectl delete` a static pod... - ... But the kubelet will re-mirror it immediately + ...But the kubelet will re-mirror it immediately - Static pods can be selected just like other pods diff --git a/slides/k8s/versions-k8s.md b/slides/k8s/versions-k8s.md index 937ccb02..b12c45af 100644 --- a/slides/k8s/versions-k8s.md +++ b/slides/k8s/versions-k8s.md @@ -1,6 +1,6 @@ ## Versions installed -- Kubernetes 1.14.2 +- Kubernetes 1.15.0 - Docker Engine 18.09.6 - Docker Compose 1.21.1 @@ -23,7 +23,7 @@ class: extra-details ## Kubernetes and Docker compatibility -- Kubernetes 1.14 validates Docker Engine versions [up to 18.09](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#external-dependencies) +- Kubernetes 1.15 validates Docker Engine versions [up to 18.09](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md#dependencies)
(the latest version when Kubernetes 1.14 was released)