merge

prepare-vms/lib/infra/aws.sh

@@ -31,6 +31,7 @@ infra_start() {
         die "I could not find which AMI to use in this region. Try another region?"
     fi
     AWS_KEY_NAME=$(make_key_name)
+    AWS_INSTANCE_TYPE=${AWS_INSTANCE_TYPE-t3a.medium}
 
     sep "Starting instances"
     info "         Count: $COUNT"
@@ -38,10 +39,11 @@ infra_start() {
     info "     Token/tag: $TAG"
     info "           AMI: $AMI"
     info "      Key name: $AWS_KEY_NAME"
+    info " Instance type: $AWS_INSTANCE_TYPE"
     result=$(aws ec2 run-instances \
         --key-name $AWS_KEY_NAME \
         --count $COUNT \
-        --instance-type ${AWS_INSTANCE_TYPE-t2.medium} \
+        --instance-type $AWS_INSTANCE_TYPE \
         --client-token $TAG \
         --block-device-mapping 'DeviceName=/dev/sda1,Ebs={VolumeSize=20}' \
         --image-id $AMI)
@@ -97,7 +99,7 @@ infra_disableaddrchecks() {
 }
 
 wait_until_tag_is_running() {
-    max_retry=50
+    max_retry=100
     i=0
     done_count=0
     while [[ $done_count -lt $COUNT ]]; do

prepare-vms/setup-admin-clusters.sh

@@ -30,9 +30,9 @@ TAG=$PREFIX-$SETTINGS
 	--settings settings/$SETTINGS.yaml \
 	--count $((3*$STUDENTS))
 
+./workshopctl disableaddrchecks $TAG
 ./workshopctl deploy $TAG
 ./workshopctl kubebins $TAG
-./workshopctl disableaddrchecks $TAG
 ./workshopctl cards $TAG
 
 SETTINGS=admin-kuberouter
@@ -43,11 +43,15 @@ TAG=$PREFIX-$SETTINGS
 	--settings settings/$SETTINGS.yaml \
 	--count $((3*$STUDENTS))
 
+./workshopctl disableaddrchecks $TAG
 ./workshopctl deploy $TAG
 ./workshopctl kubebins $TAG
-./workshopctl disableaddrchecks $TAG
 ./workshopctl cards $TAG
 
+#INFRA=infra/aws-us-west-1
+
+export AWS_INSTANCE_TYPE=t3a.medium
+
 SETTINGS=admin-test
 TAG=$PREFIX-$SETTINGS
 ./workshopctl start \
@@ -59,3 +63,4 @@ TAG=$PREFIX-$SETTINGS
 ./workshopctl deploy $TAG
 ./workshopctl kube $TAG 1.13.5
 ./workshopctl cards $TAG
+

slides/alfun-2.yml

@@ -45,6 +45,7 @@ chapters:
   #- k8s/create-chart.md
   - k8s/rollout.md
 - - k8s/healthchecks.md
+  #- k8s/healthchecks-more.md
   - k8s/kubectlproxy.md
   - k8s/localkubeconfig.md
   - k8s/accessinternal.md

slides/index.yaml

@@ -1,3 +1,11 @@
+- date: [2019-11-04, 2019-11-05]
+  country: de
+  city: Berlin
+  event: Velocity
+  speaker: jpetazzo
+  title: Deploying and scaling applications with Kubernetes
+  attend: https://conferences.oreilly.com/velocity/vl-eu/public/schedule/detail/79109
+
 - date: 2019-11-13
   country: fr
   city: Marseille

slides/k8s/cloud-controller-manager.md

@@ -20,15 +20,15 @@
 
 - Configuring routing tables in the cloud network (specific to GCE)
 
-- Updating node labels to indicate region, zone, instance type ...
+- Updating node labels to indicate region, zone, instance type...
 
 - Obtain node name, internal and external addresses from cloud metadata service
 
 - Deleting nodes from Kubernetes when they're deleted in the cloud
 
-- Managing *some* volumes (e.g. ELBs, AzureDisks ...)
+- Managing *some* volumes (e.g. ELBs, AzureDisks...)
 
-  (Eventually, volumes will be managed by the CSI)
+  (Eventually, volumes will be managed by the Container Storage Interface)
 
 ---
 
@@ -83,7 +83,7 @@ The list includes the following providers:
 
 ## Audience questions
 
-- What kind of clouds are you using / planning to use?
+- What kind of clouds are you using/planning to use?
 
 - What kind of details would you like to see in this section?
 
@@ -105,7 +105,7 @@ The list includes the following providers:
 
 - When using managed clusters, this is done automatically
 
-- There is very little documentation to write the configuration file
+- There is very little documentation on writing the configuration file
 
   (except for OpenStack)
 
@@ -123,7 +123,7 @@ The list includes the following providers:
 
 - To get these addresses, the node needs to communicate with the control plane
 
-- ... Which means joining the cluster
+- ...Which means joining the cluster
 
 (The problem didn't occur when cloud-specific code was running in kubelet: kubelet could obtain the required information directly from the cloud provider's metadata service.)
 

slides/k8s/cluster-backup.md

@@ -6,7 +6,7 @@
 
   - error recovery (human or process has altered or corrupted data)
 
-  - cloning environments (for testing, validation ...)
+  - cloning environments (for testing, validation...)
 
 - Let's see the strategies and tools available with Kubernetes!
 
@@ -18,13 +18,13 @@
 
   (it gives us replication primitives)
 
-- Kubernetes helps us to clone / replicate environments
+- Kubernetes helps us clone / replicate environments
 
   (all resources can be described with manifests)
 
 - Kubernetes *does not* help us with error recovery
 
-- We still need to backup / snapshot our data:
+- We still need to back up/snapshot our data:
 
   - with database backups (mysqldump, pgdump, etc.)
 
@@ -58,7 +58,7 @@
 
 - If our deployment system isn't fully automated, it should at least be documented
 
-- Litmus test: how long does it take to deploy a cluster ...
+- Litmus test: how long does it take to deploy a cluster...
 
   - for a senior engineer?
 
@@ -66,7 +66,7 @@
 
 - Does it require external intervention?
 
-  (e.g. provisioning servers, signing TLS certs ...)
+  (e.g. provisioning servers, signing TLS certs...)
 
 ---
 
@@ -108,7 +108,7 @@
 
 - For real applications: add resources (as YAML files)
 
-- For applications deployed multiple times: Helm, Kustomize ...
+- For applications deployed multiple times: Helm, Kustomize...
 
   (staging and production count as "multiple times")
 

slides/k8s/cluster-upgrade.md

@@ -287,8 +287,8 @@
 - Download the configuration on each node, and upgrade kubelet:
   ```bash
     for N in 1 2 3; do
-    	ssh node$N sudo kubeadm upgrade node config --kubelet-version v1.14.2
-  	  ssh node $N sudo apt install kubelet=1.14.2-00
+      ssh test$N sudo kubeadm upgrade node config --kubelet-version v1.14.2
+      ssh test$N sudo apt install kubelet=1.14.2-00
     done
   ```
 ]

slides/k8s/cni.md

@@ -66,6 +66,8 @@ Look in each plugin's directory for its documentation.
 
 ---
 
+class: extra-details
+
 ## Conf vs conflist
 
 - There are two slightly different configuration formats
@@ -276,7 +278,7 @@ class: extra-details
 
 - The address of the API server will be `http://A.B.C.D:8080`
 
-  (where `A.B.C.D` is the address of `kuberouter1`, running the control plane)
+  (where `A.B.C.D` is the public address of `kuberouter1`, running the control plane)
 
 .exercise[
 
@@ -453,7 +455,7 @@ We should see the local pod CIDR connected to `kube-bridge`, and the other nodes
 
 - Or try to exec into one of the kube-router pods:
   ```bash
-  kubectl -n kube-system exec kuber-router-xxxxx bash
+  kubectl -n kube-system exec kube-router-xxxxx bash
   ```
 
 ]
@@ -575,7 +577,7 @@ done
 
 ## Starting the route reflector
 
-- Only do this if you are doing this on your own
+- Only do this slide if you are doing this on your own
 
 - There is a Compose file in the `compose/frr-route-reflector` directory
 

slides/k8s/csr-api.md

@@ -62,7 +62,7 @@ This is what I do if I want to obtain a certificate.
 
 2. Create a Certificate Signing Request (CSR).
 
-   (The CSR contains the identity that I claim and an expiration date.)
+   (The CSR contains the identity that I claim and a public key.)
 
 3. Send that CSR to the Certificate Authority (CA).
 
@@ -231,7 +231,7 @@ For a user named `jean.doe`, we will have:
 - Let's use OpenSSL; it's not the best one, but it's installed everywhere
 
   (many people prefer cfssl, easyrsa, or other tools; that's fine too!)
- 
+
 .exercise[
 
 - Generate the key and certificate signing request:
@@ -244,7 +244,7 @@ For a user named `jean.doe`, we will have:
 
 The command above generates:
 
-- a 2048-bit RSA key, without DES encryption, stored in key.pem
+- a 2048-bit RSA key, without encryption, stored in key.pem
 - a CSR for the name `jean.doe` in group `devs`
 
 ---
@@ -345,7 +345,7 @@ The command above generates:
   kctx -
   ```
 
-- Retrieve the certificate from the CSR:
+- Retrieve the updated CSR object and extract the certificate:
   ```bash
   kubectl get csr users:jean.doe \
           -o jsonpath={.status.certificate} \

slides/k8s/dmuc.md

@@ -175,7 +175,7 @@ Success!
 
 ]
 
-So far, so good.
+We should get `No resources found.` and the `kubernetes` service, respectively.
 
 Note: the API server automatically created the `kubernetes` service entry.
 
@@ -225,7 +225,7 @@ Success?
 
 ]
 
-Our Deployment is in a bad shape:
+Our Deployment is in bad shape:
 ```
 NAME                  READY   UP-TO-DATE   AVAILABLE   AGE
 deployment.apps/web   0/1     0            0           2m26s

slides/k8s/healthchecks-more.md

@@ -0,0 +1,393 @@
+## Questions to ask before adding healthchecks
+
+- Do we want liveness, readiness, both?
+
+  (sometimes, we can use the same check, but with different failure thresholds)
+
+- Do we have existing HTTP endpoints that we can use?
+
+- Do we need to add new endpoints, or perhaps use something else?
+
+- Are our healthchecks likely to use resources and/or slow down the app?
+
+- Do they depend on additional services?
+
+  (this can be particularly tricky, see next slide)
+
+---
+
+## Healthchecks and dependencies
+
+- A good healthcheck should always indicate the health of the service itself
+
+- It should not be affected by the state of the service's dependencies
+
+- Example: a web server requiring a database connection to operate
+
+  (make sure that the healthcheck can report "OK" even if the database is down;
+  <br/>
+  because it won't help us to restart the web server if the issue is with the DB!)
+
+- Example: a microservice calling other microservices
+
+- Example: a worker process
+
+  (these will generally require minor code changes to report health)
+
+---
+
+## Adding healthchecks to an app
+
+- Let's add healthchecks to DockerCoins!
+
+- We will examine the questions of the previous slide
+
+- Then we will review each component individually to add healthchecks
+
+---
+
+## Liveness, readiness, or both?
+
+- To answer that question, we need to see the app run for a while
+
+- Do we get temporary, recoverable glitches?
+
+  → then use readiness
+
+- Or do we get hard lock-ups requiring a restart?
+
+  → then use liveness
+
+- In the case of DockerCoins, we don't know yet!
+
+- Let's pick liveness
+
+---
+
+## Do we have HTTP endpoints that we can use?
+
+- Each of the 3 web services (hasher, rng, webui) has a trivial route on `/`
+
+- These routes:
+
+  - don't seem to perform anything complex or expensive
+
+  - don't seem to call other services
+
+- Perfect!
+
+  (See next slides for individual details)
+
+---
+
+- [hasher.rb](https://github.com/jpetazzo/container.training/blob/master/dockercoins/hasher/hasher.rb)
+  ```ruby
+    get '/' do
+      "HASHER running on #{Socket.gethostname}\n"
+    end
+  ```
+
+- [rng.py](https://github.com/jpetazzo/container.training/blob/master/dockercoins/rng/rng.py)
+  ```python
+    @app.route("/")
+    def index():
+      return "RNG running on {}\n".format(hostname)
+  ```
+
+- [webui.js](https://github.com/jpetazzo/container.training/blob/master/dockercoins/webui/webui.js)
+  ```javascript
+    app.get('/', function (req, res) {
+      res.redirect('/index.html');
+    });
+  ```
+
+---
+
+## Running DockerCoins
+
+- We will run DockerCoins in a new, separate namespace
+
+- We will use a set of YAML manifests and pre-built images
+
+- We will add our new liveness probe to the YAML of the `rng` DaemonSet
+
+- Then, we will deploy the application
+
+---
+
+## Creating a new namespace
+
+- This will make sure that we don't collide / conflict with previous exercises
+
+.exercise[
+
+- Create the yellow namespace:
+  ```bash
+  kubectl create namespace yellow
+  ```
+
+- Switch to that namespace:
+  ```bash
+  kns yellow
+  ```
+
+]
+
+---
+
+## Retrieving DockerCoins manifests
+
+- All the manifests that we need are on a convenient repository:
+
+  https://github.com/jpetazzo/kubercoins
+
+.exercise[
+
+- Clone that repository:
+  ```bash
+  cd ~
+  git clone https://github.com/jpetazzo/kubercoins
+  ```
+
+- Change directory to the repository:
+  ```bash
+  cd kubercoins
+  ```
+
+]
+
+---
+
+## A simple HTTP liveness probe
+
+This is what our liveness probe should look like:
+
+```yaml
+containers:
+- name: ...
+image: ...
+livenessProbe:
+  httpGet:
+    path: /
+    port: 80
+  initialDelaySeconds: 30
+  periodSeconds: 5
+```
+
+This will give 30 seconds to the service to start. (Way more than necessary!)
+<br/>
+It will run the probe every 5 seconds.
+<br/>
+It will use the default timeout (1 second).
+<br/>
+It will use the default failure threshold (3 failed attempts = dead).
+<br/>
+It will use the default success threshold (1 successful attempt = alive).
+
+---
+
+## Adding the liveness probe
+
+- Let's add the liveness probe, then deploy DockerCoins
+
+.exercise[
+
+- Edit `rng-daemonset.yaml` and add the liveness probe
+  ```bash
+  vim rng-daemonset.yaml
+  ```
+
+- Load the YAML for all the resources of DockerCoins:
+  ```bash
+  kubectl apply -f .
+  ```
+
+]
+
+---
+
+## Testing the liveness probe
+
+- The rng service needs 100ms to process a request
+
+  (because it is single-threaded and sleeps 0.1s in each request)
+
+- The probe timeout is set to 1 second
+
+- If we send more than 10 requests per second per backend, it will break
+
+- Let's generate traffic and see what happens!
+
+.exercise[
+
+- Get the ClusterIP address of the rng service:
+  ```bash
+  kubectl get svc rng
+  ```
+
+]
+
+---
+
+## Monitoring the rng service
+
+- Each command below will show us what's happening on a different level
+
+.exercise[
+
+- In one window, monitor cluster events:
+  ```bash
+  kubectl get events -w
+  ```
+
+- In another window, monitor the response time of rng:
+  ```bash
+  httping `<ClusterIP>`
+  ```
+
+- In another window, monitor pods status:
+  ```bash
+  kubectl get pods -w
+  ```
+
+]
+
+---
+
+## Generating traffic
+
+- Let's use `ab` to send concurrent requests to rng
+
+.exercise[
+
+- In yet another window, generate traffic:
+  ```bash
+  ab -c 10 -n 1000 http://`<ClusterIP>`/1
+  ```
+
+- Experiment with higher values of `-c` and see what happens
+
+]
+
+- The `-c` parameter indicates the number of concurrent requests
+
+- The final `/1` is important to generate actual traffic
+
+  (otherwise we would use the ping endpoint, which doesn't sleep 0.1s per request)
+
+---
+
+## Discussion
+
+- Above a given threshold, the liveness probe starts failing
+
+  (about 10 concurrent requests per backend should be plenty enough)
+
+- When the liveness probe fails 3 times in a row, the container is restarted
+
+- During the restart, there is *less* capacity available
+
+- ... Meaning that the other backends are likely to timeout as well
+
+- ... Eventually causing all backends to be restarted
+
+- ... And each fresh backend gets restarted, too
+
+- This goes on until the load goes down, or we add capacity
+
+*This wouldn't be a good healthcheck in a real application!*
+
+---
+
+## Better healthchecks
+
+- We need to make sure that the healthcheck doesn't trip when
+  performance degrades due to external pressure
+
+- Using a readiness check would have fewer effects
+
+  (but it would still be an imperfect solution)
+
+- A possible combination:
+
+  - readiness check with a short timeout / low failure threshold
+
+  - liveness check with a longer timeout / higher failure treshold
+
+---
+
+## Healthchecks for redis
+
+- A liveness probe is enough
+
+  (it's not useful to remove a backend from rotation when it's the only one)
+
+- We could use an exec probe running `redis-cli ping`
+
+---
+
+class: extra-details
+
+## Exec probes and zombies
+
+- When using exec probes, we should make sure that we have a *zombie reaper*
+
+  🤔🧐🧟 Wait, what?
+
+- When a process terminates, its parent must call `wait()`/`waitpid()`
+
+  (this is how the parent process retrieves the child's exit status)
+
+- In the meantime, the process is in *zombie* state
+
+  (the process state will show as `Z` in `ps`, `top` ...)
+
+- When a process is killed, its children are *orphaned* and attached to PID 1
+
+- PID 1 has the responsibility of *reaping* these processes when they terminate
+
+- OK, but how does that affect us?
+
+---
+
+class: extra-details
+
+## PID 1 in containers
+
+- On ordinary systems, PID 1 (`/sbin/init`) has logic to reap processes
+
+- In containers, PID 1 is typically our application process
+
+  (e.g. Apache, the JVM, NGINX, Redis ...)
+
+- These *do not* take care of reaping orphans
+
+- If we use exec probes, we need to add a process reaper
+
+- We can add [tini](https://github.com/krallin/tini) to our images
+
+- Or [share the PID namespace between containers of a pod](https://kubernetes.io/docs/tasks/configure-pod-container/share-process-namespace/)
+
+  (and have gcr.io/pause take care of the reaping)
+
+---
+
+## Healthchecks for worker
+
+- Readiness isn't useful
+
+  (because worker isn't a backend for a service)
+
+- Liveness may help us restart a broken worker, but how can we check it?
+
+- Embedding an HTTP server is an option
+
+  (but it has a high potential for unwanted side effects and false positives)
+
+- Using a "lease" file can be relatively easy:
+
+  - touch a file during each iteration of the main loop
+
+  - check the timestamp of that file from an exec probe
+
+- Writing logs (and checking them from the probe) also works

slides/k8s/lastwords-admin.md

@@ -48,7 +48,7 @@
 
 - Acknowledge that a lot of tasks are outsourced
 
-  (e.g. if we add "buy / rack / provision machines" in that list)
+  (e.g. if we add "buy/rack/provision machines" in that list)
 
 ---
 
@@ -122,7 +122,7 @@
 
   (YAML, Helm charts, Kustomize ...)
 
-- Team "run" adjusts some parameters and monitors the application 
+- Team "run" adjusts some parameters and monitors the application
 
 ✔️ parity between dev and prod environments
 
@@ -150,7 +150,7 @@
 
   - do we reward on-call duty without encouraging hero syndrome?
 
-  - do we give resources (time, money) to people to learn?
+  - do we give people resources (time, money) to learn?
 
 ---
 
@@ -183,9 +183,9 @@ are a few tools that can help us.*
 
 - If cloud: public vs. private
 
-- Which vendor / distribution to pick?
+- Which vendor/distribution to pick?
 
-- Which versions / features to enable?
+- Which versions/features to enable?
 
 ---
 
@@ -205,6 +205,6 @@ are a few tools that can help us.*
 
 - Transfer knowledge
 
-  (make sure everyone is on the same page / same level)
+  (make sure everyone is on the same page/level)
 
 - Iterate!

slides/k8s/multinode.md

@@ -96,7 +96,7 @@ class: extra-details
 
 - We need to generate a `kubeconfig` file for kubelet
 
-- This time, we need to put the IP address of `kubenet1`
+- This time, we need to put the public IP address of `kubenet1`
 
   (instead of `localhost` or `127.0.0.1`)
 
@@ -195,7 +195,7 @@ class: extra-details
 
 ## Check our pods
 
-- The pods will be scheduled to the nodes
+- The pods will be scheduled on the nodes
 
 - The nodes will pull the `nginx` image, and start the pods
 
@@ -325,7 +325,7 @@ class: extra-details
 
 - We will add the `--network-plugin` and `--pod-cidr` flags
 
-- We all have a "cluster number" (let's call that `C`)
+- We all have a "cluster number" (let's call that `C`) printed on your VM info card
 
 - We will use pod CIDR `10.C.N.0/24` (where `N` is the node number: 1, 2, 3)
 
@@ -480,6 +480,23 @@ Sometimes it works, sometimes it doesn't. Why?
   ```bash
   kubectl get nodes -o wide
   ```
+
+---
+
+## Firewalling
+
+- By default, Docker prevents containers from using arbitrary IP addresses
+
+  (by setting up iptables rules)
+
+- We need to allow our containers to use our pod CIDR
+
+- For simplicity, we will insert a blanket iptables rule allowing all traffic:
+
+  `iptables -I FORWARD -j ACCEPT`
+
+- This has to be done on every node
+
 ---
 
 ## Setting up routing
@@ -488,6 +505,8 @@ Sometimes it works, sometimes it doesn't. Why?
 
 - Create all the routes on all the nodes
 
+- Insert the iptables rule allowing traffic
+
 - Check that you can ping all the pods from one of the nodes
 
 - Check that you can `curl` the ClusterIP of the Service successfully

slides/k8s/operators-design.md

@@ -0,0 +1,356 @@
+## What does it take to write an operator?
+
+- Writing a quick-and-dirty operator, or a POC/MVP, is easy
+
+- Writing a robust operator is hard
+
+- We will describe the general idea
+
+- We will identify some of the associated challenges
+
+- We will list a few tools that can help us
+
+---
+
+## Top-down vs. bottom-up
+
+- Both approaches are possible
+
+- Let's see what they entail, and their respective pros and cons
+
+---
+
+## Top-down approach
+
+- Start with high-level design (see next slide)
+
+- Pros:
+
+  - can yield cleaner design that will be more robust
+
+- Cons:
+
+  - must be able to anticipate all the events that might happen
+
+  - design will be better only to the extend of what we anticipated
+
+  - hard to anticipate if we don't have production experience
+
+---
+
+## High-level design
+
+- What are we solving?
+
+  (e.g.: geographic databases backed by PostGIS with Redis caches)
+
+- What are our use-cases, stories?
+
+  (e.g.: adding/resizing caches and read replicas; load balancing queries)
+
+- What kind of outage do we want to address?
+
+  (e.g.: loss of individual node, pod, volume)
+
+- What are our *non-features*, the things we don't want to address?
+
+  (e.g.: loss of datacenter/zone; differentiating between read and write queries;
+  <br/>
+  cache invalidation; upgrading to newer major versions of Redis, PostGIS, PostgreSQL)
+
+---
+
+## Low-level design
+
+- What Custom Resource Definitions do we need?
+
+  (one, many?)
+
+- How will we store configuration information?
+
+  (part of the CRD spec fields, annotations, other?)
+
+- Do we need to store state? If so, where?
+
+  - state that is small and doesn't change much can be stored via the Kubernetes API
+    <br/>
+    (e.g.: leader information, configuration, credentials)
+
+  - things that are big and/or change a lot should go elsewhere
+    <br/>
+    (e.g.: metrics, bigger configuration file like GeoIP)
+
+---
+
+class: extra-details
+
+## What can we store via the Kubernetes API?
+
+- The API server stores most Kubernetes resources in etcd
+
+- Etcd is designed for reliability, not for performance
+
+- If our storage needs exceed what etcd can offer, we need to use something else:
+
+  - either directly
+
+  - or by extending the API server
+    <br/>(for instance by using the agregation layer, like [metrics server](https://github.com/kubernetes-incubator/metrics-server) does)
+
+---
+
+## Bottom-up approach
+
+- Start with existing Kubernetes resources (Deployment, Stateful Set...)
+
+- Run the system in production
+
+- Add scripts, automation, to facilitate day-to-day operations
+
+- Turn the scripts into an operator
+
+- Pros: simpler to get started; reflects actual use-cases
+
+- Cons: can result in convoluted designs requiring extensive refactor
+
+---
+
+## General idea
+
+- Our operator will watch its CRDs *and associated resources*
+
+- Drawing state diagrams and finite state automata helps a lot
+
+- It's OK if some transitions lead to a big catch-all "human intervention"
+
+- Over time, we will learn about new failure modes and add to these diagrams
+
+- It's OK to start with CRD creation / deletion and prevent any modification
+
+  (that's the easy POC/MVP we were talking about)
+
+- *Presentation* and *validation* will help our users
+
+  (more on that later)
+
+---
+
+## Challenges
+
+- Reacting to infrastructure disruption can seem hard at first
+
+- Kubernetes gives us a lot of primitives to help:
+
+  - Pods and Persistent Volumes will *eventually* recover
+
+  - Stateful Sets give us easy ways to "add N copies" of a thing
+
+- The real challenges come with configuration changes
+
+  (i.e., what to do when our users update our CRDs)
+
+- Keep in mind that [some] of the [largest] cloud [outages] haven't been caused by [natural catastrophes], or even code bugs, but by configuration changes
+
+[some]: https://www.datacenterdynamics.com/news/gcp-outage-mainone-leaked-google-cloudflare-ip-addresses-china-telecom/
+[largest]: https://aws.amazon.com/message/41926/
+[outages]: https://aws.amazon.com/message/65648/
+[natural catastrophes]: https://www.datacenterknowledge.com/amazon/aws-says-it-s-never-seen-whole-data-center-go-down
+
+---
+
+## Configuration changes
+
+- It is helpful to analyze and understand how Kubernetes controllers work:
+
+  - watch resource for modifications
+
+  - compare desired state (CRD) and current state
+
+  - issue actions to converge state
+
+- Configuration changes will probably require *another* state diagram or FSA
+
+- Again, it's OK to have transitions labeled as "unsupported"
+
+  (i.e. reject some modifications because we can't execute them)
+
+---
+
+## Tools
+
+- CoreOS / RedHat Operator Framework
+
+  [GitHub](https://github.com/operator-framework)
+  |
+  [Blog](https://developers.redhat.com/blog/2018/12/18/introduction-to-the-kubernetes-operator-framework/)
+  |
+  [Intro talk](https://www.youtube.com/watch?v=8k_ayO1VRXE)
+  |
+  [Deep dive talk](https://www.youtube.com/watch?v=fu7ecA2rXmc)
+
+- Zalando Kubernetes Operator Pythonic Framework (KOPF)
+
+  [GitHub](https://github.com/zalando-incubator/kopf)
+  |
+  [Docs](https://kopf.readthedocs.io/)
+  |
+  [Step-by-step tutorial](https://kopf.readthedocs.io/en/stable/walkthrough/problem/)
+
+- Mesosphere Kubernetes Universal Declarative Operator (KUDO)
+
+  [GitHub](https://github.com/kudobuilder/kudo)
+  |
+  [Blog](https://mesosphere.com/blog/announcing-maestro-a-declarative-no-code-approach-to-kubernetes-day-2-operators/)
+  |
+  [Docs](https://kudo.dev/)
+  |
+  [Zookeeper example](https://github.com/kudobuilder/frameworks/tree/master/repo/stable/zookeeper)
+
+---
+
+## Validation
+
+- By default, a CRD is "free form"
+
+  (we can put pretty much anything we want in it)
+
+- When creating a CRD, we can provide an OpenAPI v3 schema
+  ([Example](https://github.com/amaizfinance/redis-operator/blob/master/deploy/crds/k8s_v1alpha1_redis_crd.yaml#L34))
+
+- The API server will then validate resources created/edited with this schema
+
+- If we need a stronger validation, we can use a Validating Admission Webhook:
+
+  - run an [admission webhook server](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#write-an-admission-webhook-server) to receive validation requests
+
+  - register the webhook by creating a [ValidatingWebhookConfiguration](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#configure-admission-webhooks-on-the-fly)
+
+  - each time the API server receives a request matching the configuration,
+    <br/>the request is sent to our server for validation
+
+---
+
+## Presentation
+
+- By default, `kubectl get mycustomresource` won't display much information
+
+  (just the name and age of each resource)
+
+- When creating a CRD, we can specify additional columns to print
+  ([Example](https://github.com/amaizfinance/redis-operator/blob/master/deploy/crds/k8s_v1alpha1_redis_crd.yaml#L6),
+  [Docs](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#additional-printer-columns))
+
+- By default, `kubectl describe mycustomresource` will also be generic
+
+- `kubectl describe` can show events related to our custom resources
+
+  (for that, we need to create Event resources, and fill the `involvedObject` field)
+
+- For scalable resources, we can define a `scale` sub-resource
+
+- This will enable the use of `kubectl scale` and other scaling-related operations
+
+---
+
+## About scaling
+
+- It is possible to use the HPA (Horizontal Pod Autoscaler) with CRDs
+
+- But it is not always desirable
+
+- The HPA works very well for homogenous, stateless workloads
+
+- For other workloads, your mileage may vary
+
+- Some systems can scale across multiple dimensions
+
+  (for instance: increase number of replicas, or number of shards?)
+
+- If autoscaling is desired, the operator will have to take complex decisions
+
+  (example: Zalando's Elasticsearch Operator ([Video](https://www.youtube.com/watch?v=lprE0J0kAq0)))
+
+---
+
+## Versioning
+
+- As our operator evolves over time, we may have to change the CRD
+
+  (add, remove, change fields)
+
+- Like every other resource in Kubernetes, [custom resources are versioned](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definition-versioning/
+)
+
+- When creating a CRD, we need to specify a *list* of versions
+
+- Versions can be marked as `stored` and/or `served`
+
+---
+
+## Stored version
+
+- Exactly one version has to be marked as the `stored` version
+
+- As the name implies, it is the one that will be stored in etcd
+
+- Resources in storage are never converted automatically
+
+  (we need to read and re-write them ourselves)
+
+- Yes, this means that we can have different versions in etcd at any time
+
+- Our code needs to handle all the versions that still exist in storage
+
+---
+
+## Served versions
+
+- By default, the Kubernetes API will serve resources "as-is"
+
+  (using their stored version)
+
+- It will assume that all versions are compatible storage-wise
+
+  (i.e. that the spec and fields are compatible between versions)
+
+- We can provide [conversion webhooks](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definition-versioning/#webhook-conversion) to "translate" requests
+
+  (the alternative is to upgrade all stored resources and stop serving old versions)
+
+---
+
+## Operator reliability
+
+- Remember that the operator itself must be resilient
+
+  (e.g.: the node running it can fail)
+
+- Our operator must be able to restart and recover gracefully
+
+- Do not store state locally
+
+  (unless we can reconstruct that state when we restart)
+
+- As indicated earlier, we can use the Kubernetes API to store data:
+
+  - in the custom resources themselves
+
+  - in other resources' annotations
+
+---
+
+## Beyond CRDs
+
+- CRDs cannot use custom storage (e.g. for time series data)
+
+- CRDs cannot support arbitrary subresources (like logs or exec for Pods)
+
+- CRDs cannot support protobuf (for faster, more efficient communication)
+
+- If we need these things, we can use the [aggregation layer](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/apiserver-aggregation/) instead
+
+- The aggregation layer proxies all requests below a specific path to another server
+
+  (this is used e.g. by the metrics server)
+
+- [This documentation page](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#choosing-a-method-for-adding-custom-resources) compares the features of CRDs and API aggregation

slides/k8s/operators.md

@@ -0,0 +1,389 @@
+# Operators
+
+- Operators are one of the many ways to extend Kubernetes
+
+- We will define operators
+
+- We will see how they work
+
+- We will install a specific operator (for ElasticSearch)
+
+- We will use it to provision an ElasticSearch cluster
+
+---
+
+## What are operators?
+
+*An operator represents **human operational knowledge in software,**
+<br/>
+to reliably manage an application.
+— [CoreOS](https://coreos.com/blog/introducing-operators.html)*
+
+Examples:
+
+- Deploying and configuring replication with MySQL, PostgreSQL ...
+
+- Setting up Elasticsearch, Kafka, RabbitMQ, Zookeeper ...
+
+- Reacting to failures when intervention is needed
+
+- Scaling up and down these systems
+
+---
+
+## What are they made from?
+
+- Operators combine two things:
+
+  - Custom Resource Definitions
+
+  - controller code watching the corresponding resources and acting upon them
+
+- A given operator can define one or multiple CRDs
+
+- The controller code (control loop) typically runs within the cluster
+
+  (running as a Deployment with 1 replica is a common scenario)
+
+- But it could also run elsewhere
+
+  (nothing mandates that the code run on the cluster, as long as it has API access)
+
+---
+
+## Why use operators?
+
+- Kubernetes gives us Deployments, StatefulSets, Services ...
+
+- These mechanisms give us building blocks to deploy applications
+
+- They work great for services that are made of *N* identical containers
+
+  (like stateless ones)
+
+- They also work great for some stateful applications like Consul, etcd ...
+
+  (with the help of highly persistent volumes)
+
+- They're not enough for complex services:
+
+  - where different containers have different roles
+
+  - where extra steps have to be taken when scaling or replacing containers
+
+---
+
+## Use-cases for operators
+
+- Systems with primary/secondary replication
+
+  Examples: MariaDB, MySQL, PostgreSQL, Redis ...
+
+- Systems where different groups of nodes have different roles
+
+  Examples: ElasticSearch, MongoDB ...
+
+- Systems with complex dependencies (that are themselves managed with operators)
+
+  Examples: Flink or Kafka, which both depend on Zookeeper
+
+---
+
+## More use-cases
+
+- Representing and managing external resources
+
+  (Example: [AWS Service Operator](https://operatorhub.io/operator/alpha/aws-service-operator.v0.0.1))
+
+- Managing complex cluster add-ons
+
+  (Example: [Istio operator](https://operatorhub.io/operator/beta/istio-operator.0.1.6))
+
+- Deploying and managing our applications' lifecycles
+
+  (more on that later)
+
+---
+
+## How operators work
+
+- An operator creates one or more CRDs
+
+  (i.e., it creates new "Kinds" of resources on our cluster)
+
+- The operator also runs a *controller* that will watch its resources
+
+- Each time we create/update/delete a resource, the controller is notified
+
+  (we could write our own cheap controller with `kubectl get --watch`)
+
+---
+
+## One operator in action
+
+- We will install the UPMC Enterprises ElasticSearch operator
+
+- This operator requires PersistentVolumes
+
+- We will install Rancher's [local path storage provisioner](https://github.com/rancher/local-path-provisioner) to automatically create these
+
+- Then, we will create an ElasticSearch resource
+
+- The operator will detect that resource and provision the cluster
+
+---
+
+## Installing a Persistent Volume provisioner
+
+(This step can be skipped if you already have a dynamic volume provisioner.)
+
+- This provisioner creates Persistent Volumes backed by `hostPath`
+
+  (local directories on our nodes)
+
+- It doesn't require anything special ...
+
+- ... But losing a node = losing the volumes on that node!
+
+.exercise[
+
+- Install the local path storage provisioner:
+  ```bash
+  kubectl apply -f ~/container.training/k8s/local-path-storage.yaml
+  ```
+
+]
+
+---
+
+## Making sure we have a default StorageClass
+
+- The ElasticSearch operator will create StatefulSets
+
+- These StatefulSets will instantiate PersistentVolumeClaims
+
+- These PVCs need to be explicitly associated with a StorageClass
+
+- Or we need to tag a StorageClass to be used as the default one
+
+.exercise[
+
+- List StorageClasses:
+  ```bash
+  kubectl get storageclasses
+  ```
+
+]
+
+We should see the `local-path` StorageClass.
+
+---
+
+## Setting a default StorageClass
+
+- This is done by adding an annotation to the StorageClass:
+
+  `storageclass.kubernetes.io/is-default-class: true`
+
+.exercise[
+
+- Tag the StorageClass so that it's the default one:
+  ```bash
+  kubectl annotate storageclass local-path \
+            storageclass.kubernetes.io/is-default-class=true
+  ```
+
+- Check the result:
+  ```bash
+  kubectl get storageclasses
+  ```
+
+]
+
+Now, the StorageClass should have `(default)` next to its name.
+
+---
+
+## Install the ElasticSearch operator
+
+- The operator needs:
+
+  - a Deployment for its controller
+  - a ServiceAccount, ClusterRole, ClusterRoleBinding for permissions
+  - a Namespace
+
+- We have grouped all the definitions for these resources in a YAML file
+
+.exercise[
+
+- Install the operator:
+  ```bash
+  kubectl apply -f ~/container.training/k8s/elasticsearch-operator.yaml
+  ```
+
+]
+
+---
+
+## Wait for the operator to be ready
+
+- Some operators require to create their CRDs separately
+
+- This operator will create its CRD itself
+
+  (i.e. the CRD is not listed in the YAML that we applied earlier)
+
+.exercise[
+
+- Wait until the `elasticsearchclusters` CRD shows up:
+  ```bash
+  kubectl get crds
+  ```
+
+]
+
+---
+
+## Create an ElasticSearch resource
+
+- We can now create a resource with `kind: ElasticsearchCluster`
+
+- The YAML for that resource will specify all the desired parameters:
+
+  - how many nodes do we want of each type (client, master, data)
+  - image to use
+  - add-ons (kibana, cerebro, ...)
+  - whether to use TLS or not
+  - etc.
+
+.exercise[
+
+- Create our ElasticSearch cluster:
+  ```bash
+  kubectl apply -f ~/container.training/k8s/elasticsearch-cluster.yaml
+  ```
+
+]
+
+---
+
+## Operator in action
+
+- Over the next minutes, the operator will create:
+
+  - StatefulSets (one for master nodes, one for data nodes)
+
+  - Deployments (for client nodes; and for add-ons like cerebro and kibana)
+
+  - Services (for all these pods)
+
+.exercise[
+
+- Wait for all the StatefulSets to be fully up and running:
+  ```bash
+  kubectl get statefulsets -w
+  ```
+
+]
+
+---
+
+## Connecting to our cluster
+
+- Since connecting directly to the ElasticSearch API is a bit raw,
+  <br/>we'll connect to the cerebro frontend instead
+
+.exercise[
+
+- Edit the cerebro service to change its type from ClusterIP to NodePort:
+  ```bash
+  kubectl patch svc cerebro-es -p "spec: { type: NodePort }"
+  ```
+
+- Retrieve the NodePort that was allocated:
+  ```bash
+  kubectl get svc cerebreo-es
+  ```
+
+- Connect to that port with a browser
+
+]
+
+---
+
+## (Bonus) Setup filebeat
+
+- Let's send some data to our brand new ElasticSearch cluster!
+
+- We'll deploy a filebeat DaemonSet to collect node logs
+
+.exercise[
+
+- Deploy filebeat:
+  ```bash
+  kubectl apply -f ~/container.training/k8s/filebeat.yaml
+  ```
+
+]
+
+We should see at least one index being created in cerebro.
+
+---
+
+## (Bonus) Access log data with kibana
+
+- Let's expose kibana (by making kibana-es a NodePort too)
+
+- Then access kibana
+
+- We'll need to configure kibana indexes
+
+---
+
+## Deploying our apps with operators
+
+- It is very simple to deploy with `kubectl run` / `kubectl expose`
+
+- We can unlock more features by writing YAML and using `kubectl apply`
+
+- Kustomize or Helm let us deploy in multiple environments
+
+  (and adjust/tweak parameters in each environment)
+
+- We can also use an operator to deploy our application
+
+---
+
+## Pros and cons of deploying with operators
+
+- The app definition and configuration is persisted in the Kubernetes API
+
+- Multiple instances of the app can be manipulated with `kubectl get`
+
+- We can add labels, annotations to the app instances
+
+- Our controller can execute custom code for any lifecycle event
+
+- However, we need to write this controller
+
+- We need to be careful about changes
+
+  (what happens when the resource `spec` is updated?)
+
+---
+
+## Operators are not magic
+
+- Look at the ElasticSearch resource definition
+
+  (`~/container.training/k8s/elasticsearch-cluster.yaml`)
+
+- What should happen if we flip the `use-tls` flag? Twice?
+
+- What should happen if we remove / re-add the kibana or cerebro sections?
+
+- What should happen if we change the number of nodes?
+
+- What if we want different images or parameters for the different nodes?
+
+*Operators can be very powerful, iff we know exactly the scenarios that they can handle.*

slides/k8s/podsecuritypolicy.md

@@ -212,7 +212,7 @@
 
 - Have a look at the static pods:
   ```bash
-  ls -l /etc/kubernetes/manifest
+  ls -l /etc/kubernetes/manifests
   ```
 
 - Edit the one corresponding to the API server:
@@ -236,7 +236,7 @@
 
 - Add `PodSecurityPolicy`
 
-  (It should read `--enable-admission-plugins=NodeRestriction,PodSecurityPolicy`)
+  It should read: `--enable-admission-plugins=NodeRestriction,PodSecurityPolicy`
 
 - Save, quit
 

slides/k8s/setup-k8s.md

@@ -90,4 +90,4 @@
 
 - For a longer list, check the Kubernetes documentation:
   <br/>
-  it has a great guide to [pick the right solution](https://kubernetes.io/docs/setup/pick-right-solution/) to set up Kubernetes.
+  it has a great guide to [pick the right solution](https://kubernetes.io/docs/setup/#production-environment) to set up Kubernetes.

slides/k8s/staticpods.md

@@ -18,7 +18,7 @@
 
 ## A possible approach
 
-- Since each component of the control plane can be replicated ...
+- Since each component of the control plane can be replicated...
 
 - We could set up the control plane outside of the cluster
 
@@ -39,9 +39,9 @@
 - Worst case scenario, we might need to:
 
   - set up a new control plane (outside of the cluster)
-  
+
   - restore a backup from the old control plane
-  
+
   - move the new control plane to the cluster (again)
 
 - This doesn't sound like a great experience
@@ -57,7 +57,7 @@
 - The kubelet can also get a list of *static pods* from:
 
   - a directory containing one (or multiple) *manifests*, and/or
-  
+
   - a URL (serving a *manifest*)
 
 - These "manifests" are basically YAML definitions
@@ -100,11 +100,11 @@
 
 ## Static pods vs normal pods
 
-- The API only gives us a read-only access to static pods
+- The API only gives us read-only access to static pods
 
-- We can `kubectl delete` a static pod ...
+- We can `kubectl delete` a static pod...
 
-  ... But the kubelet will re-mirror it immediately
+  ...But the kubelet will re-mirror it immediately
 
 - Static pods can be selected just like other pods
 

slides/k8s/versions-k8s.md

@@ -1,6 +1,6 @@
 ## Versions installed
 
-- Kubernetes 1.14.2
+- Kubernetes 1.15.0
 - Docker Engine 18.09.6
 - Docker Compose 1.21.1
 
@@ -23,7 +23,7 @@ class: extra-details
 
 ## Kubernetes and Docker compatibility
 
-- Kubernetes 1.14 validates Docker Engine versions [up to 18.09](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#external-dependencies)
+- Kubernetes 1.15 validates Docker Engine versions [up to 18.09](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md#dependencies)
   <br/>
   (the latest version when Kubernetes 1.14 was released)