From f3eb9ce12f4b157ff404ee2dc3d867423c2d4dc2 Mon Sep 17 00:00:00 2001
From: Jerome Petazzoni <jerome.petazzoni@gmail.com>
Date: Fri, 9 Apr 2021 12:51:38 +0200
Subject: [PATCH] =?UTF-8?q?=F0=9F=91=80=20Review=20+=20improve=20OpenEBS?=
 =?UTF-8?q?=20content?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 k8s/openebs-pod.yaml      |   7 +-
 slides/k8s/openebs.md     | 514 ++++++++++++++++++++++++++++++++------
 slides/kube-fullday.yml   |   1 +
 slides/kube-selfpaced.yml |   1 +
 slides/kube-twodays.yml   |   1 +
 5 files changed, 444 insertions(+), 80 deletions(-)

diff --git a/k8s/openebs-pod.yaml b/k8s/openebs-pod.yaml
index 1a5cc6de..9a38a86f 100644
--- a/k8s/openebs-pod.yaml
+++ b/k8s/openebs-pod.yaml
@@ -13,7 +13,12 @@ spec:
     command:
        - sh
        - -c
-       - 'while true; do echo "`date` [`hostname`] Kubernetes is better with PVs." >> /mnt/storage/greet.txt; sleep $(($RANDOM % 5 + 20)); done'
+       - |
+         while true; do
+           echo "$(date) [$(hostname)] Kubernetes is better with PVs." >> /mnt/storage/greet.txt
+           sleep $(($RANDOM % 5 + 20))
+         done
     volumeMounts:
     - mountPath: /mnt/storage
       name: storage
+
diff --git a/slides/k8s/openebs.md b/slides/k8s/openebs.md
index 7193e2e4..942b398e 100644
--- a/slides/k8s/openebs.md
+++ b/slides/k8s/openebs.md
@@ -1,111 +1,172 @@
 # OpenEBS 
 
- - OpenEBS is a popular open-source storage solution for Kubernetes
+ - [OpenEBS] is a popular open-source storage solution for Kubernetes
 
- - Think "Container Attached Storage"
+ - Uses the concept of "Container Attached Storage"
+
+   (1 volume = 1 dedicated controller pod + a set of replica pods)
 
  - Supports a wide range of storage engines:
+
+   - LocalPV: local volumes (hostpath or device), no replication
+
    - Jiva: for lighter workloads with basic cloning/snapshotting
 
-   - cStor: based on iSCSI
+   - cStor: more powerful engine that also supports resizing, RAID, disk pools ...
 
-   - Mayastor: light-weight abstraction layer with nVME and vhost-user support
+   - [Mayastor]: newer, even more powerful engine with NVMe and vhost-user support
 
-   - OpenEBS Local PV - for lowest latency local volumes
+[OpenEBS]: https://openebs.io/
+
+[Mayastor]: https://github.com/openebs/MayaStor#mayastor
 
 ---
-## Installing OpenEBS with Helm
 
-OpenEBS control plane runs as a set of containers on Kubernetes worker nodes. 
-It can be installed with helm:
+class: extra-details
 
-.exercise[
+## What are all these storage engines?
 
-  - Install OpenEBS
-```bash  
-    kubectl create ns openebs
-    helm repo add openebs https://openebs.github.io/charts
-    helm repo update
-    helm install openebs openebs/openebs --namespace openebs
-```
-]
+- LocalPV is great if we want good performance, no replication, easy setup
+
+  (it is similar to the Rancher local path provisioner)
+
+- Jiva is great if we want replication and easy setup
+
+  (data is stored in containers' filesystems)
+
+- cStor is more powerful and flexible, but requires more extensive setup
+
+- Mayastor is designed to achieve extreme performance levels
+
+  (with the right hardware and disks)
+
+- The OpenEBS documentation has a [good comparison of engines] to help us pick
+
+[good comparison of engines]: https://docs.openebs.io/docs/next/casengines.html#cstor-vs-jiva-vs-localpv-features-comparison
 
 ---
 
 ## Installing OpenEBS with Helm
 
-Let's check the running OpenEBS components:
+- The OpenEBS control plane can be installed with Helm
+
+- It will run as a set of containers on Kubernetes worker nodes
 
 .exercise[
 
-```bash  
-    kubectl get pod -n openebs
-```
-]
-
-Let's check the new StorageClasses:
-
-.exercise[
-
-```bash  
-    kubectl get sc
-```
+  - Install OpenEBS:
+  ```bash  
+    helm upgrade --install openebs openebs \
+         --repo https://openebs.github.io/charts \
+         --namespace openebs --create-namespace
+  ```
 ]
 
 ---
 
-## Default Storage Classes 
+## Checking what was installed
 
-For a simple testing of OpenEBS, you can use the below default storage classes:
+- Wait a little bit ...
 
- - **openebs-jiva-default** for provisioning Jiva Volume (this uses default pool which means the data replicas are created in the /mnt/openebs_disk directory of the Jiva replica pod)
+.exercise[
 
- - **openebs-hostpath** for provisioning Local PV on hostpath.
+- Look at the pods in the `openebs` namespace:
+  ```bash  
+      kubectl get pods --namespace openebs
+  ```
 
- - **openebs-device** for provisioning Local PV on device.
+- And the StorageClasses that were created:
+  ```bash  
+      kubectl get sc
+  ```
 
-For using real disks, you have to create *cStorPools* or *Jiva* pools or *OpenEBS Local PV* based on the requirement and then create corresponding StorageClasses or use default StorageClasses to use them.
+]
 
 ---
 
-## Selecting an OpenEBS Storage Engine
+## The default StorageClasses
 
-Storage engine is chosen by specifying the annotation `openebs.io/cas-type` in the StorageClass specification. StorageClass defines the provisioner details. Separate provisioners are specified for each CAS engine.
+- OpenEBS typically creates three default StorageClasses
 
-Example for Local PV host path:
+- `openebs-jiva-default` provisions 3 replicated Jiva pods per volume
+
+  - data is stored in `/openebs` in the replica pods
+  - `/openebs` is a localpath volume mapped to `/var/openebs/pvc-...` on the node
+
+- `openebs-hostpath` uses LocalPV with local directories
+
+  - volumes are hostpath volumes created in `/var/openebs/local` on each node
+
+- `openebs-device` uses LocalPV with local block devices
+
+  - requires available disks and/or a bit of extra configuration
+  - the default configuration filters out loop, LVM, MD devices
+
+---
+
+## When do we need custom StorageClasses?
+
+- To store LocalPV hostpath volumes on a different path on the host
+
+- To change the number of replicated Jiva pods
+
+- To use a different Jiva pool
+
+  (i.e. a different path on the host to store the Jiva volumes)
+
+- To create a cStor pool
+
+- ...
+
+---
+
+class: extra-details
+
+## Defining a custom StorageClass
+
+Example for a LocalPV hostpath class using an extra mount on `/mnt/vol001`:
 
 ```yaml
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
-  name: localpv-hostpath-sc
+  name: localpv-hostpath-mntvol001
   annotations:
     openebs.io/cas-type: local
     cas.openebs.io/config: |
       - name: BasePath
-        value: "/var/openebs/local"
+        value: "/mnt/vol001"
       - name: StorageType
         value: "hostpath"
 provisioner: openebs.io/local
 ```
 
+- `provisioner` needs to be set accordingly
+- Storage engine is chosen by specifying the annotation `openebs.io/cas-type`
+- Storage engine configuration is set with the annotation `cas.openebs.io/config` 
+
 ---
 
-## Exploring the host path StorageClass
+## Checking the default hostpath StorageClass
+
+- Let's inspect the StorageClass that OpenEBS created for us
 
 .exercise[
-    - Let's look at the OpenEBS Local PV host path StorageClass
-    ```bash
-    kubectl get sc openebs-hostpath -oyaml
-    ``` 
+
+- Let's look at the OpenEBS LocalPV hostpath StorageClass:
+  ```bash
+  kubectl get storageclass openebs-hostpath -o yaml
+  ``` 
 ]
 
 ---
 
-##  Create a host path PVC
+## Create a host path PVC
+
+- Let's create a Persistent Volume Claim using an explicit StorageClass
 
-Let's create a Persistent Volume Claim
 .exercise[
+
 ```bash
 kubectl apply -f - <<EOF
 kind: PersistentVolumeClaim
@@ -120,46 +181,341 @@ spec:
     requests:
       storage: 1G
 EOF
-
-kubectl get pvc
 ```
+
 ]
 
 ---
-## Create a pod to consume the PV
+
+## Making sure that a PV was created for our PVC
+
+- Normally, the `openebs-hostpath` StorageClass created a PV for our PVC
 
 .exercise[
-- Create a pod from yaml:
-```
-kubectl apply -f ~/container.training/k8s/openebs-pod.yaml
-```
-- Look at the pod definition:
-```
-  volumes:
-  \- name: my-storage
-     persistentVolumeClaim:
-       claimName: local-hostpath-pvc
-  containers:
-....  
-    volumeMounts:
-    \- mountPath: /mnt/storage
-      name: my-storage
-```
+
+- Look at the PV and PVC:
+  ```bash
+  kubectl get pv,pvc
+  ```
+
 ]
 
 ---
-### Verify the data is written
+
+## Create a Pod to consume the PV
 
 .exercise[
- - Get the worker node where the pod is located
- ```
-kubectl get pod openebs-local-hostpath-pod -ojsonpath
-="{ .spec.nodeName }"
-```
-- ssh into the node
 
-- check the volume content
-```
-sudo cat /var/openebs/local/pvc-*/greet.txt
-```
-]
\ No newline at end of file
+- Create a Pod using that PVC:
+  ```bash
+  kubectl apply -f ~/container.training/k8s/openebs-pod.yaml
+  ```
+
+- Here are the sections that declare and use the volume:
+  ```yaml
+    volumes:
+    - name: my-storage
+      persistentVolumeClaim:
+        claimName: local-hostpath-pvc
+    containers:
+    ...  
+      volumeMounts:
+      - mountPath: /mnt/storage
+        name: my-storage
+  ```
+
+]
+
+---
+
+## Verify that data is written on the node
+
+- Let's find the file written by the Pod on the node where the Pod is running
+
+.exercise[
+
+- Get the worker node where the pod is located
+  ```bash
+  kubectl get pod openebs-local-hostpath-pod -ojsonpath={.spec.nodeName}
+  ```
+
+- SSH into the node
+
+- Check the volume content
+  ```bash
+  sudo tail /var/openebs/local/pvc-*/greet.txt
+  ```
+
+]
+
+---
+
+## Heads up!
+
+- The following labs and exercises will use the Jiva storage class
+
+- This storage class creates 3 replicas by default
+
+- It uses *anti-affinity* placement constraits to put these replicas on different nodes
+
+- **This requires a cluster with multiple nodes!**
+
+- It also requires the iSCSI client (aka *initiator*) to be installed on the nodes
+
+- On many platforms, the iSCSI client is preinstalled and will start automatically
+
+- If it doesn't, you might want to check [this documentation page] for details
+
+[this documentation page]: https://docs.openebs.io/docs/next/prerequisites.html
+
+---
+
+## The default StorageClass
+
+- The PVC that we defined earlier specified an explicit StorageClass
+
+- We can also set a default StorageClass
+
+- It will then be used for all PVC that *don't* specify and explicit StorageClass
+
+- This is done with the annotation `storageclass.kubernetes.io/is-default-class`
+
+.exercise[
+
+- Check if we have a default StorageClass:
+  ```bash
+  kubectl get storageclasses
+  ```
+]
+
+- The default StorageClass (if there is one) is shown with `(default)`
+
+---
+
+## Setting a default StorageClass
+
+- Let's set the default StorageClass to use `openebs-jiva-default`
+
+.exercise[
+
+- Remove the annotation (just in case we already have a default class):
+  ```bash
+  kubectl annotate storageclass storageclass.kubernetes.io/is-default-class- --all
+  ```
+
+- Annotate the Jiva StorageClass:
+  ```bash
+  kubectl annotate storageclasses \
+      openebs-jiva-default storageclass.kubernetes.io/is-default-class=true
+  ```
+
+- Check the result:
+  ```bash
+  kuectl get storageclasses
+  ```
+
+]
+
+---
+
+## Creating a Pod using the Jiva class
+
+- We will create a Pod running PostgreSQL, using the default class
+
+.exercise[
+
+- Create the Pod:
+  ```bash
+  kubectl apply -f ~/container.training/k8s/postgres.yaml
+  ```
+
+- Wait for the PV, PVC, and Pod to be up:
+  ```bash
+  watch kubectl get pv,pvc,pod
+  ```
+
+- We can also check what's going on in the `openebs` namespace:
+  ```bash
+  watch kubectl get pods --namespace openebs
+  ```
+
+]
+
+---
+
+## Node failover
+
+⚠️ This will partially break your cluster!
+
+- We are going to disconnect the node running PostgreSQL from the cluster
+
+- We will see what happens, and how to recover
+
+- We will not reconnect the node to the cluster
+
+- This whole lab will take at least 10-15 minutes (due to various timeouts)
+
+⚠️ Only do this lab at the very end, when you don't want to run anything else after!
+
+---
+
+## Disconnecting the node from the cluster
+
+.exercise[
+
+- Find out where the Pod is running, and SSH into that node:
+  ```bash
+  kubectl get pod postgres-0 -o jsonpath={.spec.nodeName}
+  ssh nodeX
+  ```
+
+- Check the name of the network interface:
+  ```bash
+  sudo ip route ls default
+  ```
+
+- The output should look like this:
+  ```
+  default via 10.10.0.1 `dev ensX` proto dhcp src 10.10.0.13 metric 100 
+  ```
+
+- Shutdown the network interface:
+  ```bash
+  sudo ip link set ensX down
+  ```
+
+]
+
+---
+
+## Watch what's going on
+
+- Let's look at the status of Nodes, Pods, and Events
+
+.exercise[
+
+- In a first pane/tab/window, check Nodes and Pods:
+  ```bash
+  watch kubectl get nodes,pods -o wide
+  ```
+
+- In another pane/tab/window, check Events:
+  ```bash
+  kubectl get events --watch
+  ```
+
+]
+
+---
+
+## Node Ready → NotReady
+
+- After \~30 seconds, the control plane stops receiving heartbeats from the Node
+
+- The Node is marked NotReady
+
+- It is not *schedulable* anymore
+
+  (the scheduler won't place new pods there, except some special cases)
+
+- All Pods on that Node are also *not ready*
+
+  (they get removed from service Endpoints)
+
+- ... But nothing else happens for now
+
+  (the control plane is waiting: maybe the Node will come back shortly?)
+
+---
+
+## Pod eviction
+
+- After \~5 minutes, the control plane will evict most Pods from the Node
+
+- These Pods are now `Terminating`
+
+- The Pods controlled by e.g. ReplicaSets are automatically moved
+
+  (or rather: new Pods are created to replace them)
+
+- But nothing happens to the Pods controlled by StatefulSets at this point
+
+  (they remain `Terminating` forever)
+
+- Why? 🤔
+
+--
+
+- This is to avoid *split brain scenarios*
+
+---
+
+class: extra-details
+
+## Split brain 🧠⚡️🧠
+
+- Imagine that we create a replacement pod `postgres-0` on another Node
+
+- And 15 minutes later, the Node is reconnected and the original `postgres-0` comes back
+
+- Which one is the "right" one?
+
+- What if they have conflicting data?
+
+😱
+
+- We *cannot* let that happen!
+
+- Kubernetes won't do it
+
+- ... Unless we tell it to
+
+---
+
+## The Node is gone
+
+- One thing we can do, is tell Kubernetes "the Node won't come back"
+
+  (there are other methods; but this one is the simplest one here)
+
+- This is done with a simple `kubectl delete node`
+
+.exercise[
+
+- `kubectl delete` the Node that we disconnected
+
+]
+
+---
+
+## Pod rescheduling
+
+- Kubernetes removes the Node
+
+- After a brief period of time (\~1 minute) the "Terminating" Pods are removed
+
+- A replacement Pod is created on another Node
+
+- ... But it doens't start yet!
+
+- Why? 🤔
+
+---
+
+## Multiple attachment
+
+- By default, a disk can only be attached to one Node at a time
+
+  (sometimes it's a hardware or API limitation; sometimes enforced in software)
+
+- In our Events, we should see `FailedAttachVolume` and `FailedMount` messages
+
+- After \~5 more minutes, the disk will be force-detached from the old Node
+
+- ... Which will allow attaching it to the new Node!
+
+🎉
+
+- The Pod will then be able to start
+
+- Failover is complete!
diff --git a/slides/kube-fullday.yml b/slides/kube-fullday.yml
index c43169f4..047587d6 100644
--- a/slides/kube-fullday.yml
+++ b/slides/kube-fullday.yml
@@ -106,6 +106,7 @@ content:
   #- k8s/statefulsets.md
   #- k8s/local-persistent-volumes.md
   #- k8s/portworx.md
+  #- k8s/openebs.md
   #- k8s/extending-api.md
   #- k8s/crd.md
   #- k8s/admission.md
diff --git a/slides/kube-selfpaced.yml b/slides/kube-selfpaced.yml
index aa533663..f2d3fde5 100644
--- a/slides/kube-selfpaced.yml
+++ b/slides/kube-selfpaced.yml
@@ -110,6 +110,7 @@ content:
   - k8s/statefulsets.md
   - k8s/local-persistent-volumes.md
   - k8s/portworx.md
+  - k8s/openebs.md
 -
   - k8s/logs-centralized.md
   - k8s/prometheus.md
diff --git a/slides/kube-twodays.yml b/slides/kube-twodays.yml
index bb4272f9..0e384ccd 100644
--- a/slides/kube-twodays.yml
+++ b/slides/kube-twodays.yml
@@ -107,6 +107,7 @@ content:
   - k8s/statefulsets.md
   - k8s/local-persistent-volumes.md
   - k8s/portworx.md
+  #- k8s/openebs.md
   #- k8s/extending-api.md
   #- k8s/admission.md
   #- k8s/operators.md