diff --git a/k8s/persistent-consul.yaml b/k8s/persistent-consul.yaml new file mode 100644 index 00000000..ff9d5955 --- /dev/null +++ b/k8s/persistent-consul.yaml @@ -0,0 +1,95 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: consul +rules: + - apiGroups: [ "" ] + resources: [ pods ] + verbs: [ get, list ] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: consul +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: consul +subjects: + - kind: ServiceAccount + name: consul + namespace: orange +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: consul +--- +apiVersion: v1 +kind: Service +metadata: + name: consul +spec: + ports: + - port: 8500 + name: http + selector: + app: consul +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: consul +spec: + serviceName: consul + replicas: 3 + selector: + matchLabels: + app: consul + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + template: + metadata: + labels: + app: consul + spec: + serviceAccountName: consul + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - consul + topologyKey: kubernetes.io/hostname + terminationGracePeriodSeconds: 10 + containers: + - name: consul + image: "consul:1.4.4" + volumeMounts: + - name: data + mountPath: /consul/data + args: + - "agent" + - "-bootstrap-expect=3" + - "-retry-join=provider=k8s namespace=orange label_selector=\"app=consul\"" + - "-client=0.0.0.0" + - "-data-dir=/consul/data" + - "-server" + - "-ui" + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - consul leave diff --git a/k8s/volumes-for-consul.yaml b/k8s/volumes-for-consul.yaml new file mode 100644 index 00000000..8d75e8ea --- /dev/null +++ b/k8s/volumes-for-consul.yaml @@ -0,0 +1,70 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: consul-node2 + annotations: + node: node2 +spec: + capacity: + storage: 10Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Delete + local: + path: /mnt/consul + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - node2 +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: consul-node3 + annotations: + node: node3 +spec: + capacity: + storage: 10Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Delete + local: + path: /mnt/consul + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - node3 +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: consul-node4 + annotations: + node: node4 +spec: + capacity: + storage: 10Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Delete + local: + path: /mnt/consul + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - node4 + diff --git a/prepare-vms/lib/commands.sh b/prepare-vms/lib/commands.sh index 6daec799..f6d440b2 100644 --- a/prepare-vms/lib/commands.sh +++ b/prepare-vms/lib/commands.sh @@ -248,6 +248,14 @@ EOF" sudo tar -C /usr/local/bin -zx ship fi" + # Install the AWS IAM authenticator + pssh " + if [ ! -x /usr/local/bin/aws-iam-authenticator ]; then + ##VERSION## + sudo curl -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.12.7/2019-03-27/bin/linux/amd64/aws-iam-authenticator + sudo chmod +x /usr/local/bin/aws-iam-authenticator + fi" + sep "Done" } diff --git a/slides/k8s/local-persistent-volumes.md b/slides/k8s/local-persistent-volumes.md new file mode 100644 index 00000000..b7718bd0 --- /dev/null +++ b/slides/k8s/local-persistent-volumes.md @@ -0,0 +1,244 @@ +# Local Persistent Volumes + +- We want to run that Consul cluster *and* actually persist data + +- But we don't have a distributed storage system + +- We are going to use local volumes instead + + (similar conceptually to `hostPath` volumes) + +- We can use local volumes without installing extra plugins + +- However, they are tied to a node + +- If that node goes down, the volume becomes unavailable + +--- + +## With or without dynamic provisioning + +- We will deploy a Consul cluster *with* persistence + +- That cluster's StatefulSet will create PVCs + +- These PVCs will remain unbound¹, until we will create local volumes manually + + (we will basically do the job of the dynamic provisioner) + +- Then, we will see how to automate that with a dynamic provisioner + +.footnote[¹Unbound = without an associated Persistent Volume.] + +--- + +## Work in a separate namespace + +- To avoid conflicts with existing resources, let's create and use a new namespace + +.exercise[ + +- Create a new namespace: + ```bash + kubectl create namespace orange + ``` + +- Switch to that namespace: + ```bash + kns orange + ``` + +] + +.warning[Make sure to call that namespace `orange`, because that name is hardcoded in the YAML files.] + +--- + +## Deploying Consul + +- We will use a slightly different YAML file + +- The only differences between that file and the previous one are: + + - `volumeClaimTemplate` defined in the Stateful Set spec + + - the corresponding `volumeMounts` in the Pod spec + + - the namespace `orange` used for discovery of Pods + +.exercise[ + +- Apply the persistent Consul YAML file: + ```bash + kubectl apply -f ~/container.training/k8s/persistent-consul.yaml + ``` + +] + +--- + +## Observing the situation + +- Let's look at Persistent Volume Claims and Pods + +.exercise[ + +- Check that we now have an unbound Persistent Volume Claim: + ```bash + kubectl get pvc + ``` + +- We don't have any Persistent Volume: + ```bash + kubectl get pv + ``` + +- The Pod `consul-0` is not scheduled yet: + ```bash + kubectl get pods -o wide + ``` + +] + +*Hint: leave these commands running with `-w` in different windows.* + +--- + +## Explanations + +- In a Stateful Set, the Pods are started one by one + +- `consul-1` won't be created until `consul-0` is running + +- `consul-0` has a dependency on an unbound Persistent Volume Claim + +- The scheduler won't schedule the Pod until the PVC is bound + + (because the PVC might be bound to a volume that is only available on a subset of nodes; for instance EBS are tied to an availability zone) + +--- + +## Creating Persistent Volumes + +- Let's create 3 local directories (`/mnt/consul`) on node2, node3, node4 + +- Then create 3 Persistent Volumes corresponding to these directories + +.exercise[ + +- Create the local directories: + ```bash + for NODE in node2 node3 node4; do + ssh $NODE sudo mkdir -p /mnt/consul + done + ``` + +- Create the PV objects: + ```bash + kubectl apply -f ~/container.training/k8s/volumes-for-consul.yaml + ``` + +] + +--- + +## Check our Consul cluster + +- The PVs that we created will be automatically matched with the PVCs + +- Once a PVC is bound, its pod can start normally + +- Once the pod `consul-0` has started, `consul-1` can be created, etc. + +- Eventually, our Consul cluster is up, and backend by "persistent" volumes + +.exercise[ + +- Check that our Consul clusters has 3 members indeed: + ```bash + kubectl exec consul-0 consul members + ``` + +] + +--- + +## Devil is in the details (1/2) + +- The size of the Persistent Volumes is bogus + + (it is used when matching PVs and PVCs together, but there is no actual quota or limit) + +--- + +## Devil is in the details (2/2) + +- This specific example worked because we had exactly 1 free PV per node: + + - if we had created multiple PVs per node ... + + - we could have ended with two PVCs bound to PVs on the same node ... + + - which would have required two pods to be on the same node ... + + - which is forbidden by the anti-affinity constraints in the StatefulSet + +- To avoid that, we need to associated the PVs with a Storage Class that has: + ```yaml + volumeBindingMode: WaitForFirstConsumer + ``` + (this means that a PVC will be bound to a PV only after being used by a Pod) + +- See [this blog post](https://kubernetes.io/blog/2018/04/13/local-persistent-volumes-beta/) for more details + +--- + +## Bulk provisioning + +- It's not practical to manually create directories and PVs for each app + +- We *could* pre-provision a number of PVs across our fleet + +- We could even automate that with a Daemon Set: + + - creating a number of directories on each node + + - creating the corresponding PV objects + +- We also need to recycle volumes + +- ... This can quickly get out of hand + +--- + +## Dynamic provisioning + +- We could also write our own provisioner, which would: + + - watch the PVCs across all namespaces + + - when a PVC is created, create a corresponding PV on a node + +- Or we could use one of the dynamic provisioners for local persistent volumes + + (for instance the [Rancher local path provisioner](https://github.com/rancher/local-path-provisioner)) + +--- + +## Strategies for local persistent volumes + +- Remember, when a node goes down, the volumes on that node become unavailable + +- High availability will require another layer of replication + + (like what we've just seen with Consul; or primary/secondary; etc) + +- Pre-provisioning PVs makes sense for machines with local storage + + (e.g. cloud instance storage; or storage directly attached to a physical machine) + +- Dynamic provisioning makes sense for large number of applications + + (when we can't or won't dedicate a whole disk to a volume) + +- It's possible to mix both (using distinct Storage Classes) diff --git a/slides/k8s/statefulsets.md b/slides/k8s/statefulsets.md index 3dc45286..9692eef0 100644 --- a/slides/k8s/statefulsets.md +++ b/slides/k8s/statefulsets.md @@ -34,13 +34,13 @@ - Each pod can discover the IP address of the others easily -- The pods can have persistent volumes attached to them +- The pods can persist data on attached volumes 🤔 Wait a minute ... Can't we already attach volumes to pods and deployments? --- -## Volumes and Persistent Volumes +## Revisiting volumes - [Volumes](https://kubernetes.io/docs/concepts/storage/volumes/) are used for many purposes: @@ -50,13 +50,13 @@ - accessing storage systems -- The last type of volumes is known as a "Persistent Volume" +- Let's see examples of the latter usage --- -## Persistent Volumes types +## Volumes types -- There are many [types of Persistent Volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#types-of-persistent-volumes) available: +- There are many [types of volumes](https://kubernetes.io/docs/concepts/storage/volumes/#types-of-volumes) available: - public cloud storage (GCEPersistentDisk, AWSElasticBlockStore, AzureDisk...) @@ -74,7 +74,7 @@ --- -## Using a Persistent Volume +## Using a cloud volume Here is a pod definition using an AWS EBS volume (that has to be created first): @@ -99,7 +99,32 @@ spec: --- -## Shortcomings of Persistent Volumes +## Using an NFS volume + +Here is another example using a volume on an NFS server: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: pod-using-my-nfs-volume +spec: + containers: + - image: ... + name: container-using-my-nfs-volume + volumeMounts: + - mountPath: /my-nfs + name: my-nfs-volume + volumes: + - name: my-nfs-volume + nfs: + server: 192.168.0.55 + path: "/exports/assets" +``` + +--- + +## Shortcomings of volumes - Their lifecycle (creation, deletion...) is managed outside of the Kubernetes API @@ -125,17 +150,47 @@ spec: - This type is a *Persistent Volume Claim* +- A Persistent Volume Claim (PVC) is a resource type + + (visible with `kubectl get persistentvolumeclaims` or `kubectl get pvc`) + +- A PVC is not a volume; it is a *request for a volume* + +--- + +## Persistent Volume Claims in practice + - Using a Persistent Volume Claim is a two-step process: - creating the claim - using the claim in a pod (as if it were any other kind of volume) -- Between these two steps, something will happen behind the scenes: +- A PVC starts by being Unbound (without an associated volume) - - Kubernetes will associate an existing volume with the claim +- Once it is associated with a Persistent Volume, it becomes Bound - - ... or dynamically create a volume if possible and necessary +- A Pod referring an unbound PVC will not start + + (but as soon as the PVC is bound, the Pod can start) + +--- + +## Binding PV and PVC + +- A Kubernetes controller continuously watches PV and PVC objects + +- When it notices an unbound PVC, it tries to find a satisfactory PV + + ("satisfactory" in terms of size and other characteristics; see next slide) + +- If no PV fits the PVC, a PV can be created dynamically + + (this requires to configure a *dynamic provisioner*, more on that later) + +- Otherwise, the PVC remains unbound indefinitely + + (until we manually create a PV or setup dynamic provisioning) --- @@ -147,7 +202,9 @@ spec: - the access mode (e.g. "read-write by a single pod") -- It can also give extra details, like: +- Optionally, it can also specify a Storage Class + +- The Storage Class indicates: - which storage system to use (e.g. Portworx, EBS...) @@ -155,8 +212,6 @@ spec: e.g.: "replicate the data 3 times, and use SSD media" -- The extra details are provided by specifying a Storage Class - --- ## What's a Storage Class? @@ -167,15 +222,15 @@ spec: - It indicates which *provisioner* to use + (which controller will create the actual volume) + - And arbitrary parameters for that provisioner (replication levels, type of disk ... anything relevant!) -- It is necessary to define a Storage Class to use [dynamic provisioning](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/) +- Storage Classes are required if we want to use [dynamic provisioning](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/) -- Conversely, it is not necessary to define one if you will create volumes manually - - (we will see dynamic provisioning in action later) + (but we can also create volumes manually, and ignore Storage Classes) --- @@ -200,7 +255,7 @@ spec: ## Using a Persistent Volume Claim -Here is the same definition as earlier, but using a PVC: +Here is a Pod definition like the ones shown earlier, but using a PVC: ```yaml apiVersion: v1 @@ -212,7 +267,7 @@ spec: - image: ... name: container-using-a-claim volumeMounts: - - mountPath: /my-ebs + - mountPath: /my-vol name: my-volume volumes: - name: my-volume diff --git a/slides/k8s/volumes.md b/slides/k8s/volumes.md index 2773d185..dc7c355a 100644 --- a/slides/k8s/volumes.md +++ b/slides/k8s/volumes.md @@ -18,6 +18,8 @@ --- +class: extra-details + ## Kubernetes volumes vs. Docker volumes - Kubernetes and Docker volumes are very similar @@ -35,13 +37,35 @@ - Kubernetes volumes are also used to expose configuration and secrets - Docker has specific concepts for configuration and secrets - +
(but under the hood, the technical implementation is similar) - If you're not familiar with Docker volumes, you can safely ignore this slide! --- +## Volumes ≠ Persistent Volumes + +- Volumes and Persistent Volumes are related, but very different! + +- *Volumes*: + + - appear in Pod specifications (see next slide) + + - do not exist as API resources (**cannot** do `kubectl get volumes`) + +- *Persistent Volumes*: + + - are API resources (**can** do `kubectl get persistentvolumes`) + + - correspond to concrete volumes (e.g. on a SAN, EBS, etc.) + + - cannot be associated to a Pod directly; but through a Persistent Volume Claim + + - won't be discussed further in this section + +--- + ## A simple volume example ```yaml diff --git a/slides/kube-fullday.yml b/slides/kube-fullday.yml index 32dc3c87..c857a9c6 100644 --- a/slides/kube-fullday.yml +++ b/slides/kube-fullday.yml @@ -67,6 +67,7 @@ chapters: #- - k8s/owners-and-dependents.md # - k8s/extending-api.md # - k8s/statefulsets.md +# - k8s/local-persistent-volumes.md # - k8s/portworx.md - - k8s/whatsnext.md - k8s/links.md diff --git a/slides/kube-selfpaced.yml b/slides/kube-selfpaced.yml index d4fb1269..d4a20d43 100644 --- a/slides/kube-selfpaced.yml +++ b/slides/kube-selfpaced.yml @@ -67,6 +67,7 @@ chapters: - - k8s/owners-and-dependents.md - k8s/extending-api.md - k8s/statefulsets.md + - k8s/local-persistent-volumes.md - k8s/portworx.md - k8s/staticpods.md - - k8s/whatsnext.md diff --git a/slides/kube-twodays.yml b/slides/kube-twodays.yml index d02693a6..b7bd9a9a 100644 --- a/slides/kube-twodays.yml +++ b/slides/kube-twodays.yml @@ -67,6 +67,7 @@ chapters: #- k8s/owners-and-dependents.md - k8s/extending-api.md - - k8s/statefulsets.md + - k8s/local-persistent-volumes.md - k8s/portworx.md - k8s/staticpods.md - - k8s/whatsnext.md