From b69119eed4e236aa81354b7bbb21d6a061719479 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Thu, 16 May 2019 22:21:40 -0500 Subject: [PATCH 01/27] Add operator chapter with nice ElasticSearch demo --- k8s/elasticsearch-cluster.yaml | 21 ++ k8s/elasticsearch-operator.yaml | 94 ++++++++ k8s/filebeat.yaml | 167 +++++++++++++++ k8s/local-path-storage.yaml | 110 ++++++++++ slides/k8s/operators.md | 366 ++++++++++++++++++++++++++++++++ slides/kube-fullday.yml | 1 + slides/kube-selfpaced.yml | 1 + slides/kube-twodays.yml | 1 + 8 files changed, 761 insertions(+) create mode 100644 k8s/elasticsearch-cluster.yaml create mode 100644 k8s/elasticsearch-operator.yaml create mode 100644 k8s/filebeat.yaml create mode 100644 k8s/local-path-storage.yaml create mode 100644 slides/k8s/operators.md diff --git a/k8s/elasticsearch-cluster.yaml b/k8s/elasticsearch-cluster.yaml new file mode 100644 index 00000000..23d8108d --- /dev/null +++ b/k8s/elasticsearch-cluster.yaml @@ -0,0 +1,21 @@ +apiVersion: enterprises.upmc.com/v1 +kind: ElasticsearchCluster +metadata: + name: es +spec: + kibana: + image: docker.elastic.co/kibana/kibana-oss:6.1.3 + image-pull-policy: Always + cerebro: + image: upmcenterprises/cerebro:0.7.2 + image-pull-policy: Always + elastic-search-image: upmcenterprises/docker-elasticsearch-kubernetes:6.1.3_0 + image-pull-policy: Always + client-node-replicas: 2 + master-node-replicas: 3 + data-node-replicas: 3 + network-host: 0.0.0.0 + use-ssl: false + data-volume-size: 10Gi + java-options: "-Xms512m -Xmx512m" + diff --git a/k8s/elasticsearch-operator.yaml b/k8s/elasticsearch-operator.yaml new file mode 100644 index 00000000..0049541e --- /dev/null +++ b/k8s/elasticsearch-operator.yaml @@ -0,0 +1,94 @@ +# This is mirrored from https://github.com/upmc-enterprises/elasticsearch-operator/blob/master/example/controller.yaml but using the elasticsearch-operator namespace instead of operator +--- +apiVersion: v1 +kind: Namespace +metadata: + name: elasticsearch-operator +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: elasticsearch-operator + namespace: elasticsearch-operator +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: elasticsearch-operator +rules: +- apiGroups: ["extensions"] + resources: ["deployments", "replicasets", "daemonsets"] + verbs: ["create", "get", "update", "delete", "list"] +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "get", "update", "delete", "list"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "create", "delete", "deletecollection"] +- apiGroups: [""] + resources: ["persistentvolumes", "persistentvolumeclaims", "services", "secrets", "configmaps"] + verbs: ["create", "get", "update", "delete", "list"] +- apiGroups: ["batch"] + resources: ["cronjobs", "jobs"] + verbs: ["create", "get", "deletecollection", "delete"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["list", "get", "watch"] +- apiGroups: ["apps"] + resources: ["statefulsets", "deployments"] + verbs: ["*"] +- apiGroups: ["enterprises.upmc.com"] + resources: ["elasticsearchclusters"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: elasticsearch-operator + namespace: elasticsearch-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: elasticsearch-operator +subjects: +- kind: ServiceAccount + name: elasticsearch-operator + namespace: elasticsearch-operator +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: elasticsearch-operator + namespace: elasticsearch-operator +spec: + replicas: 1 + template: + metadata: + labels: + name: elasticsearch-operator + spec: + containers: + - name: operator + image: upmcenterprises/elasticsearch-operator:0.2.0 + imagePullPolicy: Always + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 8000 + name: http + livenessProbe: + httpGet: + path: /live + port: 8000 + initialDelaySeconds: 10 + timeoutSeconds: 10 + readinessProbe: + httpGet: + path: /ready + port: 8000 + initialDelaySeconds: 10 + timeoutSeconds: 5 + serviceAccount: elasticsearch-operator diff --git a/k8s/filebeat.yaml b/k8s/filebeat.yaml new file mode 100644 index 00000000..690e9613 --- /dev/null +++ b/k8s/filebeat.yaml @@ -0,0 +1,167 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-config + namespace: kube-system + labels: + k8s-app: filebeat +data: + filebeat.yml: |- + filebeat.config: + inputs: + # Mounted `filebeat-inputs` configmap: + path: ${path.config}/inputs.d/*.yml + # Reload inputs configs as they change: + reload.enabled: false + modules: + path: ${path.config}/modules.d/*.yml + # Reload module configs as they change: + reload.enabled: false + + # To enable hints based autodiscover, remove `filebeat.config.inputs` configuration and uncomment this: + #filebeat.autodiscover: + # providers: + # - type: kubernetes + # hints.enabled: true + + processors: + - add_cloud_metadata: + + cloud.id: ${ELASTIC_CLOUD_ID} + cloud.auth: ${ELASTIC_CLOUD_AUTH} + + output.elasticsearch: + hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-inputs + namespace: kube-system + labels: + k8s-app: filebeat +data: + kubernetes.yml: |- + - type: docker + containers.ids: + - "*" + processors: + - add_kubernetes_metadata: + in_cluster: true +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat +spec: + template: + metadata: + labels: + k8s-app: filebeat + spec: + serviceAccountName: filebeat + terminationGracePeriodSeconds: 30 + containers: + - name: filebeat + image: docker.elastic.co/beats/filebeat-oss:7.0.1 + args: [ + "-c", "/etc/filebeat.yml", + "-e", + ] + env: + - name: ELASTICSEARCH_HOST + value: elasticsearch-es.default.svc.cluster.local + - name: ELASTICSEARCH_PORT + value: "9200" + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: ELASTIC_CLOUD_ID + value: + - name: ELASTIC_CLOUD_AUTH + value: + securityContext: + runAsUser: 0 + # If using Red Hat OpenShift uncomment this: + #privileged: true + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/filebeat.yml + readOnly: true + subPath: filebeat.yml + - name: inputs + mountPath: /usr/share/filebeat/inputs.d + readOnly: true + - name: data + mountPath: /usr/share/filebeat/data + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + volumes: + - name: config + configMap: + defaultMode: 0600 + name: filebeat-config + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: inputs + configMap: + defaultMode: 0600 + name: filebeat-inputs + # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart + - name: data + hostPath: + path: /var/lib/filebeat-data + type: DirectoryOrCreate +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: filebeat +subjects: +- kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: filebeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: filebeat + labels: + k8s-app: filebeat +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - pods + verbs: + - get + - watch + - list +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat +--- diff --git a/k8s/local-path-storage.yaml b/k8s/local-path-storage.yaml new file mode 100644 index 00000000..7374a02a --- /dev/null +++ b/k8s/local-path-storage.yaml @@ -0,0 +1,110 @@ +# This is a local copy of: +# https://github.com/rancher/local-path-provisioner/blob/master/deploy/local-path-storage.yaml +--- +apiVersion: v1 +kind: Namespace +metadata: + name: local-path-storage +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: local-path-provisioner-service-account + namespace: local-path-storage +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: local-path-provisioner-role + namespace: local-path-storage +rules: +- apiGroups: [""] + resources: ["nodes", "persistentvolumeclaims"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["endpoints", "persistentvolumes", "pods"] + verbs: ["*"] +- apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: local-path-provisioner-bind + namespace: local-path-storage +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: local-path-provisioner-role +subjects: +- kind: ServiceAccount + name: local-path-provisioner-service-account + namespace: local-path-storage +--- +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: local-path-provisioner + namespace: local-path-storage +spec: + replicas: 1 + selector: + matchLabels: + app: local-path-provisioner + template: + metadata: + labels: + app: local-path-provisioner + spec: + serviceAccountName: local-path-provisioner-service-account + containers: + - name: local-path-provisioner + image: rancher/local-path-provisioner:v0.0.8 + imagePullPolicy: Always + command: + - local-path-provisioner + - --debug + - start + - --config + - /etc/config/config.json + volumeMounts: + - name: config-volume + mountPath: /etc/config/ + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumes: + - name: config-volume + configMap: + name: local-path-config +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: local-path +provisioner: rancher.io/local-path +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Delete +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: local-path-config + namespace: local-path-storage +data: + config.json: |- + { + "nodePathMap":[ + { + "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES", + "paths":["/opt/local-path-provisioner"] + } + ] + } + diff --git a/slides/k8s/operators.md b/slides/k8s/operators.md new file mode 100644 index 00000000..66253edb --- /dev/null +++ b/slides/k8s/operators.md @@ -0,0 +1,366 @@ +# Operators + +- Operators are one of the many ways to extend Kubernetes + +- We will define operators + +- We will see how they work + +- We will install a specific operator (for ElasticSearch) + +- We will use it to provision an ElasticSearch cluster + +--- + +## What are operators? + +- Operators combine two things: + + - Custom Resource Definitions + + - controller code watching the corresponding resources and acting upon them + +- A given operator can define one or multiple CRDs + +- The controller code (control loop) can run within the cluster or elsewhere + + (running as a Deployment with 1 replica is a common scenario) + +--- + +## Why use operators? + +- Kubernetes gives us Deployments, StatefulSets, Services ... + +- These mechanisms give us building blocks to deploy applications + +- They work great for services that are made of *N* identical containers + + (like stateless ones) + +- They also work great for some stateful applications like Consul, etcd ... + + (with the help of highly persistent volumes) + +- They're not enough for complex services: + + - where different containers have different roles + + - where extra steps have to be taken when scaling or replacing containers + +--- + +## Use-cases for operators + +- Systems with primary/secondary replication + + Examples: MariaDB, MySQL, PostgreSQL, Redis ... + +- Systems where different groups of nodes have different roles + + Examples: ElasticSearch, MongoDB ... + +- Systems that depend on other non-trivial systems + + Examples: Flink, Kafka, ... + +--- + +## More use-cases + +- Representing and managing external resources + + (Example: [AWS Service Operator](https://operatorhub.io/operator/alpha/aws-service-operator.v0.0.1)) + +- Managing complex cluster add-ons + + (Example: [Istio operator](https://operatorhub.io/operator/beta/istio-operator.0.1.6)) + +- Deploying and managing our applications lifecycles + + (more on that later) + +--- + +## How operators work + +- An operator creates one or more CRDs + + (i.e., it creates new "Kinds" of resources on our cluster) + +- The operator also runs a *controller* that will watch its resources + +- Each time we create/updated/delete a resource, the controller is notified + + (we could write our own cheap controller with `kubectl get --watch`) + +--- + +## One operator in action + +- We will install the UPMC Enterprises ElasticSearch operator + +- This operator requires PersistentVolumes + +- We will install Rancher's [local path storage provisioner](https://github.com/rancher/local-path-provisioner) to automatically create these + +- Then, we will create an ElasticSearch resource + +- The operator will detect that resource and provision the cluster + +--- + +## Installing a Persistent Volume provisioner + +(This step can be skipped if you already have a dynamic volume provisioner.) + +- This provisioner creates Persistent Volumes backed by `hostPath` + + (local directories on our nodes) + +- It doesn't require anything special ... + +- ... But losing a node = losing the volumes on that node! + +.exercise[ + +- Install the local path storage provisioner: + ```bash + kubectl apply -f ~/container.training/k8s/local-path-storage.yaml + ``` + +] + +--- + +## Making sure we have a default StorageClass + +- The ElasticSearch operator will create StatefulSets + +- These StatefulSets will instantiate PersistentVolumeClaims + +- These PVCs need to be explicitly associated with a StorageClass + +- Or we need to tag a StorageClass to be used as the default one + +.exercise[ + +- List StorageClasses: + ```bash + kubectl get storageclasses + ``` + +] + +We should see the `local-path` StorageClass. + +--- + +## Setting a default StorageClass + +- This is done by adding an annotation to the StorageClass: + + `storageclass.kubernetes.io/is-default-class: true` + +.exercise[ + +- Tag the StorageClass so that it's the default one: + ```bash + kubectl annotate storageclass local-path \ + storageclass.kubernetes.io/is-default-class=true + ``` + +- Check the result: + ```bash + kubectl get storageclasses + ``` + +] + +Now, the StorageClass should have `(default)` next to its name. + +--- + +## Install the ElasticSearch operator + +- The operator needs: + + - a Deployment for its controller + - a ServiceAccount, ClusterRole, ClusterRoleBinding for permissions + - a Namespace + +- We have grouped all the definitions for these resources in a YAML file + +.exercise[ + +- Install the operator: + ```bash + kubectl apply -f ~/container.training/k8s/elasticsearch-operator.yaml + ``` + +] + +--- + +## Wait for the operator to be ready + +- Some operators require to create their CRDs separately + +- This operator will create its CRD itself + + (i.e. the CRD is not listed in the YAML that we applied earlier) + +.exercise[ + +- Wait until the `elasticsearchclusters` CRD shows up: + ```bash + kubectl get crds + ``` + +] + +--- + +## Create an ElasticSearch resource + +- We can now create a resource with `kind: ElasticsearchCluster` + +- The YAML for that resource will specify all the desired parameters: + + - how many nodes do we want of each type (client, master, data) + - image to use + - add-ons (kibana, cerebro, ...) + - whether to use TLS or not + - etc. + +.exercise[ + +- Create our ElasticSearch cluster: + ```bash + kubectl apply -f ~/container.training/k8s/elasticsearch-cluster.yaml + ``` + +] + +--- + +## Operator in action + +- Over the next minutes, the operator will create: + + - StatefulSets (one for master nodes, one for data nodes) + + - Deployments (for client nodes; and for add-ons like cerebro and kibana) + + - Services (for all these pods) + +.exercise[ + +- Wait for all the StatefulSets to be fully up and running: + ```bash + kubectl get statefulsets -w + ``` + +] + +--- + +## Connecting to our cluster + +- Since connecting directly to the ElasticSearch API is a bit raw, +
we'll connect to the cerebro frontend instead + +.exercise[ + +- Edit the cerebro service to change its type from ClusterIP to NodePort: + ```bash + kubectl patch svc cerebro-es -p "spec: { type: NodePort }" + ``` + +- Retrieve the NodePort that was allocated: + ```bash + kubectl get svc cerebreo-es + ``` + +- Connect to that port with a browser + +] + +--- + +## (Bonus) Setup filebeat + +- Let's send some data to our brand new ElasticSearch cluster! + +- We'll deploy a filebeat DaemonSet to collect node logs + +.exercise[ + +- Deploy filebeat: + ```bash + kubectl apply -f ~/container.training/k8s/filebeat.yaml + ``` + +] + +We should see at least one index being created in cerebro. + +--- + +## (Bonus) Access log data with kibana + +- Let's expose kibana (by making kibana-es a NodePort too) + +- Then access kibana + +- We'll need to configure kibana indexes + +--- + +## Deploying our apps with operators + +- It is very simple to deploy with `kubectl run` / `kubectl expose` + +- We can unlock more features by writing YAML and using `kubectl apply` + +- Kustomize or Helm let us deploy in multiple environments + + (and adjust/tweak parameters in each environment) + +- We can also use an operator to deploy our application + +--- + +## Pros and cons of deploying with operators + +- The app definition and configuration is persisted in the Kubernetes API + +- Multiple instances of the app can be manipulated with `kubectl get` + +- We can add labels, annotations to the app instances + +- Our controller can execute custom code for any lifecycle event + +- However, we need to write this controller + +- We need to be careful about changes + + (what happens when the resource `spec` is updated?) + +--- + +## Operators are not magic + +- Look at the ElasticSearch resource definition + + (`~/container.training/k8s/elasticsearch-cluster.yaml`) + +- What should happen if we flip the `use-tls` flag? Twice? + +- What should happen if we remove / re-add the kibana or cerebro sections? + +- What should happen if we change the number of nodes? + +- What if we want different images or parameters for the different nodes? + +*Operators can be very powerful, iff we know exactly the scenarios that they can handle.* diff --git a/slides/kube-fullday.yml b/slides/kube-fullday.yml index 110c1b60..0fc48380 100644 --- a/slides/kube-fullday.yml +++ b/slides/kube-fullday.yml @@ -65,6 +65,7 @@ chapters: # - k8s/configuration.md #- - k8s/owners-and-dependents.md # - k8s/extending-api.md +# - k8s/operators.md # - k8s/statefulsets.md # - k8s/portworx.md - - k8s/whatsnext.md diff --git a/slides/kube-selfpaced.yml b/slides/kube-selfpaced.yml index 358f5925..528d2311 100644 --- a/slides/kube-selfpaced.yml +++ b/slides/kube-selfpaced.yml @@ -65,6 +65,7 @@ chapters: - k8s/configuration.md - - k8s/owners-and-dependents.md - k8s/extending-api.md + - k8s/operators.md - k8s/statefulsets.md - k8s/portworx.md - k8s/staticpods.md diff --git a/slides/kube-twodays.yml b/slides/kube-twodays.yml index 61aaac18..f286d081 100644 --- a/slides/kube-twodays.yml +++ b/slides/kube-twodays.yml @@ -65,6 +65,7 @@ chapters: - k8s/configuration.md #- k8s/owners-and-dependents.md - k8s/extending-api.md + - k8s/operators.md - - k8s/statefulsets.md - k8s/portworx.md - k8s/staticpods.md From 69f9cee6c951a0e846651a084940117ae31caaae Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Mon, 20 May 2019 23:15:44 -0500 Subject: [PATCH 02/27] More on healthchecks! Exercises! Fun! Action! --- slides/k8s/healthchecks-more.md | 345 ++++++++++++++++++++++++++++++++ slides/kube-fullday.yml | 1 + slides/kube-selfpaced.yml | 1 + slides/kube-twodays.yml | 1 + 4 files changed, 348 insertions(+) create mode 100644 slides/k8s/healthchecks-more.md diff --git a/slides/k8s/healthchecks-more.md b/slides/k8s/healthchecks-more.md new file mode 100644 index 00000000..dc161b76 --- /dev/null +++ b/slides/k8s/healthchecks-more.md @@ -0,0 +1,345 @@ +## Questions to ask before adding healthchecks + +- Do we want liveness, readiness, both? + + (sometimes, we can use the same check, but with different failure thresholds) + +- Do we have existing HTTP endpoints that we can use? + +- Do we need to add new endpoints, or perhaps use something else? + +- Are our healthchecks likely to use resources and/or slow down the app? + +- Do they depend on additional services? + + (this can be particularly tricky, see next slide) + +--- + +## Healthchecks and dependencies + +- A good healthcheck should always indicate the health of the service itself + +- It should not be affected by the state of the service's dependencies + +- Example: a web server requiring a database connection to operate + + (make sure that the healthcheck can report "OK" even if the database is down) + +- Example: a microservice calling other microservices + +- Example: a worker process + + (these will generally require minor code changes to report health) + +--- + +## Adding healthchecks to an app + +- Let's add healthchecks to DockerCoins! + +- We will examine the questions of the previous slide + +- Then we will review each component individually to add healthchecks + +--- + +## Liveness, readiness, or both? + +- To answer that question, we need to see the app run for a while + +- Do we get temporary, recoverable glitches? + + → then use readiness + +- Or do we get hard lock-ups requiring a restart? + + → then use liveness + +- In the case of DockerCoins, we don't know yet! + +- Let's pick liveness + +--- + +## Do we have HTTP endpoints that we can use? + +- Each of the 3 web services (hasher, rng, webui) has a trivial route on `/` + +- These routes: + + - don't seem to perform anything complex or expensive + + - don't seem to call other services + +- Perfect! + + (See next slides for individual details) + +--- + +- [hasher.rb](https://github.com/jpetazzo/container.training/blob/master/dockercoins/hasher/hasher.rb) + ```ruby + get '/' do + "HASHER running on #{Socket.gethostname}\n" + end + ``` + +- [rng.py](https://github.com/jpetazzo/container.training/blob/master/dockercoins/rng/rng.py) + ```python + @app.route("/") + def index(): + return "RNG running on {}\n".format(hostname) + ``` + +- [webui.js](https://github.com/jpetazzo/container.training/blob/master/dockercoins/webui/webui.js) + ```javascript + app.get('/', function (req, res) { + res.redirect('/index.html'); + }); + ``` + +--- + +## Running DockerCoins + +- We will run DockerCoins in a new, separate namespace + +- We will use a set of YAML manifests and pre-built images + +- We will add our new liveness probe to the YAML of the `rng` DaemonSet + +- Then, we will deploy the application + +--- + +## Creating a new namespace + +- This will make sure that we don't collide / conflict with previous exercises + +.exercise[ + +- Create the yellow namespace: + ```bash + kubectl create namespace yellow + ``` + +- Switch to that namespace: + ```bash + kns yellow + ``` + +] + +--- + +## Retrieving DockerCoins manifests + +- All the manifests that we need are on a convenient repository: + + https://github.com/jpetazzo/kubercoins + +.exercise[ + +- Clone that repository: + ```bash + cd ~ + git clone https://github.com/jpetazzo/kubercoins + ``` + +- Change directory to the repository: + ```bash + cd kubercoins + ``` + +] + +--- + +## A simple HTTP liveness probe + +This is what our liveness probe should look like: + +```yaml +containers: +- name: ... +image: ... +livenessProbe: + httpGet: + path: / + port: 80 + initialDelaySeconds: 30 + periodSeconds: 5 +``` + +This will give 30 seconds to the service to start. (Way more than necessary!) +
+It will run the probe every 5 seconds. +
+It will use the default timeout (1 second). +
+It will use the default failure threshold (3 failed attempts = dead). +
+It will use the default success threshold (1 successful attempt = alive). + +--- + +## Adding the liveness probe + +- Let's add the liveness probe, then deploy DockerCoins + +.exercise[ + +- Edit `rng-daemonset.yaml` and add the liveness probe + ```bash + vim rng-daemonset.yaml + ``` + +- Load the YAML for all the resources of DockerCoins: + ```bash + kubectl apply -f . + ``` + +] + +--- + +## Testing the liveness probe + +- The rng service needs 100ms to process a request + + (because it is single-threaded and sleeps 0.1s in each request) + +- The probe timeout is set to 1 second + +- If we send more than 10 requests per second per backend, it will break + +- Let's generate traffic and see what happens! + +.exercise[ + +- Get the ClusterIP address of the rng service: + ```bash + kubectl get svc rng + ``` + +] + +--- + +## Monitoring the rng service + +- Each command below will show us what's happening on a different level + +.exercise[ + +- In one window, monitor cluster events: + ```bash + kubectl get events -w + ``` + +- In another window, monitor the response time of rng: + ```bash + httping `` + ``` + +- In another window, monitor pods status: + ```bash + kubectl get pods -w + ``` + +] + +--- + +## Generating traffic + +- Let's use `ab` to send concurrent requests to rng + +.exercise[ + +- In yet another window, generate traffic: + ```bash + ab -c 10 -n 1000 http://``/1 + ``` + +- Experiment with higher values of `-c` and see what happens + +] + +- The `-c` parameter indicates the number of concurrent requests + +- The final `/1` is important to generate actual traffic + + (otherwise we would use the ping endpoint, which doesn't sleep 0.1s per request) + +--- + +## Discussion + +- Above a given threshold, the liveness probe starts failing + + (about 10 concurrent requests per backend should be plenty enough) + +- When the liveness probe fails 3 times in a row, the container is restarted + +- During the restart, there is *less* capacity available + +- ... Meaning that the other backends are likely to timeout as well + +- ... Eventually causing all backends to be restarted + +- ... And each fresh backend gets restarted, too + +- This goes on until the load goes down, or we add capacity + +*This wouldn't be a good healthcheck in a real application!* + +--- + +## Better healthchecks + +- We need to make sure that the healthcheck doesn't trip when + performance degrades due to external pressure + +- Using a readiness check would have lesser effects + + (but it still would be an imperfect solution) + +- A possible combination: + + - readiness check with a short timeout / low failure threshold + + - liveness check with a longer timeout / higher failure treshold + +--- + +## Healthchecks for redis + +- A liveness probe is enough + + (it's not useful to remove a backend from rotation when it's the only one) + +- We could use an exec probe running `redis-cli ping` + +--- + +## Healthchecks for worker + +- Readiness isn't useful + + (because worker isn't a backend for a service) + +- Liveness may help us to restart a broken worker, but how can we check it? + +- Embedding an HTTP server is an option + + (but it has a high potential for unwanted side-effects and false positives) + +- Using a "lease" file can be relatively easy: + + - touch a file during each iteration of the main loop + + - check the timestamp of that file from an exec probe + +- Writing logs (and checking them from the probe) also works diff --git a/slides/kube-fullday.yml b/slides/kube-fullday.yml index 110c1b60..cd988f74 100644 --- a/slides/kube-fullday.yml +++ b/slides/kube-fullday.yml @@ -48,6 +48,7 @@ chapters: - k8s/daemonset.md - - k8s/rollout.md # - k8s/healthchecks.md +# - k8s/healthchecks-more.md - k8s/logs-cli.md - k8s/logs-centralized.md #- - k8s/helm.md diff --git a/slides/kube-selfpaced.yml b/slides/kube-selfpaced.yml index 358f5925..fcbf8f06 100644 --- a/slides/kube-selfpaced.yml +++ b/slides/kube-selfpaced.yml @@ -48,6 +48,7 @@ chapters: - k8s/daemonset.md - - k8s/rollout.md - k8s/healthchecks.md + - k8s/healthchecks-more.md - k8s/logs-cli.md - k8s/logs-centralized.md - - k8s/helm.md diff --git a/slides/kube-twodays.yml b/slides/kube-twodays.yml index 61aaac18..c22531c4 100644 --- a/slides/kube-twodays.yml +++ b/slides/kube-twodays.yml @@ -48,6 +48,7 @@ chapters: - - k8s/daemonset.md - k8s/rollout.md - k8s/healthchecks.md + - k8s/healthchecks-more.md - k8s/logs-cli.md - k8s/logs-centralized.md - - k8s/helm.md From b0f01e018c20765244d93691453deb8ab415aed6 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Fri, 24 May 2019 18:44:41 -0500 Subject: [PATCH 03/27] Clarify healthchecks and dependencies --- slides/k8s/healthchecks-more.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/slides/k8s/healthchecks-more.md b/slides/k8s/healthchecks-more.md index dc161b76..709e980d 100644 --- a/slides/k8s/healthchecks-more.md +++ b/slides/k8s/healthchecks-more.md @@ -24,7 +24,9 @@ - Example: a web server requiring a database connection to operate - (make sure that the healthcheck can report "OK" even if the database is down) + (make sure that the healthcheck can report "OK" even if the database is down; +
+ because it won't help us to restart the web server if the issue is with the DB!) - Example: a microservice calling other microservices From c565dad43c5f184cf201c36d199ff2e35a35f6de Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Fri, 24 May 2019 19:33:23 -0500 Subject: [PATCH 04/27] Fix typos and add precisions --- slides/k8s/operators.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/slides/k8s/operators.md b/slides/k8s/operators.md index 66253edb..b3dbf6b4 100644 --- a/slides/k8s/operators.md +++ b/slides/k8s/operators.md @@ -22,10 +22,14 @@ - A given operator can define one or multiple CRDs -- The controller code (control loop) can run within the cluster or elsewhere +- The controller code (control loop) typically runs within the cluster (running as a Deployment with 1 replica is a common scenario) +- But it could also run elsewhere + + (nothing mandates that the code run on the cluster, as long as it has API access) + --- ## Why use operators? @@ -76,7 +80,7 @@ (Example: [Istio operator](https://operatorhub.io/operator/beta/istio-operator.0.1.6)) -- Deploying and managing our applications lifecycles +- Deploying and managing our applications' lifecycles (more on that later) @@ -90,7 +94,7 @@ - The operator also runs a *controller* that will watch its resources -- Each time we create/updated/delete a resource, the controller is notified +- Each time we create/update/delete a resource, the controller is notified (we could write our own cheap controller with `kubectl get --watch`) From 39027675d52c3c19ae40f484e0ac19154897233e Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Sat, 25 May 2019 12:53:15 -0500 Subject: [PATCH 05/27] Add a whole chapter about operator design --- slides/k8s/operators-design.md | 257 +++++++++++++++++++++++++++++++++ slides/k8s/operators.md | 21 ++- slides/kube-fullday.yml | 1 + slides/kube-selfpaced.yml | 1 + slides/kube-twodays.yml | 1 + 5 files changed, 280 insertions(+), 1 deletion(-) create mode 100644 slides/k8s/operators-design.md diff --git a/slides/k8s/operators-design.md b/slides/k8s/operators-design.md new file mode 100644 index 00000000..16369542 --- /dev/null +++ b/slides/k8s/operators-design.md @@ -0,0 +1,257 @@ +## What does it take to write an operator? + +- Writing a quick-and-dirty operator, or a POC/MVP, is easy + +- Writing a robust operator is hard + +- We will describe the general idea + +- We will identify some of the associated challenges + +- We will list a few tools that can help us + +--- + +## High-level design + +- What are we solving? + + (e.g.: geographic databases backed by PostGIS with Redis caches) + +- What are our use-cases, stories? + + (e.g.: adding/resizing caches and read replicas; load balancing queries) + +- What kind of outage do we want to address? + + (e.g.: loss of individual node, pod, volume) + +- What are our *non-features*, the things we don't want to address? + + (e.g.: loss of datacenter/zone; differentiating between read and write queries; +
+ cache invalidation; upgrading to newer major versions of Redis, PostGIS, PostgreSQL) + +--- + +## Low-level design + +- What Custom Resource Definitions do we need? + + (one, many?) + +- How will we store configuration information? + + (part of the CRD spec fields, annotations, other?) + +- Do we need to store state? If so, where? + + - the Kubernetes API can store state that is small and doesn't change much +
+ (e.g.: leader information, configuration, credentials) + + - things that are big and/or change a lot should go elsewhere +
+ (e.g.: metrics, bigger configuration file like GeoIP) + +--- + +## General idea + +- Our operator will watch its CRDs *and associated resources* + +- Drawing state diagrams and finite state automatons helps a lot + +- It's OK if some transitions lead to a big catch-all "human intervention" + +- Over time, we will learn about new failure modes and add to these diagrams + +- It's OK to start with CRD creation / deletion and prevent any modification + + (that's the easy POC/MVP we were talking about) + +- *Presentation* and *validation* will help our users + + (more on that later) + +--- + +## Challenges + +- Reacting to infrastructure disruption can seem hard at first + +- Kubernetes gives us a lot of primitives to help: + + - Pods and Persistent Volumes will *eventually* recover + + - Stateful Sets give us easy ways to "add N copies" of a thing + +- The real challenges come with configuration changes + + (i.e., what to do when our users update our CRDs) + +- Keep in mind that [some] of the [largest] cloud [outages] haven't been caused by [natural catastrophes], or even code bugs, but by configuration changes + +[some]: https://www.datacenterdynamics.com/news/gcp-outage-mainone-leaked-google-cloudflare-ip-addresses-china-telecom/ +[largest]: https://aws.amazon.com/message/41926/ +[outages]: https://aws.amazon.com/message/65648/ +[natural catastrophes]: https://www.datacenterknowledge.com/amazon/aws-says-it-s-never-seen-whole-data-center-go-down + +--- + +## Configuration changes + +- It is helpful to analyze and understand how Kubernetes controllers work: + + - watch resource for modifications + + - compare desired state (CRD) and current state + + - issue actions to converge state + +- Configuration changes will probably require *another* state diagram or FSA + +- Again, it's OK to have transitions labeled as "unsupported" + + (i.e. reject some modifications because we can't execute them) + +--- + +## Tools + +- CoreOS / RedHat Operator Framework + + [GitHub](https://github.com/operator-framework) + | + [Blog](https://developers.redhat.com/blog/2018/12/18/introduction-to-the-kubernetes-operator-framework/) + | + [Intro talk](https://www.youtube.com/watch?v=8k_ayO1VRXE) + | + [Deep dive talk](https://www.youtube.com/watch?v=fu7ecA2rXmc) + +- Zalando Kubernetes Operator Pythonic Framework (KOPF) + + [GitHub](https://github.com/zalando-incubator/kopf) + | + [Docs](https://kopf.readthedocs.io/) + | + [Step-by-step tutorial](https://kopf.readthedocs.io/en/stable/walkthrough/problem/) + +- Mesosphere Kubernetes Universal Declarative Operator (KUDO) + + [GitHub](https://github.com/kudobuilder/kudo) + | + [Blog](https://mesosphere.com/blog/announcing-maestro-a-declarative-no-code-approach-to-kubernetes-day-2-operators/) + | + [Docs](https://kudo.dev/) + | + [Zookeeper example](https://github.com/kudobuilder/frameworks/tree/master/repo/stable/zookeeper) + +--- + +## Validation + +- By default, a CRD is "free form" + + (we can put pretty much anything we want in it) + +- When creating a CRD, we can provide an OpenAPI v3 schema + ([Example](https://github.com/amaizfinance/redis-operator/blob/master/deploy/crds/k8s_v1alpha1_redis_crd.yaml#L34)) + +- The API server will then validate resources created/edited with this schema + +- If we need a stronger validation, we can use a Validating Admission Webhook: + + - run an [admission webhook server](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#write-an-admission-webhook-server) to receive validation requests + + - register the webhook by creating a [ValidatingWebhookConfiguration](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#configure-admission-webhooks-on-the-fly) + + - each time the API server receives a request matching the configuration, +
the request is sent to our server for validation + +--- + +## Presentation + +- By default, `kubectl get mycustomresource` won't display much information + + (just the name and age of each resource) + +- When creating a CRD, we can specify additional columns to print + ([Example](https://github.com/amaizfinance/redis-operator/blob/master/deploy/crds/k8s_v1alpha1_redis_crd.yaml#L6), + [Docs](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#additional-printer-columns)) + +- By default, `kubectl describe mycustomresource` will also be generic + +- `kubectl describe` can show events related to our custom resources + + (for that, we need to create Event resources, and fill the `involvedObject` field) + +- For scalable resources, we can define a `scale` sub-resource + +- This will enable the use of `kubectl scale` and other scaling-related operations + +--- + +## Versioning + +- As our operator evolves over time, we may have to change the CRD + + (add, remove, change fields) + +- Like every other resource in Kubernetes, [custom resources are versioned](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definition-versioning/ +) + +- When creating a CRD, we need to specify a *list* of versions + +- Versions can be marked as `stored` and/or `served` + +--- + +## Stored version + +- Exactly one version has to be marked as the `stored` version + +- As the name implies, it is the one that will be stored in etcd + +- Resources in storage are never converted automatically + + (we need to read and re-write them ourselves) + +- Yes, this means that we can have different versions in etcd at any time + +- Our code needs to handle all the versions that still exist in storage + +--- + +## Served versions + +- By default, the Kubernetes API will serve resources "as-is" + + (using their stored version) + +- It will assume that all versions are compatible storage-wise + + (i.e. that the spec and fields are compatible between versions) + +- We can provide [conversion webhooks](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definition-versioning/#webhook-conversion) to "translate" requests + + (the alternative is to upgrade all stored resources and stop serving old versions) + +--- + +## Beyond CRDs + +- CRDs cannot use custom storage (e.g. for time series data) + +- CRDs cannot support arbitrary subresources (like logs or exec for Pods) + +- CRDs cannot support protobuf (for faster, more efficient communication) + +- If we need these things, we can use the [aggregation layer](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/apiserver-aggregation/) instead + +- The aggregation layer proxies all requests below a specific path to another server + + (this is used e.g. by the metrics server) + +- [This documentation page](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#choosing-a-method-for-adding-custom-resources) compares the features of CRDs and API aggregation diff --git a/slides/k8s/operators.md b/slides/k8s/operators.md index b3dbf6b4..b92b0b52 100644 --- a/slides/k8s/operators.md +++ b/slides/k8s/operators.md @@ -14,6 +14,25 @@ ## What are operators? +*An operator represents **human operational knowledge in software,** +
+to reliably manage an application. +— [CoreOS](https://coreos.com/blog/introducing-operators.html)* + +Examples: + +- Deploying and configuring replication with MySQL, PostgreSQL ... + +- Setting up Elasticsearch, Kafka, RabbitMQ, Zookeeper ... + +- Reacting to failures when intervention is needed + +- Scaling up and down these systems + +--- + +## What are they made from? + - Operators combine two things: - Custom Resource Definitions @@ -66,7 +85,7 @@ - Systems that depend on other non-trivial systems - Examples: Flink, Kafka, ... + Examples: Flink or Kafka, which both depend on Zookeeper --- diff --git a/slides/kube-fullday.yml b/slides/kube-fullday.yml index 0fc48380..451589ac 100644 --- a/slides/kube-fullday.yml +++ b/slides/kube-fullday.yml @@ -66,6 +66,7 @@ chapters: #- - k8s/owners-and-dependents.md # - k8s/extending-api.md # - k8s/operators.md +# - k8s/operators-design.md # - k8s/statefulsets.md # - k8s/portworx.md - - k8s/whatsnext.md diff --git a/slides/kube-selfpaced.yml b/slides/kube-selfpaced.yml index 528d2311..8459bc9e 100644 --- a/slides/kube-selfpaced.yml +++ b/slides/kube-selfpaced.yml @@ -66,6 +66,7 @@ chapters: - - k8s/owners-and-dependents.md - k8s/extending-api.md - k8s/operators.md + - k8s/operators-design.md - k8s/statefulsets.md - k8s/portworx.md - k8s/staticpods.md diff --git a/slides/kube-twodays.yml b/slides/kube-twodays.yml index f286d081..51dfc5fb 100644 --- a/slides/kube-twodays.yml +++ b/slides/kube-twodays.yml @@ -66,6 +66,7 @@ chapters: #- k8s/owners-and-dependents.md - k8s/extending-api.md - k8s/operators.md + - k8s/operators-design.md - - k8s/statefulsets.md - k8s/portworx.md - k8s/staticpods.md From 4c553360794ae66158e2c53d9079c2c4e28b0117 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Sat, 25 May 2019 21:43:07 -0500 Subject: [PATCH 06/27] automatons -> automata --- slides/k8s/operators-design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slides/k8s/operators-design.md b/slides/k8s/operators-design.md index 16369542..5ad144f6 100644 --- a/slides/k8s/operators-design.md +++ b/slides/k8s/operators-design.md @@ -60,7 +60,7 @@ - Our operator will watch its CRDs *and associated resources* -- Drawing state diagrams and finite state automatons helps a lot +- Drawing state diagrams and finite state automata helps a lot - It's OK if some transitions lead to a big catch-all "human intervention" From 3f70ee2c2a5a505df00197c2944a83ba65701be2 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Sun, 26 May 2019 22:17:20 -0500 Subject: [PATCH 07/27] Add note about operator scaling --- slides/k8s/operators-design.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/slides/k8s/operators-design.md b/slides/k8s/operators-design.md index 5ad144f6..0e5e6704 100644 --- a/slides/k8s/operators-design.md +++ b/slides/k8s/operators-design.md @@ -193,6 +193,26 @@ --- +## About scaling + +- It is possible to use the HPA (Horizontal Pod Autoscaler) with CRDs + +- But it is not always desirable + +- The HPA works very well for homogenous, stateless workloads + +- For other workloads, your mileage may vary + +- Some systems can scale across multiple dimensions + + (for instance: increase number of replicas, or number of shards?) + +- If autoscaling is desired, the operator will have to take complex decisions + + (example: Zalando's Elasticsearch Operator ([Video](https://www.youtube.com/watch?v=lprE0J0kAq0))) + +--- + ## Versioning - As our operator evolves over time, we may have to change the CRD From 3a87183a66685049c334be3b9ee1468c3628c175 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Sun, 26 May 2019 22:39:11 -0500 Subject: [PATCH 08/27] Add bottom-us vs top-down approaches Bottom-us is inspired by the Zalando ES operator --- slides/k8s/operators-design.md | 42 ++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/slides/k8s/operators-design.md b/slides/k8s/operators-design.md index 0e5e6704..62786d4e 100644 --- a/slides/k8s/operators-design.md +++ b/slides/k8s/operators-design.md @@ -12,6 +12,32 @@ --- +## Top-down vs. bottom-up + +- Both approaches are possible + +- Let's see what they entail, and their respective pros and cons + +--- + +## Top-down approach + +- Start with high-level design (see next slide) + +- Pros: + + - can yield cleaner design that will be more robust + +- Cons: + + - must be able to anticipate all the events that might happen + + - design will be better only to the extend of what we anticipated + + - hard to anticipate if we don't have production experience + +--- + ## High-level design - What are we solving? @@ -56,6 +82,22 @@ --- +## Bottom-up approach + +- Start with existing Kubernetes resources (Deployment, Stateful Set...) + +- Run the system in production + +- Add scripts, automation, to facilitate day-to-day operations + +- Turn the scripts into an operator + +- Pros: simpler to get started; reflects actual use-cases + +- Cons: can result in convoluted designs requiring extensive refactor + +--- + ## General idea - Our operator will watch its CRDs *and associated resources* From baf428ebdba8e00c7539f69157aa72a803d7b323 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Sun, 26 May 2019 22:46:24 -0500 Subject: [PATCH 09/27] Add note about operator reliability --- slides/k8s/operators-design.md | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/slides/k8s/operators-design.md b/slides/k8s/operators-design.md index 62786d4e..d259303b 100644 --- a/slides/k8s/operators-design.md +++ b/slides/k8s/operators-design.md @@ -4,7 +4,7 @@ - Writing a robust operator is hard -- We will describe the general idea +- We will describe the general idea - We will identify some of the associated challenges @@ -164,7 +164,7 @@ - CoreOS / RedHat Operator Framework [GitHub](https://github.com/operator-framework) - | + | [Blog](https://developers.redhat.com/blog/2018/12/18/introduction-to-the-kubernetes-operator-framework/) | [Intro talk](https://www.youtube.com/watch?v=8k_ayO1VRXE) @@ -302,6 +302,26 @@ --- +## Operator reliability + +- Remember that the operator itself must be resilient + + (e.g.: the node running it can fail) + +- Our operator must be able to restart and recover gracefully + +- Do not store state locally + + (unless we can reconstruct that state when we restart) + +- As indicated earlier, we can use the Kubernetes API to store data: + + - in the custom resources themselves + + - in other resources' annotations + +--- + ## Beyond CRDs - CRDs cannot use custom storage (e.g. for time series data) From 68f35bd2ed5fe11f13e2edcd8e8a4a628bded999 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Mon, 27 May 2019 19:11:04 -0500 Subject: [PATCH 10/27] Add info about zombies and exec probes (courtesy of @lbernail) --- slides/k8s/healthchecks-more.md | 57 +++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/slides/k8s/healthchecks-more.md b/slides/k8s/healthchecks-more.md index 709e980d..d75aa112 100644 --- a/slides/k8s/healthchecks-more.md +++ b/slides/k8s/healthchecks-more.md @@ -324,6 +324,63 @@ It will use the default success threshold (1 successful attempt = alive). - We could use an exec probe running `redis-cli ping` +--- + +class: extra-details + +## Exec probes and zombies + +- When using exec probes, we should make sure that we have a *zombie reaper* + + 🤔🧐🧟 Wait, what? + +- When a process terminates, its parent must call `wait()`/`waitpid()` + + (this is how the parent process retrieves the child's exit status) + +- In the meantime, the process is in *zombie* state + + (the process state will show as `Z` in `ps`, `top` ...) + +- When a process is killed, its children are *orphaned* and attached to PID 1 + +- PID 1 has the responsibility if *reaping* these processes when they terminate + +- OK, but how does that affect us? + +--- + +class: extra-details + +## PID 1 in containers + +- On ordinary systems, PID 1 (`/sbin/init`) has logic to reap processes + +- In containers, PID 1 is typically our application process + + (e.g. Apache, the JVM, NGINX, Redis ...) + +- These *do not* take care of reaping orphans + +- If we use exec probes, we need to add a process reaper + +- We can add [tini](https://github.com/krallin/tini) to our images + +- Or [share the PID namespace between containers of a pod](https://kubernetes.io/docs/tasks/configure-pod-container/share-process-namespace/) + + (and have gcr.io/pause take care of the reaping) + +--- + +## PID 1 + + +- W + +- On normal systems + + + --- ## Healthchecks for worker From 879e7f2ec9d84365543a3f645c88802e866dfbe6 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Fri, 31 May 2019 21:06:17 -0500 Subject: [PATCH 11/27] Improve phrasing following Bridget's feedback --- slides/k8s/operators-design.md | 19 ++++++++++++++++++- slides/k8s/operators.md | 2 +- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/slides/k8s/operators-design.md b/slides/k8s/operators-design.md index d259303b..9a02f2b8 100644 --- a/slides/k8s/operators-design.md +++ b/slides/k8s/operators-design.md @@ -72,7 +72,7 @@ - Do we need to store state? If so, where? - - the Kubernetes API can store state that is small and doesn't change much + - state that is small and doesn't change much can be stored via the Kubernetes API
(e.g.: leader information, configuration, credentials) @@ -82,6 +82,23 @@ --- +class: extra-details + +## What can we store via the Kubernetes API? + +- The API server stores most Kubernetes resources into etcd + +- Etcd is designed for reliability, not for performance + +- If our storage needs exceed what etcd can offer, we need to use something else: + + - either directly + + - or by extending the API server +
(for instance by using the agregation layer, like [metrics server](https://github.com/kubernetes-incubator/metrics-server) does) + +--- + ## Bottom-up approach - Start with existing Kubernetes resources (Deployment, Stateful Set...) diff --git a/slides/k8s/operators.md b/slides/k8s/operators.md index b92b0b52..fdf6bbf6 100644 --- a/slides/k8s/operators.md +++ b/slides/k8s/operators.md @@ -83,7 +83,7 @@ Examples: Examples: ElasticSearch, MongoDB ... -- Systems that depend on other non-trivial systems +- Systems with complex dependencies (that are themselves managed with operators) Examples: Flink or Kafka, which both depend on Zookeeper From 2c0b4b15bacfdae26ac47e685c06d65ff63d797d Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Sat, 1 Jun 2019 19:31:27 -0500 Subject: [PATCH 12/27] Remove bogus slide --- slides/k8s/healthchecks-more.md | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/slides/k8s/healthchecks-more.md b/slides/k8s/healthchecks-more.md index d75aa112..328fe254 100644 --- a/slides/k8s/healthchecks-more.md +++ b/slides/k8s/healthchecks-more.md @@ -370,17 +370,6 @@ class: extra-details (and have gcr.io/pause take care of the reaping) ---- - -## PID 1 - - -- W - -- On normal systems - - - --- ## Healthchecks for worker From 5e520dfbe5aa7b061c75c0c5fdbdd393ab1857a8 Mon Sep 17 00:00:00 2001 From: AJ Bowen Date: Mon, 3 Jun 2019 20:42:04 -0700 Subject: [PATCH 13/27] wording tweaks --- slides/k8s/healthchecks-more.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/slides/k8s/healthchecks-more.md b/slides/k8s/healthchecks-more.md index 328fe254..887ce5b3 100644 --- a/slides/k8s/healthchecks-more.md +++ b/slides/k8s/healthchecks-more.md @@ -304,9 +304,9 @@ It will use the default success threshold (1 successful attempt = alive). - We need to make sure that the healthcheck doesn't trip when performance degrades due to external pressure -- Using a readiness check would have lesser effects +- Using a readiness check would have fewer effects - (but it still would be an imperfect solution) + (but it would still be an imperfect solution) - A possible combination: @@ -344,7 +344,7 @@ class: extra-details - When a process is killed, its children are *orphaned* and attached to PID 1 -- PID 1 has the responsibility if *reaping* these processes when they terminate +- PID 1 has the responsibility of *reaping* these processes when they terminate - OK, but how does that affect us? @@ -378,11 +378,11 @@ class: extra-details (because worker isn't a backend for a service) -- Liveness may help us to restart a broken worker, but how can we check it? +- Liveness may help us restart a broken worker, but how can we check it? - Embedding an HTTP server is an option - (but it has a high potential for unwanted side-effects and false positives) + (but it has a high potential for unwanted side effects and false positives) - Using a "lease" file can be relatively easy: From c3a796faef184e5eb2a4651e44e3cf3393b475be Mon Sep 17 00:00:00 2001 From: AJ Bowen Date: Tue, 11 Jun 2019 16:28:50 -0700 Subject: [PATCH 14/27] observations from Velocity workshop --- slides/k8s/cloud-controller-manager.md | 12 ++++++------ slides/k8s/cluster-backup.md | 12 ++++++------ slides/k8s/cluster-upgrade.md | 4 ++-- slides/k8s/cni.md | 6 +++--- slides/k8s/csr-api.md | 6 +++--- slides/k8s/multinode.md | 2 +- slides/k8s/podsecuritypolicy.md | 4 ++-- slides/k8s/staticpods.md | 14 +++++++------- 8 files changed, 30 insertions(+), 30 deletions(-) diff --git a/slides/k8s/cloud-controller-manager.md b/slides/k8s/cloud-controller-manager.md index d0975278..cbf25cf8 100644 --- a/slides/k8s/cloud-controller-manager.md +++ b/slides/k8s/cloud-controller-manager.md @@ -20,15 +20,15 @@ - Configuring routing tables in the cloud network (specific to GCE) -- Updating node labels to indicate region, zone, instance type ... +- Updating node labels to indicate region, zone, instance type... - Obtain node name, internal and external addresses from cloud metadata service - Deleting nodes from Kubernetes when they're deleted in the cloud -- Managing *some* volumes (e.g. ELBs, AzureDisks ...) +- Managing *some* volumes (e.g. ELBs, AzureDisks...) - (Eventually, volumes will be managed by the CSI) + (Eventually, volumes will be managed by the CSI) (the what?) --- @@ -83,7 +83,7 @@ The list includes the following providers: ## Audience questions -- What kind of clouds are you using / planning to use? +- What kind of clouds are you using/planning to use? - What kind of details would you like to see in this section? @@ -105,7 +105,7 @@ The list includes the following providers: - When using managed clusters, this is done automatically -- There is very little documentation to write the configuration file +- There is very little documentation on writing the configuration file (except for OpenStack) @@ -123,7 +123,7 @@ The list includes the following providers: - To get these addresses, the node needs to communicate with the control plane -- ... Which means joining the cluster +- ...Which means joining the cluster (The problem didn't occur when cloud-specific code was running in kubelet: kubelet could obtain the required information directly from the cloud provider's metadata service.) diff --git a/slides/k8s/cluster-backup.md b/slides/k8s/cluster-backup.md index eb34a58c..c42a153d 100644 --- a/slides/k8s/cluster-backup.md +++ b/slides/k8s/cluster-backup.md @@ -6,7 +6,7 @@ - error recovery (human or process has altered or corrupted data) - - cloning environments (for testing, validation ...) + - cloning environments (for testing, validation...) - Let's see the strategies and tools available with Kubernetes! @@ -18,13 +18,13 @@ (it gives us replication primitives) -- Kubernetes helps us to clone / replicate environments +- Kubernetes helps us to clone/replicate environments (all resources can be described with manifests) - Kubernetes *does not* help us with error recovery -- We still need to backup / snapshot our data: +- We still need to back up/snapshot our data: - with database backups (mysqldump, pgdump, etc.) @@ -58,7 +58,7 @@ - If our deployment system isn't fully automated, it should at least be documented -- Litmus test: how long does it take to deploy a cluster ... +- Litmus test: how long does it take to deploy a cluster... - for a senior engineer? @@ -66,7 +66,7 @@ - Does it require external intervention? - (e.g. provisioning servers, signing TLS certs ...) + (e.g. provisioning servers, signing TLS certs...) --- @@ -108,7 +108,7 @@ - For real applications: add resources (as YAML files) -- For applications deployed multiple times: Helm, Kustomize ... +- For applications deployed multiple times: Helm, Kustomize... (staging and production count as "multiple times") diff --git a/slides/k8s/cluster-upgrade.md b/slides/k8s/cluster-upgrade.md index 48197842..0b477ce9 100644 --- a/slides/k8s/cluster-upgrade.md +++ b/slides/k8s/cluster-upgrade.md @@ -287,8 +287,8 @@ - Download the configuration on each node, and upgrade kubelet: ```bash for N in 1 2 3; do - ssh node$N sudo kubeadm upgrade node config --kubelet-version v1.14.2 - ssh node $N sudo apt install kubelet=1.14.2-00 + ssh test$N sudo kubeadm upgrade node config --kubelet-version v1.14.2 + ssh test$N sudo apt install kubelet=1.14.2-00 done ``` ] diff --git a/slides/k8s/cni.md b/slides/k8s/cni.md index 3970f119..cd206ab1 100644 --- a/slides/k8s/cni.md +++ b/slides/k8s/cni.md @@ -276,7 +276,7 @@ class: extra-details - The address of the API server will be `http://A.B.C.D:8080` - (where `A.B.C.D` is the address of `kuberouter1`, running the control plane) + (where `A.B.C.D` is the public address of `kuberouter1`, running the control plane) .exercise[ @@ -453,7 +453,7 @@ We should see the local pod CIDR connected to `kube-bridge`, and the other nodes - Or try to exec into one of the kube-router pods: ```bash - kubectl -n kube-system exec kuber-router-xxxxx bash + kubectl -n kube-system exec kube-router-xxxxx bash ``` ] @@ -575,7 +575,7 @@ done ## Starting the route reflector -- Only do this if you are doing this on your own +- Only do this slide if you are doing this on your own - There is a Compose file in the `compose/frr-route-reflector` directory diff --git a/slides/k8s/csr-api.md b/slides/k8s/csr-api.md index 2673c4a1..35e2fa02 100644 --- a/slides/k8s/csr-api.md +++ b/slides/k8s/csr-api.md @@ -231,7 +231,7 @@ For a user named `jean.doe`, we will have: - Let's use OpenSSL; it's not the best one, but it's installed everywhere (many people prefer cfssl, easyrsa, or other tools; that's fine too!) - + .exercise[ - Generate the key and certificate signing request: @@ -244,7 +244,7 @@ For a user named `jean.doe`, we will have: The command above generates: -- a 2048-bit RSA key, without DES encryption, stored in key.pem +- a 2048-bit RSA key, without encryption, stored in key.pem - a CSR for the name `jean.doe` in group `devs` --- @@ -319,7 +319,7 @@ The command above generates: - Switch back to `cluster-admin`: ```bash - kctx - + kctx - # mraa? FIXME? ``` - Inspect the CSR: diff --git a/slides/k8s/multinode.md b/slides/k8s/multinode.md index 0efb7dc5..9a4b1313 100644 --- a/slides/k8s/multinode.md +++ b/slides/k8s/multinode.md @@ -96,7 +96,7 @@ class: extra-details - We need to generate a `kubeconfig` file for kubelet -- This time, we need to put the IP address of `kubenet1` +- This time, we need to put the public IP address of `kubenet1` (instead of `localhost` or `127.0.0.1`) diff --git a/slides/k8s/podsecuritypolicy.md b/slides/k8s/podsecuritypolicy.md index fd333919..97c721d1 100644 --- a/slides/k8s/podsecuritypolicy.md +++ b/slides/k8s/podsecuritypolicy.md @@ -212,7 +212,7 @@ - Have a look at the static pods: ```bash - ls -l /etc/kubernetes/manifest + ls -l /etc/kubernetes/manifests ``` - Edit the one corresponding to the API server: @@ -236,7 +236,7 @@ - Add `PodSecurityPolicy` - (It should read `--enable-admission-plugins=NodeRestriction,PodSecurityPolicy`) + It should read: `--enable-admission-plugins=NodeRestriction,PodSecurityPolicy` - Save, quit diff --git a/slides/k8s/staticpods.md b/slides/k8s/staticpods.md index e95b0840..31ad23ea 100644 --- a/slides/k8s/staticpods.md +++ b/slides/k8s/staticpods.md @@ -18,7 +18,7 @@ ## A possible approach -- Since each component of the control plane can be replicated ... +- Since each component of the control plane can be replicated... - We could set up the control plane outside of the cluster @@ -39,9 +39,9 @@ - Worst case scenario, we might need to: - set up a new control plane (outside of the cluster) - + - restore a backup from the old control plane - + - move the new control plane to the cluster (again) - This doesn't sound like a great experience @@ -57,7 +57,7 @@ - The kubelet can also get a list of *static pods* from: - a directory containing one (or multiple) *manifests*, and/or - + - a URL (serving a *manifest*) - These "manifests" are basically YAML definitions @@ -100,11 +100,11 @@ ## Static pods vs normal pods -- The API only gives us a read-only access to static pods +- The API only gives us read-only access to static pods -- We can `kubectl delete` a static pod ... +- We can `kubectl delete` a static pod... - ... But the kubelet will re-mirror it immediately + ...But the kubelet will re-mirror it immediately - Static pods can be selected just like other pods From c959a4c4a124176583c7d8de2bbaa2c2da078b93 Mon Sep 17 00:00:00 2001 From: AJ Bowen Date: Tue, 11 Jun 2019 17:02:26 -0700 Subject: [PATCH 15/27] a few more --- slides/k8s/cloud-controller-manager.md | 2 +- slides/k8s/lastwords-admin.md | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/slides/k8s/cloud-controller-manager.md b/slides/k8s/cloud-controller-manager.md index cbf25cf8..b1c1f7cc 100644 --- a/slides/k8s/cloud-controller-manager.md +++ b/slides/k8s/cloud-controller-manager.md @@ -28,7 +28,7 @@ - Managing *some* volumes (e.g. ELBs, AzureDisks...) - (Eventually, volumes will be managed by the CSI) (the what?) + (Eventually, volumes will be managed by the CSI) (FIXME: the what?) --- diff --git a/slides/k8s/lastwords-admin.md b/slides/k8s/lastwords-admin.md index 4fbde579..df9bab7a 100644 --- a/slides/k8s/lastwords-admin.md +++ b/slides/k8s/lastwords-admin.md @@ -48,7 +48,7 @@ - Acknowledge that a lot of tasks are outsourced - (e.g. if we add "buy / rack / provision machines" in that list) + (e.g. if we add "buy/rack/provision machines" in that list) --- @@ -122,7 +122,7 @@ (YAML, Helm charts, Kustomize ...) -- Team "run" adjusts some parameters and monitors the application +- Team "run" adjusts some parameters and monitors the application ✔️ parity between dev and prod environments @@ -150,7 +150,7 @@ - do we reward on-call duty without encouraging hero syndrome? - - do we give resources (time, money) to people to learn? + - do we give people resources (time, money) to learn? --- @@ -183,9 +183,9 @@ are a few tools that can help us.* - If cloud: public vs. private -- Which vendor / distribution to pick? +- Which vendor/distribution to pick? -- Which versions / features to enable? +- Which versions/features to enable? --- @@ -205,6 +205,6 @@ are a few tools that can help us.* - Transfer knowledge - (make sure everyone is on the same page / same level) + (make sure everyone is on the same page/level) - Iterate! From 98836d85cf8779d3da18fe031ea2f45380519ff6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Wed, 12 Jun 2019 15:53:26 -0500 Subject: [PATCH 16/27] Update cloud-controller-manager.md --- slides/k8s/cloud-controller-manager.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slides/k8s/cloud-controller-manager.md b/slides/k8s/cloud-controller-manager.md index b1c1f7cc..0e6dd06f 100644 --- a/slides/k8s/cloud-controller-manager.md +++ b/slides/k8s/cloud-controller-manager.md @@ -28,7 +28,7 @@ - Managing *some* volumes (e.g. ELBs, AzureDisks...) - (Eventually, volumes will be managed by the CSI) (FIXME: the what?) + (Eventually, volumes will be managed by the Container Storage Interface) --- From 990a873e8164c541fb8d5209c58a94cf59a1e65f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Wed, 12 Jun 2019 15:55:35 -0500 Subject: [PATCH 17/27] Update csr-api.md --- slides/k8s/csr-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slides/k8s/csr-api.md b/slides/k8s/csr-api.md index 35e2fa02..2ad1a836 100644 --- a/slides/k8s/csr-api.md +++ b/slides/k8s/csr-api.md @@ -319,7 +319,7 @@ The command above generates: - Switch back to `cluster-admin`: ```bash - kctx - # mraa? FIXME? + kctx - ``` - Inspect the CSR: From ef6a5f05f8bca0300058d47df24caf2820feeb57 Mon Sep 17 00:00:00 2001 From: Carl Date: Tue, 11 Jun 2019 11:30:39 -0700 Subject: [PATCH 18/27] clarify language around CSRs three changes: CSRs don't have expiry dates "-nodes" just means "no encryption" it's not really specific to DES the cert comes from the controller not the CSR --- slides/k8s/csr-api.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/slides/k8s/csr-api.md b/slides/k8s/csr-api.md index 2ad1a836..f462caab 100644 --- a/slides/k8s/csr-api.md +++ b/slides/k8s/csr-api.md @@ -62,7 +62,7 @@ This is what I do if I want to obtain a certificate. 2. Create a Certificate Signing Request (CSR). - (The CSR contains the identity that I claim and an expiration date.) + (The CSR contains the identity that I claim and a public key.) 3. Send that CSR to the Certificate Authority (CA). @@ -345,7 +345,7 @@ The command above generates: kctx - ``` -- Retrieve the certificate from the CSR: +- Retrieve the certificate from the controller: ```bash kubectl get csr users:jean.doe \ -o jsonpath={.status.certificate} \ From 313df8f9ffbe222a074e17a32c8be4bfd12ed7dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Wed, 12 Jun 2019 16:00:15 -0500 Subject: [PATCH 19/27] Update csr-api.md --- slides/k8s/csr-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slides/k8s/csr-api.md b/slides/k8s/csr-api.md index f462caab..f5bc4976 100644 --- a/slides/k8s/csr-api.md +++ b/slides/k8s/csr-api.md @@ -345,7 +345,7 @@ The command above generates: kctx - ``` -- Retrieve the certificate from the controller: +- Retrieve the updated CSR object and extract the certificate: ```bash kubectl get csr users:jean.doe \ -o jsonpath={.status.certificate} \ From b21f61ad2740bb34c9b2d882d27205a97a100931 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Wed, 12 Jun 2019 23:03:28 -0500 Subject: [PATCH 20/27] Update link to distributions (thanks @cem-) --- slides/k8s/setup-k8s.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slides/k8s/setup-k8s.md b/slides/k8s/setup-k8s.md index 3800e78e..6b704ef0 100644 --- a/slides/k8s/setup-k8s.md +++ b/slides/k8s/setup-k8s.md @@ -90,4 +90,4 @@ - For a longer list, check the Kubernetes documentation:
- it has a great guide to [pick the right solution](https://kubernetes.io/docs/setup/pick-right-solution/) to set up Kubernetes. + it has a great guide to [pick the right solution](https://kubernetes.io/docs/setup/#production-environment) to set up Kubernetes. From 626e4a8e35cbdfe72ce458de0ec0d6690072c435 Mon Sep 17 00:00:00 2001 From: AJ Bowen Date: Mon, 17 Jun 2019 04:16:15 -0700 Subject: [PATCH 21/27] Tweaks (#482) Add Firewalling slide; alter some wording --- slides/k8s/cluster-backup.md | 2 +- slides/k8s/cni.md | 2 ++ slides/k8s/dmuc.md | 4 ++-- slides/k8s/multinode.md | 23 +++++++++++++++++++++-- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/slides/k8s/cluster-backup.md b/slides/k8s/cluster-backup.md index c42a153d..048d09ee 100644 --- a/slides/k8s/cluster-backup.md +++ b/slides/k8s/cluster-backup.md @@ -18,7 +18,7 @@ (it gives us replication primitives) -- Kubernetes helps us to clone/replicate environments +- Kubernetes helps us clone / replicate environments (all resources can be described with manifests) diff --git a/slides/k8s/cni.md b/slides/k8s/cni.md index cd206ab1..26a736b9 100644 --- a/slides/k8s/cni.md +++ b/slides/k8s/cni.md @@ -66,6 +66,8 @@ Look in each plugin's directory for its documentation. --- +class: extra-details + ## Conf vs conflist - There are two slightly different configuration formats diff --git a/slides/k8s/dmuc.md b/slides/k8s/dmuc.md index d70972aa..17f60543 100644 --- a/slides/k8s/dmuc.md +++ b/slides/k8s/dmuc.md @@ -175,7 +175,7 @@ Success! ] -So far, so good. +We should get `No resources found.` and the `kubernetes` service, respectively. Note: the API server automatically created the `kubernetes` service entry. @@ -225,7 +225,7 @@ Success? ] -Our Deployment is in a bad shape: +Our Deployment is in bad shape: ``` NAME READY UP-TO-DATE AVAILABLE AGE deployment.apps/web 0/1 0 0 2m26s diff --git a/slides/k8s/multinode.md b/slides/k8s/multinode.md index 9a4b1313..af6b6194 100644 --- a/slides/k8s/multinode.md +++ b/slides/k8s/multinode.md @@ -195,7 +195,7 @@ class: extra-details ## Check our pods -- The pods will be scheduled to the nodes +- The pods will be scheduled on the nodes - The nodes will pull the `nginx` image, and start the pods @@ -325,7 +325,7 @@ class: extra-details - We will add the `--network-plugin` and `--pod-cidr` flags -- We all have a "cluster number" (let's call that `C`) +- We all have a "cluster number" (let's call that `C`) printed on your VM info card - We will use pod CIDR `10.C.N.0/24` (where `N` is the node number: 1, 2, 3) @@ -480,6 +480,23 @@ Sometimes it works, sometimes it doesn't. Why? ```bash kubectl get nodes -o wide ``` + +--- + +## Firewalling + +- By default, Docker prevents containers from using arbitrary IP addresses + + (by setting up iptables rules) + +- We need to allow our containers to use our pod CIDR + +- For simplicity, we will insert a blanket iptables rule allowing all traffic: + + `iptables -I FORWARD -j ACCEPT` + +- This has to be done on every node + --- ## Setting up routing @@ -488,6 +505,8 @@ Sometimes it works, sometimes it doesn't. Why? - Create all the routes on all the nodes +- Insert the iptables rule allowing traffic + - Check that you can ping all the pods from one of the nodes - Check that you can `curl` the ClusterIP of the Service successfully From 8f7ca0d261fa7a56fe5e207a66a753bfaed628df Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Mon, 17 Jun 2019 20:55:57 -0500 Subject: [PATCH 22/27] Bump k8s version --- slides/k8s/versions-k8s.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slides/k8s/versions-k8s.md b/slides/k8s/versions-k8s.md index 937ccb02..285d008e 100644 --- a/slides/k8s/versions-k8s.md +++ b/slides/k8s/versions-k8s.md @@ -1,6 +1,6 @@ ## Versions installed -- Kubernetes 1.14.2 +- Kubernetes 1.14.3 - Docker Engine 18.09.6 - Docker Compose 1.21.1 From ad4c86b3f4b6f6dd8d5ba0ed540635fdfcc51d6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Thu, 20 Jun 2019 01:47:48 +0000 Subject: [PATCH 23/27] Show instance type when provisioning; change it to t3.medium by default --- prepare-vms/lib/infra/aws.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/prepare-vms/lib/infra/aws.sh b/prepare-vms/lib/infra/aws.sh index cb5f8fd0..f7721003 100644 --- a/prepare-vms/lib/infra/aws.sh +++ b/prepare-vms/lib/infra/aws.sh @@ -31,6 +31,7 @@ infra_start() { die "I could not find which AMI to use in this region. Try another region?" fi AWS_KEY_NAME=$(make_key_name) + AWS_INSTANCE_TYPE=${AWS_INSTANCE_TYPE-t3a.medium} sep "Starting instances" info " Count: $COUNT" @@ -38,10 +39,11 @@ infra_start() { info " Token/tag: $TAG" info " AMI: $AMI" info " Key name: $AWS_KEY_NAME" + info " Instance type: $AWS_INSTANCE_TYPE" result=$(aws ec2 run-instances \ --key-name $AWS_KEY_NAME \ --count $COUNT \ - --instance-type ${AWS_INSTANCE_TYPE-t2.medium} \ + --instance-type $AWS_INSTANCE_TYPE \ --client-token $TAG \ --block-device-mapping 'DeviceName=/dev/sda1,Ebs={VolumeSize=20}' \ --image-id $AMI) @@ -97,7 +99,7 @@ infra_disableaddrchecks() { } wait_until_tag_is_running() { - max_retry=50 + max_retry=100 i=0 done_count=0 while [[ $done_count -lt $COUNT ]]; do From 860907ccf0c93f23740e5272db878bcd510b2cf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Thu, 20 Jun 2019 01:50:01 +0000 Subject: [PATCH 24/27] Optimize admin clusters --- prepare-vms/setup-admin-clusters.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/prepare-vms/setup-admin-clusters.sh b/prepare-vms/setup-admin-clusters.sh index a4719d7a..75717e52 100755 --- a/prepare-vms/setup-admin-clusters.sh +++ b/prepare-vms/setup-admin-clusters.sh @@ -30,9 +30,9 @@ TAG=$PREFIX-$SETTINGS --settings settings/$SETTINGS.yaml \ --count $((3*$STUDENTS)) +./workshopctl disableaddrchecks $TAG ./workshopctl deploy $TAG ./workshopctl kubebins $TAG -./workshopctl disableaddrchecks $TAG ./workshopctl cards $TAG SETTINGS=admin-kuberouter @@ -43,11 +43,15 @@ TAG=$PREFIX-$SETTINGS --settings settings/$SETTINGS.yaml \ --count $((3*$STUDENTS)) +./workshopctl disableaddrchecks $TAG ./workshopctl deploy $TAG ./workshopctl kubebins $TAG -./workshopctl disableaddrchecks $TAG ./workshopctl cards $TAG +#INFRA=infra/aws-us-west-1 + +export AWS_INSTANCE_TYPE=t3a.medium + SETTINGS=admin-test TAG=$PREFIX-$SETTINGS ./workshopctl start \ @@ -59,3 +63,4 @@ TAG=$PREFIX-$SETTINGS ./workshopctl deploy $TAG ./workshopctl kube $TAG 1.13.5 ./workshopctl cards $TAG + From ab1a360cdcb677756bebe56d36c74bd671540ba3 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Wed, 19 Jun 2019 21:45:59 -0500 Subject: [PATCH 25/27] Add link to Velocity Berlin --- slides/index.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/slides/index.yaml b/slides/index.yaml index 668fdbe2..3a9b8517 100644 --- a/slides/index.yaml +++ b/slides/index.yaml @@ -1,3 +1,11 @@ +- date: [2019-11-04, 2019-11-05] + country: de + city: Berlin + event: Velocity + speaker: jpetazzo + title: Deploying and scaling applications with Kubernetes + attend: https://conferences.oreilly.com/velocity/vl-eu/public/schedule/detail/79109 + - date: 2019-11-13 country: fr city: Marseille From 9f1d2581fce73de039bf6d6d3437f6a066dafed8 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Fri, 21 Jun 2019 07:49:01 -0500 Subject: [PATCH 26/27] Bump k8s version --- slides/k8s/versions-k8s.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/slides/k8s/versions-k8s.md b/slides/k8s/versions-k8s.md index 285d008e..b12c45af 100644 --- a/slides/k8s/versions-k8s.md +++ b/slides/k8s/versions-k8s.md @@ -1,6 +1,6 @@ ## Versions installed -- Kubernetes 1.14.3 +- Kubernetes 1.15.0 - Docker Engine 18.09.6 - Docker Compose 1.21.1 @@ -23,7 +23,7 @@ class: extra-details ## Kubernetes and Docker compatibility -- Kubernetes 1.14 validates Docker Engine versions [up to 18.09](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.14.md#external-dependencies) +- Kubernetes 1.15 validates Docker Engine versions [up to 18.09](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG-1.15.md#dependencies)
(the latest version when Kubernetes 1.14 was released) From be72fbe80a7b1e3922d1290d33dbfbaf6d9220d0 Mon Sep 17 00:00:00 2001 From: Bridget Kromhout Date: Sun, 23 Jun 2019 10:59:25 -0500 Subject: [PATCH 27/27] Update operators-design.md Using "in" instead of "into" is correct for this case. --- slides/k8s/operators-design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slides/k8s/operators-design.md b/slides/k8s/operators-design.md index 9a02f2b8..c7b31d18 100644 --- a/slides/k8s/operators-design.md +++ b/slides/k8s/operators-design.md @@ -86,7 +86,7 @@ class: extra-details ## What can we store via the Kubernetes API? -- The API server stores most Kubernetes resources into etcd +- The API server stores most Kubernetes resources in etcd - Etcd is designed for reliability, not for performance