From b69119eed4e236aa81354b7bbb21d6a061719479 Mon Sep 17 00:00:00 2001 From: Jerome Petazzoni Date: Thu, 16 May 2019 22:21:40 -0500 Subject: [PATCH] Add operator chapter with nice ElasticSearch demo --- k8s/elasticsearch-cluster.yaml | 21 ++ k8s/elasticsearch-operator.yaml | 94 ++++++++ k8s/filebeat.yaml | 167 +++++++++++++++ k8s/local-path-storage.yaml | 110 ++++++++++ slides/k8s/operators.md | 366 ++++++++++++++++++++++++++++++++ slides/kube-fullday.yml | 1 + slides/kube-selfpaced.yml | 1 + slides/kube-twodays.yml | 1 + 8 files changed, 761 insertions(+) create mode 100644 k8s/elasticsearch-cluster.yaml create mode 100644 k8s/elasticsearch-operator.yaml create mode 100644 k8s/filebeat.yaml create mode 100644 k8s/local-path-storage.yaml create mode 100644 slides/k8s/operators.md diff --git a/k8s/elasticsearch-cluster.yaml b/k8s/elasticsearch-cluster.yaml new file mode 100644 index 00000000..23d8108d --- /dev/null +++ b/k8s/elasticsearch-cluster.yaml @@ -0,0 +1,21 @@ +apiVersion: enterprises.upmc.com/v1 +kind: ElasticsearchCluster +metadata: + name: es +spec: + kibana: + image: docker.elastic.co/kibana/kibana-oss:6.1.3 + image-pull-policy: Always + cerebro: + image: upmcenterprises/cerebro:0.7.2 + image-pull-policy: Always + elastic-search-image: upmcenterprises/docker-elasticsearch-kubernetes:6.1.3_0 + image-pull-policy: Always + client-node-replicas: 2 + master-node-replicas: 3 + data-node-replicas: 3 + network-host: 0.0.0.0 + use-ssl: false + data-volume-size: 10Gi + java-options: "-Xms512m -Xmx512m" + diff --git a/k8s/elasticsearch-operator.yaml b/k8s/elasticsearch-operator.yaml new file mode 100644 index 00000000..0049541e --- /dev/null +++ b/k8s/elasticsearch-operator.yaml @@ -0,0 +1,94 @@ +# This is mirrored from https://github.com/upmc-enterprises/elasticsearch-operator/blob/master/example/controller.yaml but using the elasticsearch-operator namespace instead of operator +--- +apiVersion: v1 +kind: Namespace +metadata: + name: elasticsearch-operator +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: elasticsearch-operator + namespace: elasticsearch-operator +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: elasticsearch-operator +rules: +- apiGroups: ["extensions"] + resources: ["deployments", "replicasets", "daemonsets"] + verbs: ["create", "get", "update", "delete", "list"] +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "get", "update", "delete", "list"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "create", "delete", "deletecollection"] +- apiGroups: [""] + resources: ["persistentvolumes", "persistentvolumeclaims", "services", "secrets", "configmaps"] + verbs: ["create", "get", "update", "delete", "list"] +- apiGroups: ["batch"] + resources: ["cronjobs", "jobs"] + verbs: ["create", "get", "deletecollection", "delete"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["list", "get", "watch"] +- apiGroups: ["apps"] + resources: ["statefulsets", "deployments"] + verbs: ["*"] +- apiGroups: ["enterprises.upmc.com"] + resources: ["elasticsearchclusters"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: elasticsearch-operator + namespace: elasticsearch-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: elasticsearch-operator +subjects: +- kind: ServiceAccount + name: elasticsearch-operator + namespace: elasticsearch-operator +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: elasticsearch-operator + namespace: elasticsearch-operator +spec: + replicas: 1 + template: + metadata: + labels: + name: elasticsearch-operator + spec: + containers: + - name: operator + image: upmcenterprises/elasticsearch-operator:0.2.0 + imagePullPolicy: Always + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 8000 + name: http + livenessProbe: + httpGet: + path: /live + port: 8000 + initialDelaySeconds: 10 + timeoutSeconds: 10 + readinessProbe: + httpGet: + path: /ready + port: 8000 + initialDelaySeconds: 10 + timeoutSeconds: 5 + serviceAccount: elasticsearch-operator diff --git a/k8s/filebeat.yaml b/k8s/filebeat.yaml new file mode 100644 index 00000000..690e9613 --- /dev/null +++ b/k8s/filebeat.yaml @@ -0,0 +1,167 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-config + namespace: kube-system + labels: + k8s-app: filebeat +data: + filebeat.yml: |- + filebeat.config: + inputs: + # Mounted `filebeat-inputs` configmap: + path: ${path.config}/inputs.d/*.yml + # Reload inputs configs as they change: + reload.enabled: false + modules: + path: ${path.config}/modules.d/*.yml + # Reload module configs as they change: + reload.enabled: false + + # To enable hints based autodiscover, remove `filebeat.config.inputs` configuration and uncomment this: + #filebeat.autodiscover: + # providers: + # - type: kubernetes + # hints.enabled: true + + processors: + - add_cloud_metadata: + + cloud.id: ${ELASTIC_CLOUD_ID} + cloud.auth: ${ELASTIC_CLOUD_AUTH} + + output.elasticsearch: + hosts: ['${ELASTICSEARCH_HOST:elasticsearch}:${ELASTICSEARCH_PORT:9200}'] + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-inputs + namespace: kube-system + labels: + k8s-app: filebeat +data: + kubernetes.yml: |- + - type: docker + containers.ids: + - "*" + processors: + - add_kubernetes_metadata: + in_cluster: true +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat +spec: + template: + metadata: + labels: + k8s-app: filebeat + spec: + serviceAccountName: filebeat + terminationGracePeriodSeconds: 30 + containers: + - name: filebeat + image: docker.elastic.co/beats/filebeat-oss:7.0.1 + args: [ + "-c", "/etc/filebeat.yml", + "-e", + ] + env: + - name: ELASTICSEARCH_HOST + value: elasticsearch-es.default.svc.cluster.local + - name: ELASTICSEARCH_PORT + value: "9200" + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: ELASTIC_CLOUD_ID + value: + - name: ELASTIC_CLOUD_AUTH + value: + securityContext: + runAsUser: 0 + # If using Red Hat OpenShift uncomment this: + #privileged: true + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/filebeat.yml + readOnly: true + subPath: filebeat.yml + - name: inputs + mountPath: /usr/share/filebeat/inputs.d + readOnly: true + - name: data + mountPath: /usr/share/filebeat/data + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + volumes: + - name: config + configMap: + defaultMode: 0600 + name: filebeat-config + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: inputs + configMap: + defaultMode: 0600 + name: filebeat-inputs + # data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart + - name: data + hostPath: + path: /var/lib/filebeat-data + type: DirectoryOrCreate +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: filebeat +subjects: +- kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: filebeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: filebeat + labels: + k8s-app: filebeat +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - pods + verbs: + - get + - watch + - list +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat +--- diff --git a/k8s/local-path-storage.yaml b/k8s/local-path-storage.yaml new file mode 100644 index 00000000..7374a02a --- /dev/null +++ b/k8s/local-path-storage.yaml @@ -0,0 +1,110 @@ +# This is a local copy of: +# https://github.com/rancher/local-path-provisioner/blob/master/deploy/local-path-storage.yaml +--- +apiVersion: v1 +kind: Namespace +metadata: + name: local-path-storage +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: local-path-provisioner-service-account + namespace: local-path-storage +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: local-path-provisioner-role + namespace: local-path-storage +rules: +- apiGroups: [""] + resources: ["nodes", "persistentvolumeclaims"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["endpoints", "persistentvolumes", "pods"] + verbs: ["*"] +- apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: local-path-provisioner-bind + namespace: local-path-storage +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: local-path-provisioner-role +subjects: +- kind: ServiceAccount + name: local-path-provisioner-service-account + namespace: local-path-storage +--- +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: local-path-provisioner + namespace: local-path-storage +spec: + replicas: 1 + selector: + matchLabels: + app: local-path-provisioner + template: + metadata: + labels: + app: local-path-provisioner + spec: + serviceAccountName: local-path-provisioner-service-account + containers: + - name: local-path-provisioner + image: rancher/local-path-provisioner:v0.0.8 + imagePullPolicy: Always + command: + - local-path-provisioner + - --debug + - start + - --config + - /etc/config/config.json + volumeMounts: + - name: config-volume + mountPath: /etc/config/ + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumes: + - name: config-volume + configMap: + name: local-path-config +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: local-path +provisioner: rancher.io/local-path +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Delete +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: local-path-config + namespace: local-path-storage +data: + config.json: |- + { + "nodePathMap":[ + { + "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES", + "paths":["/opt/local-path-provisioner"] + } + ] + } + diff --git a/slides/k8s/operators.md b/slides/k8s/operators.md new file mode 100644 index 00000000..66253edb --- /dev/null +++ b/slides/k8s/operators.md @@ -0,0 +1,366 @@ +# Operators + +- Operators are one of the many ways to extend Kubernetes + +- We will define operators + +- We will see how they work + +- We will install a specific operator (for ElasticSearch) + +- We will use it to provision an ElasticSearch cluster + +--- + +## What are operators? + +- Operators combine two things: + + - Custom Resource Definitions + + - controller code watching the corresponding resources and acting upon them + +- A given operator can define one or multiple CRDs + +- The controller code (control loop) can run within the cluster or elsewhere + + (running as a Deployment with 1 replica is a common scenario) + +--- + +## Why use operators? + +- Kubernetes gives us Deployments, StatefulSets, Services ... + +- These mechanisms give us building blocks to deploy applications + +- They work great for services that are made of *N* identical containers + + (like stateless ones) + +- They also work great for some stateful applications like Consul, etcd ... + + (with the help of highly persistent volumes) + +- They're not enough for complex services: + + - where different containers have different roles + + - where extra steps have to be taken when scaling or replacing containers + +--- + +## Use-cases for operators + +- Systems with primary/secondary replication + + Examples: MariaDB, MySQL, PostgreSQL, Redis ... + +- Systems where different groups of nodes have different roles + + Examples: ElasticSearch, MongoDB ... + +- Systems that depend on other non-trivial systems + + Examples: Flink, Kafka, ... + +--- + +## More use-cases + +- Representing and managing external resources + + (Example: [AWS Service Operator](https://operatorhub.io/operator/alpha/aws-service-operator.v0.0.1)) + +- Managing complex cluster add-ons + + (Example: [Istio operator](https://operatorhub.io/operator/beta/istio-operator.0.1.6)) + +- Deploying and managing our applications lifecycles + + (more on that later) + +--- + +## How operators work + +- An operator creates one or more CRDs + + (i.e., it creates new "Kinds" of resources on our cluster) + +- The operator also runs a *controller* that will watch its resources + +- Each time we create/updated/delete a resource, the controller is notified + + (we could write our own cheap controller with `kubectl get --watch`) + +--- + +## One operator in action + +- We will install the UPMC Enterprises ElasticSearch operator + +- This operator requires PersistentVolumes + +- We will install Rancher's [local path storage provisioner](https://github.com/rancher/local-path-provisioner) to automatically create these + +- Then, we will create an ElasticSearch resource + +- The operator will detect that resource and provision the cluster + +--- + +## Installing a Persistent Volume provisioner + +(This step can be skipped if you already have a dynamic volume provisioner.) + +- This provisioner creates Persistent Volumes backed by `hostPath` + + (local directories on our nodes) + +- It doesn't require anything special ... + +- ... But losing a node = losing the volumes on that node! + +.exercise[ + +- Install the local path storage provisioner: + ```bash + kubectl apply -f ~/container.training/k8s/local-path-storage.yaml + ``` + +] + +--- + +## Making sure we have a default StorageClass + +- The ElasticSearch operator will create StatefulSets + +- These StatefulSets will instantiate PersistentVolumeClaims + +- These PVCs need to be explicitly associated with a StorageClass + +- Or we need to tag a StorageClass to be used as the default one + +.exercise[ + +- List StorageClasses: + ```bash + kubectl get storageclasses + ``` + +] + +We should see the `local-path` StorageClass. + +--- + +## Setting a default StorageClass + +- This is done by adding an annotation to the StorageClass: + + `storageclass.kubernetes.io/is-default-class: true` + +.exercise[ + +- Tag the StorageClass so that it's the default one: + ```bash + kubectl annotate storageclass local-path \ + storageclass.kubernetes.io/is-default-class=true + ``` + +- Check the result: + ```bash + kubectl get storageclasses + ``` + +] + +Now, the StorageClass should have `(default)` next to its name. + +--- + +## Install the ElasticSearch operator + +- The operator needs: + + - a Deployment for its controller + - a ServiceAccount, ClusterRole, ClusterRoleBinding for permissions + - a Namespace + +- We have grouped all the definitions for these resources in a YAML file + +.exercise[ + +- Install the operator: + ```bash + kubectl apply -f ~/container.training/k8s/elasticsearch-operator.yaml + ``` + +] + +--- + +## Wait for the operator to be ready + +- Some operators require to create their CRDs separately + +- This operator will create its CRD itself + + (i.e. the CRD is not listed in the YAML that we applied earlier) + +.exercise[ + +- Wait until the `elasticsearchclusters` CRD shows up: + ```bash + kubectl get crds + ``` + +] + +--- + +## Create an ElasticSearch resource + +- We can now create a resource with `kind: ElasticsearchCluster` + +- The YAML for that resource will specify all the desired parameters: + + - how many nodes do we want of each type (client, master, data) + - image to use + - add-ons (kibana, cerebro, ...) + - whether to use TLS or not + - etc. + +.exercise[ + +- Create our ElasticSearch cluster: + ```bash + kubectl apply -f ~/container.training/k8s/elasticsearch-cluster.yaml + ``` + +] + +--- + +## Operator in action + +- Over the next minutes, the operator will create: + + - StatefulSets (one for master nodes, one for data nodes) + + - Deployments (for client nodes; and for add-ons like cerebro and kibana) + + - Services (for all these pods) + +.exercise[ + +- Wait for all the StatefulSets to be fully up and running: + ```bash + kubectl get statefulsets -w + ``` + +] + +--- + +## Connecting to our cluster + +- Since connecting directly to the ElasticSearch API is a bit raw, +
we'll connect to the cerebro frontend instead + +.exercise[ + +- Edit the cerebro service to change its type from ClusterIP to NodePort: + ```bash + kubectl patch svc cerebro-es -p "spec: { type: NodePort }" + ``` + +- Retrieve the NodePort that was allocated: + ```bash + kubectl get svc cerebreo-es + ``` + +- Connect to that port with a browser + +] + +--- + +## (Bonus) Setup filebeat + +- Let's send some data to our brand new ElasticSearch cluster! + +- We'll deploy a filebeat DaemonSet to collect node logs + +.exercise[ + +- Deploy filebeat: + ```bash + kubectl apply -f ~/container.training/k8s/filebeat.yaml + ``` + +] + +We should see at least one index being created in cerebro. + +--- + +## (Bonus) Access log data with kibana + +- Let's expose kibana (by making kibana-es a NodePort too) + +- Then access kibana + +- We'll need to configure kibana indexes + +--- + +## Deploying our apps with operators + +- It is very simple to deploy with `kubectl run` / `kubectl expose` + +- We can unlock more features by writing YAML and using `kubectl apply` + +- Kustomize or Helm let us deploy in multiple environments + + (and adjust/tweak parameters in each environment) + +- We can also use an operator to deploy our application + +--- + +## Pros and cons of deploying with operators + +- The app definition and configuration is persisted in the Kubernetes API + +- Multiple instances of the app can be manipulated with `kubectl get` + +- We can add labels, annotations to the app instances + +- Our controller can execute custom code for any lifecycle event + +- However, we need to write this controller + +- We need to be careful about changes + + (what happens when the resource `spec` is updated?) + +--- + +## Operators are not magic + +- Look at the ElasticSearch resource definition + + (`~/container.training/k8s/elasticsearch-cluster.yaml`) + +- What should happen if we flip the `use-tls` flag? Twice? + +- What should happen if we remove / re-add the kibana or cerebro sections? + +- What should happen if we change the number of nodes? + +- What if we want different images or parameters for the different nodes? + +*Operators can be very powerful, iff we know exactly the scenarios that they can handle.* diff --git a/slides/kube-fullday.yml b/slides/kube-fullday.yml index 110c1b60..0fc48380 100644 --- a/slides/kube-fullday.yml +++ b/slides/kube-fullday.yml @@ -65,6 +65,7 @@ chapters: # - k8s/configuration.md #- - k8s/owners-and-dependents.md # - k8s/extending-api.md +# - k8s/operators.md # - k8s/statefulsets.md # - k8s/portworx.md - - k8s/whatsnext.md diff --git a/slides/kube-selfpaced.yml b/slides/kube-selfpaced.yml index 358f5925..528d2311 100644 --- a/slides/kube-selfpaced.yml +++ b/slides/kube-selfpaced.yml @@ -65,6 +65,7 @@ chapters: - k8s/configuration.md - - k8s/owners-and-dependents.md - k8s/extending-api.md + - k8s/operators.md - k8s/statefulsets.md - k8s/portworx.md - k8s/staticpods.md diff --git a/slides/kube-twodays.yml b/slides/kube-twodays.yml index 61aaac18..f286d081 100644 --- a/slides/kube-twodays.yml +++ b/slides/kube-twodays.yml @@ -65,6 +65,7 @@ chapters: - k8s/configuration.md #- k8s/owners-and-dependents.md - k8s/extending-api.md + - k8s/operators.md - - k8s/statefulsets.md - k8s/portworx.md - k8s/staticpods.md