diff --git a/k8s/consul.yaml b/k8s/consul.yaml new file mode 100644 index 00000000..2e5bc138 --- /dev/null +++ b/k8s/consul.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: Service +metadata: + name: consul +spec: + ports: + - port: 8500 + name: http + selector: + app: consul +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: consul +spec: + serviceName: consul + replicas: 3 + selector: + matchLabels: + app: consul + template: + metadata: + labels: + app: consul + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - consul + topologyKey: kubernetes.io/hostname + terminationGracePeriodSeconds: 10 + containers: + - name: consul + image: "consul:1.2.2" + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + args: + - "agent" + - "-bootstrap-expect=3" + - "-retry-join=consul-0.consul.$(NAMESPACE).svc.cluster.local" + - "-retry-join=consul-1.consul.$(NAMESPACE).svc.cluster.local" + - "-retry-join=consul-2.consul.$(NAMESPACE).svc.cluster.local" + - "-client=0.0.0.0" + - "-data-dir=/consul/data" + - "-server" + - "-ui" + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - consul leave diff --git a/k8s/docker-build.yaml b/k8s/docker-build.yaml new file mode 100644 index 00000000..07c3d35d --- /dev/null +++ b/k8s/docker-build.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Pod +metadata: + name: build-image +spec: + restartPolicy: OnFailure + containers: + - name: docker-build + image: docker + env: + - name: REGISTRY_PORT + value: #"30000" + command: ["sh", "-c"] + args: + - | + apk add --no-cache git && + mkdir /workspace && + git clone https://github.com/jpetazzo/container.training /workspace && + docker build -t localhost:$REGISTRY_PORT/worker /workspace/dockercoins/worker && + docker push localhost:$REGISTRY_PORT/worker + volumeMounts: + - name: docker-socket + mountPath: /var/run/docker.sock + volumes: + - name: docker-socket + hostPath: + path: /var/run/docker.sock + diff --git a/k8s/haproxy.cfg b/k8s/haproxy.cfg new file mode 100644 index 00000000..8cce5563 --- /dev/null +++ b/k8s/haproxy.cfg @@ -0,0 +1,18 @@ +global + daemon + maxconn 256 + +defaults + mode tcp + timeout connect 5000ms + timeout client 50000ms + timeout server 50000ms + +frontend the-frontend + bind *:80 + default_backend the-backend + +backend the-backend + server google.com-80 google.com:80 maxconn 32 check + server bing.com-80 bing.com:80 maxconn 32 check + diff --git a/k8s/haproxy.yaml b/k8s/haproxy.yaml new file mode 100644 index 00000000..174ba9ad --- /dev/null +++ b/k8s/haproxy.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: haproxy +spec: + volumes: + - name: config + configMap: + name: haproxy + containers: + - name: haproxy + image: haproxy + volumeMounts: + - name: config + mountPath: /usr/local/etc/haproxy/ + diff --git a/k8s/ingress.yaml b/k8s/ingress.yaml new file mode 100644 index 00000000..65639357 --- /dev/null +++ b/k8s/ingress.yaml @@ -0,0 +1,14 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: cheddar +spec: + rules: + - host: cheddar.A.B.C.D.nip.io + http: + paths: + - path: / + backend: + serviceName: cheddar + servicePort: 80 + diff --git a/k8s/kaniko-build.yaml b/k8s/kaniko-build.yaml new file mode 100644 index 00000000..f6b95b54 --- /dev/null +++ b/k8s/kaniko-build.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kaniko-build +spec: + initContainers: + - name: git-clone + image: alpine + command: ["sh", "-c"] + args: + - | + apk add --no-cache git && + git clone git://github.com/jpetazzo/container.training /workspace + volumeMounts: + - name: workspace + mountPath: /workspace + containers: + - name: build-image + image: gcr.io/kaniko-project/executor:latest + args: + - "--context=/workspace/dockercoins/rng" + - "--insecure-skip-tls-verify" + - "--destination=registry:5000/rng-kaniko:latest" + volumeMounts: + - name: workspace + mountPath: /workspace + volumes: + - name: workspace + diff --git a/k8s/nginx-with-volume.yaml b/k8s/nginx-with-volume.yaml new file mode 100644 index 00000000..7ad56441 --- /dev/null +++ b/k8s/nginx-with-volume.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx-with-volume +spec: + volumes: + - name: www + containers: + - name: nginx + image: nginx + volumeMounts: + - name: www + mountPath: /usr/share/nginx/html/ + - name: git + image: alpine + command: [ "sh", "-c", "apk add --no-cache git && git clone https://github.com/octocat/Spoon-Knife /www" ] + volumeMounts: + - name: www + mountPath: /www/ + restartPolicy: OnFailure + diff --git a/k8s/portworx.yaml b/k8s/portworx.yaml new file mode 100644 index 00000000..222a9a00 --- /dev/null +++ b/k8s/portworx.yaml @@ -0,0 +1,580 @@ +# SOURCE: https://install.portworx.com/?kbver=1.11.2&b=true&s=/dev/loop0&c=px-workshop&stork=true&lh=true +apiVersion: v1 +kind: ConfigMap +metadata: + name: stork-config + namespace: kube-system +data: + policy.cfg: |- + { + "kind": "Policy", + "apiVersion": "v1", + "extenders": [ + { + "urlPrefix": "http://stork-service.kube-system.svc:8099", + "apiVersion": "v1beta1", + "filterVerb": "filter", + "prioritizeVerb": "prioritize", + "weight": 5, + "enableHttps": false, + "nodeCacheCapable": false + } + ] + } +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: stork-account + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: stork-role +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "delete"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + - apiGroups: ["volumesnapshot.external-storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["volumesnapshot.external-storage.k8s.io"] + resources: ["volumesnapshotdatas"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create", "update"] + - apiGroups: [""] + resources: ["services"] + verbs: ["get"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["*"] + resources: ["deployments", "deployments/extensions"] + verbs: ["list", "get", "watch", "patch", "update", "initialize"] + - apiGroups: ["*"] + resources: ["statefulsets", "statefulsets/extensions"] + verbs: ["list", "get", "watch", "patch", "update", "initialize"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: stork-role-binding +subjects: +- kind: ServiceAccount + name: stork-account + namespace: kube-system +roleRef: + kind: ClusterRole + name: stork-role + apiGroup: rbac.authorization.k8s.io +--- +kind: Service +apiVersion: v1 +metadata: + name: stork-service + namespace: kube-system +spec: + selector: + name: stork + ports: + - protocol: TCP + port: 8099 + targetPort: 8099 +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + tier: control-plane + name: stork + namespace: kube-system +spec: + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + replicas: 3 + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + name: stork + tier: control-plane + spec: + containers: + - command: + - /stork + - --driver=pxd + - --verbose + - --leader-elect=true + - --health-monitor-interval=120 + imagePullPolicy: Always + image: openstorage/stork:1.1.3 + resources: + requests: + cpu: '0.1' + name: stork + hostPID: false + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "name" + operator: In + values: + - stork + topologyKey: "kubernetes.io/hostname" + serviceAccountName: stork-account +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: stork-snapshot-sc +provisioner: stork-snapshot +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: stork-scheduler-account + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: stork-scheduler-role +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch", "update"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["create"] + - apiGroups: [""] + resourceNames: ["kube-scheduler"] + resources: ["endpoints"] + verbs: ["delete", "get", "patch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["delete", "get", "list", "watch"] + - apiGroups: [""] + resources: ["bindings", "pods/binding"] + verbs: ["create"] + - apiGroups: [""] + resources: ["pods/status"] + verbs: ["patch", "update"] + - apiGroups: [""] + resources: ["replicationcontrollers", "services"] + verbs: ["get", "list", "watch"] + - apiGroups: ["app", "extensions"] + resources: ["replicasets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: ["statefulsets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims", "persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: stork-scheduler-role-binding +subjects: +- kind: ServiceAccount + name: stork-scheduler-account + namespace: kube-system +roleRef: + kind: ClusterRole + name: stork-scheduler-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + labels: + component: scheduler + tier: control-plane + name: stork-scheduler + name: stork-scheduler + namespace: kube-system +spec: + replicas: 3 + template: + metadata: + labels: + component: scheduler + tier: control-plane + name: stork-scheduler + spec: + containers: + - command: + - /usr/local/bin/kube-scheduler + - --address=0.0.0.0 + - --leader-elect=true + - --scheduler-name=stork + - --policy-configmap=stork-config + - --policy-configmap-namespace=kube-system + - --lock-object-name=stork-scheduler + image: gcr.io/google_containers/kube-scheduler-amd64:v1.11.2 + livenessProbe: + httpGet: + path: /healthz + port: 10251 + initialDelaySeconds: 15 + name: stork-scheduler + readinessProbe: + httpGet: + path: /healthz + port: 10251 + resources: + requests: + cpu: '0.1' + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "name" + operator: In + values: + - stork-scheduler + topologyKey: "kubernetes.io/hostname" + hostPID: false + serviceAccountName: stork-scheduler-account +--- +kind: Service +apiVersion: v1 +metadata: + name: portworx-service + namespace: kube-system + labels: + name: portworx +spec: + selector: + name: portworx + ports: + - name: px-api + protocol: TCP + port: 9001 + targetPort: 9001 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: px-account + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: node-get-put-list-role +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "get", "update", "list"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["delete", "get", "list"] +- apiGroups: [""] + resources: ["persistentvolumeclaims", "persistentvolumes"] + verbs: ["get", "list"] +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "update", "create"] +- apiGroups: ["extensions"] + resources: ["podsecuritypolicies"] + resourceNames: ["privileged"] + verbs: ["use"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: node-role-binding +subjects: +- kind: ServiceAccount + name: px-account + namespace: kube-system +roleRef: + kind: ClusterRole + name: node-get-put-list-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: Namespace +metadata: + name: portworx +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: px-role + namespace: portworx +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "create", "update", "patch"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: px-role-binding + namespace: portworx +subjects: +- kind: ServiceAccount + name: px-account + namespace: kube-system +roleRef: + kind: Role + name: px-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: portworx + namespace: kube-system + annotations: + portworx.com/install-source: "https://install.portworx.com/?kbver=1.11.2&b=true&s=/dev/loop0&c=px-workshop&stork=true&lh=true" +spec: + minReadySeconds: 0 + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + name: portworx + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: px/enabled + operator: NotIn + values: + - "false" + - key: node-role.kubernetes.io/master + operator: DoesNotExist + hostNetwork: true + hostPID: false + containers: + - name: portworx + image: portworx/oci-monitor:1.4.2.2 + imagePullPolicy: Always + args: + ["-c", "px-workshop", "-s", "/dev/loop0", "-b", + "-x", "kubernetes"] + env: + - name: "PX_TEMPLATE_VERSION" + value: "v4" + + livenessProbe: + periodSeconds: 30 + initialDelaySeconds: 840 # allow image pull in slow networks + httpGet: + host: 127.0.0.1 + path: /status + port: 9001 + readinessProbe: + periodSeconds: 10 + httpGet: + host: 127.0.0.1 + path: /health + port: 9015 + terminationMessagePath: "/tmp/px-termination-log" + securityContext: + privileged: true + volumeMounts: + - name: dockersock + mountPath: /var/run/docker.sock + - name: etcpwx + mountPath: /etc/pwx + - name: optpwx + mountPath: /opt/pwx + - name: proc1nsmount + mountPath: /host_proc/1/ns + - name: sysdmount + mountPath: /etc/systemd/system + - name: diagsdump + mountPath: /var/cores + - name: journalmount1 + mountPath: /var/run/log + readOnly: true + - name: journalmount2 + mountPath: /var/log + readOnly: true + - name: dbusmount + mountPath: /var/run/dbus + restartPolicy: Always + serviceAccountName: px-account + volumes: + - name: dockersock + hostPath: + path: /var/run/docker.sock + - name: etcpwx + hostPath: + path: /etc/pwx + - name: optpwx + hostPath: + path: /opt/pwx + - name: proc1nsmount + hostPath: + path: /proc/1/ns + - name: sysdmount + hostPath: + path: /etc/systemd/system + - name: diagsdump + hostPath: + path: /var/cores + - name: journalmount1 + hostPath: + path: /var/run/log + - name: journalmount2 + hostPath: + path: /var/log + - name: dbusmount + hostPath: + path: /var/run/dbus +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: px-lh-account + namespace: kube-system +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: px-lh-role + namespace: kube-system +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create", "update"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: px-lh-role-binding + namespace: kube-system +subjects: +- kind: ServiceAccount + name: px-lh-account + namespace: kube-system +roleRef: + kind: Role + name: px-lh-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: Service +metadata: + name: px-lighthouse + namespace: kube-system + labels: + tier: px-web-console +spec: + type: NodePort + ports: + - name: http + port: 80 + nodePort: 32678 + - name: https + port: 443 + nodePort: 32679 + selector: + tier: px-web-console +--- +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: px-lighthouse + namespace: kube-system + labels: + tier: px-web-console +spec: + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + tier: px-web-console + replicas: 1 + template: + metadata: + labels: + tier: px-web-console + spec: + initContainers: + - name: config-init + image: portworx/lh-config-sync:0.2 + imagePullPolicy: Always + args: + - "init" + volumeMounts: + - name: config + mountPath: /config/lh + containers: + - name: px-lighthouse + image: portworx/px-lighthouse:1.5.0 + imagePullPolicy: Always + ports: + - containerPort: 80 + - containerPort: 443 + volumeMounts: + - name: config + mountPath: /config/lh + - name: config-sync + image: portworx/lh-config-sync:0.2 + imagePullPolicy: Always + args: + - "sync" + volumeMounts: + - name: config + mountPath: /config/lh + serviceAccountName: px-lh-account + volumes: + - name: config + emptyDir: {} diff --git a/k8s/postgres.yaml b/k8s/postgres.yaml new file mode 100644 index 00000000..96061da8 --- /dev/null +++ b/k8s/postgres.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres +spec: + selector: + matchLabels: + app: postgres + serviceName: postgres + template: + metadata: + labels: + app: postgres + spec: + schedulerName: stork + containers: + - name: postgres + image: postgres:10.5 + volumeMounts: + - mountPath: /var/lib/postgresql + name: postgres + volumeClaimTemplates: + - metadata: + name: postgres + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi + diff --git a/k8s/registry.yaml b/k8s/registry.yaml new file mode 100644 index 00000000..21c25147 --- /dev/null +++ b/k8s/registry.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: registry +spec: + containers: + - name: registry + image: registry + env: + - name: REGISTRY_HTTP_ADDR + valueFrom: + configMapKeyRef: + name: registry + key: http.addr + diff --git a/k8s/storage-class.yaml b/k8s/storage-class.yaml new file mode 100644 index 00000000..8bd3988f --- /dev/null +++ b/k8s/storage-class.yaml @@ -0,0 +1,11 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: portworx-replicated + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: kubernetes.io/portworx-volume +parameters: + repl: "2" + priority_io: "high" + diff --git a/k8s/traefik.yaml b/k8s/traefik.yaml new file mode 100644 index 00000000..72b4173b --- /dev/null +++ b/k8s/traefik.yaml @@ -0,0 +1,100 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: traefik-ingress-controller + namespace: kube-system +--- +kind: DaemonSet +apiVersion: extensions/v1beta1 +metadata: + name: traefik-ingress-controller + namespace: kube-system + labels: + k8s-app: traefik-ingress-lb +spec: + template: + metadata: + labels: + k8s-app: traefik-ingress-lb + name: traefik-ingress-lb + spec: + tolerations: + - effect: NoSchedule + operator: Exists + hostNetwork: true + serviceAccountName: traefik-ingress-controller + terminationGracePeriodSeconds: 60 + containers: + - image: traefik + name: traefik-ingress-lb + ports: + - name: http + containerPort: 80 + hostPort: 80 + - name: admin + containerPort: 8080 + hostPort: 8080 + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + args: + - --api + - --kubernetes + - --logLevel=INFO +--- +kind: Service +apiVersion: v1 +metadata: + name: traefik-ingress-service + namespace: kube-system +spec: + selector: + k8s-app: traefik-ingress-lb + ports: + - protocol: TCP + port: 80 + name: web + - protocol: TCP + port: 8080 + name: admin +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: traefik-ingress-controller +rules: + - apiGroups: + - "" + resources: + - services + - endpoints + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - extensions + resources: + - ingresses + verbs: + - get + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: traefik-ingress-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: traefik-ingress-controller +subjects: +- kind: ServiceAccount + name: traefik-ingress-controller + namespace: kube-system diff --git a/slides/k8s/authn-authz.md b/slides/k8s/authn-authz.md new file mode 100644 index 00000000..019537bc --- /dev/null +++ b/slides/k8s/authn-authz.md @@ -0,0 +1,529 @@ +# Authentication and authorization + +*And first, a little refresher!* + +- Authentication = verifying the identity of a person + + On a UNIX system, we can authenticate with login+password, SSH keys ... + +- Authorization = listing what they are allowed to do + + On a UNIX system, this can include file permissions, sudoer entries ... + +- Sometimes abbreviated as "authn" and "authz" + +- In good modular systems, these things are decoupled + + (so we can e.g. change a password or SSH key without having to reset access rights) + +--- + +## Authentication in Kubernetes + +- When the API server receives a request, it tries to authenticate it + + (it examines headers, certificates ... anything available) + +- Many authentication methods can be used simultaneously: + + - TLS client certificates (that's what we've been doing with `kubectl` so far) + + - bearer tokens (a secret token in the HTTP headers of the request) + + - [HTTP basic auth](https://en.wikipedia.org/wiki/Basic_access_authentication) (carrying user and password in a HTTP header) + + - authentication proxy (sitting in front of the API and setting trusted headers) + +- It's the job of the authentication method to produce: + + - the user name + - the user ID + - a list of groups + +- The API server doesn't interpret these; it'll be the job of *authorizers* + +--- + +## Anonymous requests + +- If any authentication method *rejects* a request, it's denied + + (`401 Unauthorized` HTTP code) + +- If a request is neither accepted nor accepted by anyone, it's anonymous + + - the user name is `system:anonymous` + + - the list of groups is `[system:unauthenticated]` + +- By default, the anonymous user can't do anything + + (that's what you get if you just `curl` the Kubernetes API) + +--- + +## Authentication with TLS certificates + +- This is enabled in most Kubernetes deployments + +- The user name is derived from the `CN` in the client certificates + +- The groups are derived from the `O` fields in the client certificate + +- From the point of view of the Kubernetes API, users do not exist + + (i.e. they are not stored in etcd or anywhere else) + +- Users can be created (and given membership to groups) independently of the API + +- The Kubernetes API can be set up to use your custom CA to validate client certs + +--- + +class: extra-details + +## Viewing our admin certificate + +- Let's inspect the certificate we've been using all this time! + +.exercise[ + +- This command will show the `CN` and `O` fields for our certificate: + ```bash + kubectl config view \ + --raw \ + -o json \ + | jq -r .users[0].user[\"client-certificate-data\"] \ + | base64 -d \ + | openssl x509 -text \ + | grep Subject: + ``` + +] + +Let's break down that command together! 😅 + +--- + +class: extra-details + +## Breaking down the command + +- `kubectl config view` shows the Kubernetes user configuration +- `--raw` includes certificate information (which shows as REDACTED otherwise) +- `-o json` outputs the information in JSON format +- `| jq ...` extracts the field with the user certificate (in base64) +- `| base64 -d` decodes the base64 format (now we have a PEM file) +- `| openssl x509 -text` parses the certificate and outputs it as plain text +- `| grep Subject:` shows us the line that interests us + +→ We are user `kubernetes-admin`, in group `system:masters`. + +--- + +## Authentication with tokens + +- Tokens are passed as HTTP headers: + + `Authorization: Bearer and-then-here-comes-the-token` + +- Tokens can be validated through a number of different methods: + + - static tokens hard-coded in a file on the API server + + - [bootstrap tokens](https://kubernetes.io/docs/reference/access-authn-authz/bootstrap-tokens/) (special case to create a cluster or join nodes) + + - [OpenID Connect tokens](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#openid-connect-tokens) (to delegate authentication to compatible OAuth2 providers) + + - service accounts (these deserve more details, coming right up!) + +--- + +## Service accounts + +- A service account is a user that exists in the Kubernetes API + + (it is visible with e.g. `kubectl get serviceaccounts`) + +- Service accounts can therefore be created / updated dynamically + + (they don't require hand-editing a file and restarting the API server) + +- A service account is associated with a set of secrets + + (the kind that you can view with `kubectl get secrets`) + +- Service accounts are generally used to grant permissions to applications, services ... + + (as opposed to humans) + +--- + +class: extra-details + +## Token authentication in practice + +- We are going to list existing service accounts + +- Then we will extract the token for a given service account + +- And we will use that token to authenticate with the API + +--- + +class: extra-details + +## Listing service accounts + +.exercise[ + +- The resource name is `serviceaccount` or `sa` in short: + ```bash + kubectl get sa + ``` + + ] + + There should be just one service account in the default namespace: `default`. + + --- + + class: extra-details + + ## Finding the secret + + .exercise[ + + - List the secrets for the `default` service account: + ```bash + kubectl get sa default -o yaml + SECRET=$(kubectl get sa default -o json | jq -r .secrets[0].name) + ``` + +] + +It should be named `default-token-XXXXX`. + +--- + +class: extra-details + +## Extracting the token + +- The token is stored in the secret, wrapped with base64 encoding + +.exercise[ + +- View the secret: + ```bash + kubectl get $SECRET -o yaml + ``` + +- Extract the token and decode it: + ```bash + TOKEN=$(kubectl get secret $SECRET -o json \ + | jq -r .data.token | base64 -d) + ``` + +] + +--- + +class: extra-details + +## Using the token + +- Let's send a request to the API, without and with the token + +.exercise[ + +- Find the ClusterIP for the `kubernetes` service: + ```bash + kubectl get svc kubernetes + API=$(kubectl get svc kubernetes -o json | jq -r .spec.clusterIP) + ``` + +- Connect without the token: + ```bash + curl -k https://$API + ``` + +- Connect with the token: + ```bash + curl -k -H "Authorization: Bearer $TOKEN" https://$API + ``` + +] + +--- + +class: extra-details + +## Results + +- In both cases, we will get a "Forbidden" error + +- Without authentication, the user is `system:anonymous` + +- With authentication, it is shown as `system:serviceaccount:default:default` + +- The API "sees" us as a different user + +- But neither user has any right, so we can't do nothin' + +- Let's change that! + +--- + +## Authorization in Kubernetes + +- There are multiple ways to grant permissions in Kubernetes, called [authorizers](https://kubernetes.io/docs/reference/access-authn-authz/authorization/#authorization-modules): + + - [Node Authorization](https://kubernetes.io/docs/reference/access-authn-authz/node/) (used internally by kubelet; we can ignore it) + + - [Attribute-based access control](https://kubernetes.io/docs/reference/access-authn-authz/abac/) (powerful but complex and static; ignore it too) + + - [Webhook](https://kubernetes.io/docs/reference/access-authn-authz/webhook/) (each API request is submitted to an external service for approval) + + - [Role-based access control](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) (associates permissions to users dynamically) + +- The one we want is the last one, generally abbreviated as RBAC + +--- + +## Role-based access control + +- RBAC allows to specify fine-grained permissions + +- Permissions are expressed as *rules* + +- A rule is a combination of: + + - [verbs](https://kubernetes.io/docs/reference/access-authn-authz/authorization/#determine-the-request-verb) like create, get, list, update, delete ... + + - resources (as in "API resource", like pods, nodes, services ...) + + - resource names (to specify e.g. one specific pod instead of all pods) + + - in some case, [subresources](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#referring-to-resources) (e.g. logs are subresources of pods) + +--- + +## From rules to roles to rolebindings + +- A *role* is an API object containing a list of *rules* + + Example: role "external-load-balancer-configurator" can: + - [list, get] resources [endpoints, services, pods] + - [update] resources [services] + +- A *rolebinding* associates a role with a user + + Example: rolebinding "external-load-balancer-configurator": + - associates user "external-load-balancer-configurator" + - with role "external-load-balancer-configurator" + +- Yes, there can be users, roles, and rolebindings with the same name + +- It's a good idea for 1-1-1 bindings; not so much for 1-N ones + +--- + +## Cluster-scope permissions + +- API resources Role and RoleBinding are for objects within a namespace + +- We can also define API resources ClusterRole and ClusterRoleBinding + +- These are a superset, allowing to: + + - specify actions on cluster-wide objects (like nodes) + + - operate across all namespaces + +- We can create Role and RoleBinding resources within a namespaces + +- ClusterRole and ClusterRoleBinding resources are global + +--- + +## Pods and service accounts + +- A pod can be associated to a service account + + - by default, it is associated to the `default` service account + + - as we've seen earlier, this service account has no permission anyway + +- The associated token is exposed into the pod's filesystem + + (in `/var/run/secrets/kubernetes.io/serviceaccount/token`) + +- Standard Kubernetes tooling (like `kubectl`) will look for it there + +- So Kubernetes tools running in a pod will automatically use the service account + +--- + +## In practice + +- We are going to create a service account + +- We will use an existing cluster role (`view`) + +- We will bind together this role and this service account + +- Then we will run a pod using that service account + +- In this pod, we will install `kubectl` and check our permissions + +--- + +## Creating a service account + +- We will call the new service account `viewer` + + (note that nothing prevents us from calling it `view`, like the role) + +.exercise[ + +- Create the new service account: + ```bash + kubectl create serviceaccount viewer + ``` + +- List service accounts now: + ```bash + kubectl get serviceaccounts + ``` + +] + +--- + +## Binding a role to the service account + +- Binding a role = creating a *rolebinding* object + +- We will call that object `viewercanview` + + (but again, we could call it `view`) + +.exercise[ + +- Create the new role binding: + ```bash + kubectl create rolebinding viewercanview \ + --clusterrole=view \ + --serviceaccount=default:viewer + ``` + +] + +It's important to note a couple of details in these flags ... + +--- + +## Roles vs Cluster Roles + +- We used `--clusterrole=view` + +- What would have happened if we had used `--role=view`? + + - we would have bound the role `view` from the local namespace +
(instead of the cluster role `view`) + + - the command would have worked fine (no error) + + - but later, our API requests would have been denied + +- This is a deliberate design decision + + (we can reference roles that don't exist, and create/update them later) + +--- + +## Users vs Service Accounts + +- We used `--serviceaccount=default:viewer` + +- What would have happened if we had used `--user=default:viewer`? + + - we would have bound the role to a user instead of a service account + + - again, the command would have worked fine (no error) + + - ... but our API requests would have been denied later + +- What's about the `default:` prefix? + + - that's the namespace of the service account + + - yes, it could be inferred from context, but ... `kubectl` requires it + +--- + +## Testing + +- We will run an `alpine` pod and install `kubectl` there + +.exercise[ + +- Run a one-time pod: + ```bash + kubectl run eyepod --rm -ti --restart=Never \ + --serviceaccount=viewer \ + --image alpine + ``` + +- Install `curl`, then use it to install `kubectl`: + ```bash + apk add --no-cache curl + URLBASE=https://storage.googleapis.com/kubernetes-release/release + KUBEVER=$(curl -s $URLBASE/stable.txt) + curl -LO $URLBASE/$KUBEVER/bin/linux/amd64/kubectl + chmod +x kubectl + ``` + +] + +--- + +## Running `kubectl` in the pod + +- We'll try to use our `view` permissions, then to create an object + +.exercise[ + +- Check that we can, indeed, view things: + ```bash + ./kubectl get all + ``` + +- But that we can't create things: + ```bash + ./kubectl run tryme --image=nginx + ``` + +] + +--- + +## Testing directly with `kubectl` + +- We can also check for permission with `kubectl auth can-i`: + ```bash + kubectl auth can-i list nodes + kubectl auth can-i create pods + kubectl auth can-i get pod/name-of-pod + kubectl auth can-i get /url-fragment-of-api-request/ + kubectl auth can-i '*' services + ``` + +- And we can check permissions on behalf of other users: + ```bash + kubectl auth can-i list nodes \ + --as some-user + kubectl auth can-i list nodes \ + --as system:serviceaccount:: + ``` diff --git a/slides/k8s/build-with-docker.md b/slides/k8s/build-with-docker.md new file mode 100644 index 00000000..4948fc2a --- /dev/null +++ b/slides/k8s/build-with-docker.md @@ -0,0 +1,156 @@ +# Building images with the Docker Engine + +- Until now, we have built our images manually, directly on a node + +- We are going to show how to build images from within the cluster + + (by executing code in a container controlled by Kubernetes) + +- We are going to use the Docker Engine for that purpose + +- To access the Docker Engine, we will mount the Docker socket in our container + +- After building the image, we will push it to our self-hosted registry + +--- + +## Resource specification for our builder pod + +.small[ +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: build-image +spec: + restartPolicy: OnFailure + containers: + - name: docker-build + image: docker + env: + - name: REGISTRY_PORT + value: "`3XXXX`" + command: ["sh", "-c"] + args: + - | + apk add --no-cache git && + mkdir /workspace && + git clone https://github.com/jpetazzo/container.training /workspace && + docker build -t localhost:$REGISTRY_PORT/worker /workspace/dockercoins/worker && + docker push localhost:$REGISTRY_PORT/worker + volumeMounts: + - name: docker-socket + mountPath: /var/run/docker.sock + volumes: + - name: docker-socket + hostPath: + path: /var/run/docker.sock +``` +] + +--- + +## Breaking down the pod specification (1/2) + +- `restartPolicy: OnFailure` prevents the build from running in an infinite lopo + +- We use the `docker` image (so that the `docker` CLI is available) + +- We rely on the fact that the `docker` image is based on `alpine` + + (which is why we use `apk` to install `git`) + +- The port for the registry is passed through an environment variable + + (this avoids repeating it in the specification, which would be error-prone) + +.warning[The environment variable has to be a string, so the `"`s are mandatory!] + +--- + +## Breaking down the pod specification (2/2) + +- The volume `docker-socket` is declared with a `hostPath`, indicating a bind-mount + +- It is then mounted in the container onto the default Docker socket path + +- We show a interesting way to specify the commands to run in the container: + + - the command executed will be `sh -c ` + + - `args` is a list of strings + + - `|` is used to pass a multi-line string in the YAML file + +--- + +## Running our pod + +- Let's try this out! + +.exercise[ + +- Check the port used by our self-hosted registry: + ```bash + kubectl get svc registry + ``` + +- Edit `~/container.training/k8s/docker-build.yaml` to put the port number + +- Schedule the pod by applying the resource file: + ```bash + kubectl apply -f ~/container.training/k8s/docker-build.yaml + ``` + +- Watch the logs: + ```bash + stern build-image + ``` + +] + +--- + +## What's missing? + +What do we need to change to make this production-ready? + +- Build from a long-running container (e.g. a `Deployment`) triggered by web hooks + + (the payload of the web hook could indicate the repository to build) + +- Build a specific branch or tag; tag image accordingly + +- Handle repositories where the Dockerfile is not at the root + + (or containing multiple Dockerfiles) + +- Expose build logs so that troubleshooting is straightforward + +-- + +🤔 That seems like a lot of work! + +-- + +That's why services like Docker Hub (with [automated builds](https://docs.docker.com/docker-hub/builds/)) are helpful. +
+They handle the whole "code repository → Docker image" workflow. + +--- + +## Things to be aware of + +- This is talking directly to a node's Docker Engine to build images + +- It bypasses resource allocation mechanisms used by Kubernetes + + (but you can use *taints* and *tolerations* to dedicate builder nodes) + +- Be careful not to introduce conflicts when naming images + + (e.g. do not allow the user to specify the image names!) + +- Your builds are going to be *fast* + + (because they will leverage Docker's caching system) diff --git a/slides/k8s/build-with-kaniko.md b/slides/k8s/build-with-kaniko.md new file mode 100644 index 00000000..b3245f91 --- /dev/null +++ b/slides/k8s/build-with-kaniko.md @@ -0,0 +1,213 @@ +# Building images with Kaniko + +- [Kaniko](https://github.com/GoogleContainerTools/kaniko) is an open source tool to build container images within Kubernetes + +- It can build an image using any standard Dockerfile + +- The resulting image can be pushed to a registry or exported as a tarball + +- It doesn't require any particular privilege + + (and can therefore run in a regular container in a regular pod) + +- This combination of features is pretty unique + + (most other tools use different formats, or require elevated privileges) + +--- + +## Kaniko in practice + +- Kaniko provides an "executor image", `gcr.io/kaniko-project/executor` + +- When running that image, we need to specify at least: + + - the path to the build context (=the directory with our Dockerfile) + + - the target image name (including the registry address) + +- Simplified example: + ``` + docker run \ + -v ...:/workspace gcr.io/kaniko-project/executor \ + --context=/workspace \ + --destination=registry:5000/image_name:image_tag + ``` + +--- + +## Running Kaniko in a Docker container + +- Let's build the image for the DockerCoins `worker` service with Kaniko + +.exercise[ + +- Find the port number for our self-hosted registry: + ```bash + kubectl get svc registry + PORT=$(kubectl get svc registry -o json | jq .spec.ports[0].nodePort) + ``` + +- Run Kaniko: + ```bash + docker run --net host \ + -v ~/container.training/dockercoins/worker:/workspace \ + gcr.io/kaniko-project/executor \ + --context=/workspace \ + --destination=127.0.0.1:30448/worker-kaniko:latest + ``` + +] + +We use `--net host` so that we can connect to the registry over `127.0.0.1`. + +--- + +## Running Kaniko in a Kubernetes pod + +- We need to mount or copy the build context to the pod + +- We are going to build straight from the git repository + + (to avoid depending on files sitting on a node, outside of containers) + +- We need to `git clone` the repository before running Kaniko + +- We are going to use two containers sharing a volume: + + - a first container to `git clone` the repository to the volume + + - a second container to run Kaniko, using the content of the volume + +- However, we need the first container to be done before running the second one + +🤔 How could we do that? + +--- + +## [Init Containers](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) to the rescue + +- A pod can have a list of `initContainers` + +- `initContainers` are executed in the specified order + +- Each Init Container needs to complete (exit) successfully + +- If any Init Container fails (non-zero exit status) the pod fails + + (what happens next depends on the pod's `restartPolicy`) + +- After all Init Containers have run successfully, normal `containers` are started + +- We are going to execute the `git clone` operation in an Init Container + +--- + +## Our Kaniko builder pod + +.small[ +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: kaniko-build +spec: + initContainers: + - name: git-clone + image: alpine + command: ["sh", "-c"] + args: + - | + apk add --no-cache git && + git clone git://github.com/jpetazzo/container.training /workspace + volumeMounts: + - name: workspace + mountPath: /workspace + containers: + - name: build-image + image: gcr.io/kaniko-project/executor:latest + args: + - "--context=/workspace/dockercoins/rng" + - "--insecure-skip-tls-verify" + - "--destination=registry:5000/rng-kaniko:latest" + volumeMounts: + - name: workspace + mountPath: /workspace + volumes: + - name: workspace +``` +] + +--- + +## Explanations + +- We define a volume named `workspace` (using the default `emptyDir` provider) + +- That volume is mounted to `/workspace` in both our containers + +- The `git-clone` Init Container installs `git` and runs `git clone` + +- The `build-image` container executes Kaniko + +- We use our self-hosted registry DNS name (`registry`) + +- We add `--insecure-skip-tls-verify` since our registry doesn't have TLS certs + +--- + +## Running our Kaniko builder pod + +- The YAML for the pod is in `k8s/kaniko-build.yaml` + +.exercise[ + +- Create the pod: + ```bash + kubectl apply -f ~/container.training/k8s/kaniko-build.yaml + ``` + +- Watch the logs: + ```bash + stern kaniko + ``` + +] + +--- + +## Discussion + +*What should we use? The Docker build technique shown earlier? Kaniko? Something else?* + +- The Docker build technique is simple, and has the potential to be very fast + +- However, it doesn't play nice with Kubernetes resource limits + +- Kaniko plays nice with resource limits + +- However, it's slower (there is no caching at all) + +- The ultimate building tool will probably be [Jessica Frazelle](https://twitter.com/jessfraz)'s [img](https://github.com/genuinetools/img) builder + + (it depends on upstream changes that are not in Kubernetes 1.11.2 yet) + +But ... is it all about [speed](https://github.com/AkihiroSuda/buildbench/issues/1)? (No!) + +--- + +## The big picture + +- For starters: the [Docker Hub automated builds](https://docs.docker.com/docker-hub/builds/) are very easy to set up + + - link a GitHub repository with the Docker Hub + + - each time you push to GitHub, an image gets build on the Docker Hub + +- If this doesn't work for you: why? + + - too slow (I'm far from `us-east-1`!) → consider using your cloud provider's registry + + - I'm not using a cloud provider → ok, perhaps you need to self-host then + + - I need fancy features (e.g. CI) → consider something like GitLab diff --git a/slides/k8s/configuration.md b/slides/k8s/configuration.md new file mode 100644 index 00000000..b0ce5613 --- /dev/null +++ b/slides/k8s/configuration.md @@ -0,0 +1,533 @@ +# Managing configuration + +- Some applications need to be configured (obviously!) + +- There are many ways for our code to pick up configuration: + + - command-line arguments + + - environment variables + + - configuration files + + - configuration servers (getting configuration from a database, an API...) + + - ... and more (because programmers can be very creative!) + +- How can we do these things with containers and Kubernetes? + +--- + +## Passing configuration to containers + +- There are many ways to pass configuration to code running in a container: + + - baking it in a custom image + + - command-line arguments + + - environment variables + + - injecting configuration files + + - exposing it over the Kubernetes API + + - configuration servers + +- Let's review these different strategies! + +--- + +## Baking custom images + +- Put the configuration in the image + + (it can be in a configuration file, but also `ENV` or `CMD` actions) + +- It's easy! It's simple! + +- Unfortunately, it also has downsides: + + - multiplication of images + + - different images for dev, staging, prod ... + + - minor reconfigurations require a whole build/push/pull cycle + +- Avoid doing it unless you don't have the time to figure out other options + +--- + +## Command-line arguments + +- Pass options to `args` array in the container specification + +- Example ([source](https://github.com/coreos/pods/blob/master/kubernetes.yaml#L29)): + ```yaml + args: + - "--data-dir=/var/lib/etcd" + - "--advertise-client-urls=http://127.0.0.1:2379" + - "--listen-client-urls=http://127.0.0.1:2379" + - "--listen-peer-urls=http://127.0.0.1:2380" + - "--name=etcd" + ``` + +- The options can be passed directly to the program that we run ... + + ... or to a wrapper script that will use them to e.g. generate a config file + +--- + +## Command-line arguments, pros & cons + +- Works great when options are passed directly to the running program + + (otherwise, a wrapper script can work around the issue) + +- Works great when there aren't too many parameters + + (to avoid a 20-lines `args` array) + +- Requires documentation and/or understanding of the underlying program + + ("which parameters and flags do I need, again?") + +- Well-suited for mandatory parameters (without default values) + +- Not ideal when we need to pass a real configuration file anyway + +--- + +## Environment variables + +- Pass options through the `env` map in the container specification + +- Example: + ```yaml + env: + - name: ADMIN_PORT + value: "8080" + - name: ADMIN_AUTH + value: Basic + - name: ADMIN_CRED + value: "admin:0pensesame!" + ``` + +.warning[`value` must be a string! Make sure that numbers and fancy strings are quoted.] + +🤔 Why this weird `{name: xxx, value: yyy}` scheme? It will be revealed soon! + +--- + +## The downward API + +- In the previous example, environment variables have fixed values + +- We can also use a mechanism called the *downward API* + +- The downward API allows to expose pod or container information + + - either through special files (we won't show that for now) + + - or through environment variables + +- The value of these environment variables is computed when the container is started + +- Remember: environment variables won't (can't) change after container start + +- Let's see a few concrete examples! + +--- + +## Exposing the pod's namespace + +```yaml + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +``` + +- Useful to generate FQDN of services + + (in some contexts, a short name is not enough) + +- For instance, the two commands should be equivalent: + ``` + curl api-backend + curl api-backend.$MY_POD_NAMESPACE.svc.cluster.local + ``` + +--- + +## Exposing the pod's IP address + +```yaml + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP +``` + +- Useful if we need to know our IP address + + (we could also read it from `eth0`, but this is more solid) + +--- + +## Exposing the container's resource limits + +```yaml + - name: MY_MEM_LIMIT + valueFrom: + resourceFieldRef: + containerName: test-container + resource: limits.memory +``` + +- Useful for runtimes where memory is garbage collected + +- Example: the JVM + + (the memory available to the JVM should be set with the `-Xmx ` flag) + +- Best practice: set a memory limit, and pass it to the runtime + + (see [this blog post](https://very-serio.us/2017/12/05/running-jvms-in-kubernetes/) for a detailed example) + +--- + +## More about the downward API + +- [This documentation page](https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/) tells more about these environment variables + +- And [this one](https://kubernetes.io/docs/tasks/inject-data-application/downward-api-volume-expose-pod-information/) explains the other way to use the downward API + + (through files that get created in the container filesystem) + +--- + +## Environment variables, pros and cons + +- Works great when the running program expects these variables + +- Works great for optional parameters with reasonable defaults + + (since the container image can provide these defaults) + +- Sort of auto-documented + + (we can see which environment variables are defined in the image, and their values) + +- Can be (ab)used with longer values ... + +- ... You *can* put an entire Tomcat configuration file in an environment ... + +- ... But *should* you? + +(Do it if you really need to, we're not judging! But we'll see better ways.) + +--- + +## Injecting configuration files + +- Sometimes, there is no way around it: we need to inject a full config file + +- Kubernetes provides a mechanism for that purpose: `configmaps` + +- A configmap is a Kubernetes resource that exists in a namespace + +- Conceptually, it's a key/value map + + (values are arbitrary strings) + +- We can think about them in (at least) two different ways: + + - as holding entire configuration file(s) + + - as holding individual configuration parameters + +*Note: to hold sensitive information, we can use "Secrets", which +are another type of resource behaving very much like configmaps. +We'll cover them just after!* + +--- + +## Configmaps storing entire files + +- In this case, each key/value pair corresponds to a configuration file + +- Key = name of the file + +- Value = content of the file + +- There can be one key/value pair, or as many as necessary + + (for complex apps with multiple configuration files) + +- Examples: + ``` + # Create a configmap with a single key, "app.conf" + kubectl create configmap my-app-config --from-file=app.conf + # Create a configmap with a single key, "app.conf" but another file + kubectl create configmap my-app-config --from-file=app.conf=app-prod.conf + # Create a configmap with multiple keys (one per file in the config.d directory) + kubectl create configmap my-app-config --from-file=config.d/ + ``` + +--- + +## Configmaps storing individual parameters + +- In this case, each key/value pair corresponds to a parameter + +- Key = name of the parameter + +- Value = value of the parameter + +- Examples: + ``` + # Create a configmap with two keys + kubectl create cm my-app-config \ + --from-literal=foreground=red \ + --from-literal=background=blue + + # Create a configmap from a file containing key=val pairs + kubectl create cm my-app-config \ + --from-env-file=app.conf + ``` + +--- + +## Exposing configmaps to containers + +- Configmaps can be exposed as plain files in the filesystem of a container + + - this is achieved by declaring a volume and mounting it in the container + + - this is particularly effective for configmaps containing whole files + +- Configmaps can be exposed as environment variables in the container + + - this is achieved with the downward API + + - this is particularly effective for configmaps containing individual parameters + +- Let's see how to do both! + +--- + +## Passing a configuration file with a configmap + +- We will start a load balancer powered by HAProxy + +- We will use the [official `haproxy` image](https://hub.docker.com/_/haproxy/) + +- It expects to find its configuration in `/usr/local/etc/haproxy/haproxy.cfg` + +- We will provide a simple HAproxy configuration, `k8s/haproxy.cfg` + +- It listens on port 80, and load balances connections between Google and Bing + +--- + +## Creating the configmap + +.exercise[ + +- Create a configmap named `haproxy` and holding the configuration file: + ```bash + kubectl create configmap haproxy --from-file=~/container.training/k8s/haproxy.cfg + ``` + +- Check what our configmap looks like: + ```bash + kuebectl get configmap haproxy -o yaml + ``` + +] + +--- + +## Using the configmap + +We are going to use the following pod definition: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: haproxy +spec: + volumes: + - name: config + configMap: + name: haproxy + containers: + - name: haproxy + image: haproxy + volumeMounts: + - name: config + mountPath: /usr/local/etc/haproxy/ +``` + +--- + +## Using the configmap + +- The resource definition from the previous slide is in `k8s/haproxy.yaml` + +.exercise[ + +- Create the HAProxy pod: + ```bash + kubectl apply -f ~/container.training/k8s/haproxy.yaml + ``` + +- Check the IP address allocated to the pod: + ```bash + kubectl get pod haproxy -o wide + IP=$(kubectl get pod haproxy -o json | jq -r .status.podIP) + ``` + +] + +--- + +## Testing our load balancer + +- The load balancer will send: + + - half of the connections to Google + + - the other half to Bing + +.exercise[ + +- Access the load balancer a few times: + ```bash + curl -I $IP + curl -I $IP + curl -I $IP + ``` + +] + +We should see connections served by Google (look for the `Location` header) and others served by Bing (indicated by the `X-MSEdge-Ref` header). + +--- + +## Exposing configmaps with the downward API + +- We are going to run a Docker registry on a custom port + +- By default, the registry listens on port 5000 + +- This can be changed by setting environment variable `REGISTRY_HTTP_ADDR` + +- We are going to store the port number in a configmap + +- Then we will expose that configmap to a container environment variable + +--- + +## Creating the configmap + +.exercise[ + +- Our configmap will have a single key, `http.addr`: + ```bash + kubectl create configmap registry --from-literal=http.addr=0.0.0.0:80 + ``` + +- Check our configmap: + ```bash + kubectl get configmap regsitry -o yaml + ``` + +] + +--- + +## Using the configmap + +We are going to use the following pod definition: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: registry +spec: + containers: + - name: registry + image: registry + env: + - name: REGISTRY_HTTP_ADDR + valueFrom: + configMapKeyRef: + name: registry + key: http.addr +``` + +--- + +## Using the configmap + +- The resource definition from the previous slide is in `k8s/registry.yaml` + +.exercise[ + +- Create the registry pod: + ```bash + kubectl apply -f ~/container.training/k8s/registry.yaml + ``` + +- Check the IP address allocated to the pod: + ```bash + kubectl get pod registry -o wide + IP=$(kubectl get pod registry -o json | jq -r .status.podIP) + ``` + +- Confirm that the registry is available on port 80: + ```bash + curl $IP/v2/_catalog + ``` + +] + +--- + +## Passwords, tokens, sensitive information + +- For sensitive information, there is another special resource: *Secrets* + +- Secrets and Configmaps work almost the same way + + (we'll expose the differences on the next slide) + +- The *intent* is different, though: + + *"You should use secrets for things which are actually secret like API keys, + credentials, etc., and use config map for not-secret configuration data."* + + *"In the future there will likely be some differentiators for secrets like rotation or support for backing the secret API w/ HSMs, etc."* + + (Source: [the author of both features](https://stackoverflow.com/a/36925553/580281 +)) + +--- + +## Differences between configmaps and secrets + +- Secrets are base64-encoded when shown with `kubectl get secrets -o yaml` + + - keep in mind that this is just *encoding*, not *encryption* + + - it is very easy to [automatically extract and decode secrets](https://medium.com/@mveritym/decoding-kubernetes-secrets-60deed7a96a3) + +- [Secrets can be encrypted at rest](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/) + +- With RBAC, we can authorize a user to access configmaps, but not secrets + + (since they are two different kinds of resources) diff --git a/slides/k8s/gitworkflows.md b/slides/k8s/gitworkflows.md new file mode 100644 index 00000000..6a620447 --- /dev/null +++ b/slides/k8s/gitworkflows.md @@ -0,0 +1,239 @@ +# Git-based workflows + +- Deploying with `kubectl` has downsides: + + - we don't know *who* deployed *what* and *when* + + - there is no audit trail (except the API server logs) + + - there is no easy way to undo most operations + + - there is no review/approval process (like for code reviews) + +- We have all these things for *code*, though + +- Can we manage cluster state like we manage our source code? + +--- + +## Reminder: Kubernetes is *declarative* + +- All we do is create/change resources + +- These resources have a perfect YAML representation + +- All we do is manipulating these YAML representations + + (`kubectl run` generates a YAML file that gets applied) + +- We can store these YAML representations in a code repository + +- We can version that code repository and maintain it with best practices + + - define which branch(es) can go to qa/staging/production + + - control who can push to which branches + + - have formal review processes, pull requests ... + +--- + +## Enabling git-based workflows + +- There are a few tools out there to help us do that + +- We'll see demos of two of them: [Flux] and [Gitkube] + +- There are *many* other tools, some of them with even more features + +- There are also *many* integrations with popular CI/CD systems + + (e.g.: GitLab, Jenkins, ...) + +[Flux]: https://www.weave.works/oss/flux/ +[Gitkube]: https://gitkube.sh/ + +--- + +## Flux overview + +- We put our Kubernetes resources as YAML files in a git repository + +- Flux polls that repository regularly (every 5 minutes by default) + +- The resources described by the YAML files are created/updated automatically + +- Changes are made by updating the code in the repository + +--- + +## Preparing a repository for Flux + +- We need a repository with Kubernetes YAML files + +- I have one: https://github.com/jpetazzo/kubercoins + +- Fork it to your GitHub account + +- Create a new branch in your fork; e.g. `prod` + + (e.g. by adding a line in the README through the GitHub web UI) + +- This is the branch that we are going to use for deployment + +--- + +## Setting up Flux + +- Clone the Flux repository: + ``` + git clone https://github.com/weaveworks/flux + ``` + +- Edit `deploy/flux-deployment.yaml` + +- Change the `--git-url` and `--git-branch` parameters: + ```yaml + - --git-url=git@github.com:your-git-username/kubercoins + - --git-branch=prod + ``` + +- Apply all the YAML: + ``` + kubectl apply -f deploy/ + ``` + +--- + +## Allowing Flux to access the repository + +- When it starts, Flux generates an SSH key + +- Display that key: + ``` + kubectl get logs deployment flux | grep identity + ``` + +- Then add that key to the repository, giving it **write** access + + (some Flux features require write access) + +- After a minute or so, DockerCoins will be deployed to the current namespace + +--- + +## Making changes + +- Make changes (on the `prod` branch), e.g. change `replicas` in `worker` + +- After a few minutes, the changes will be picked up by Flux and applied + +--- + +## Other features + +- Flux can keep a list of all the tags of all the images we're running + +- The `fluxctl` tool can show us if we're running the latest images + +- We can also "automate" a resource (i.e. automatically deploy new images) + +- And much more! + +--- + +## Gitkube overview + +- We put our Kubernetes resources as YAML files in a git repository + +- Gitkube is a git server (or "git remote") + +- After making changes to the repository, we push to Gitkube + +- Gitkube applies the resources to the cluster + +--- + +## Setting up Gitkube + +- Install the CLI: + ``` + sudo curl -L -o /usr/local/bin/gitkube \ + https://github.com/hasura/gitkube/releases/download/v0.2.1/gitkube_linux_amd64 + sudo chmod +x /usr/local/bin/gitkube + ``` + +- Install Gitkube on the cluster: + ``` + gitkube install --expose ClusterIP + ``` + +--- + +## Creating a Remote + +- Gitkube provides a new type of API resource: *Remote* + + (this is using a mechanism called Custom Resource Definitions or CRD) + +- Create and apply a YAML file containing the following manifest: + ```yaml + apiVersion: gitkube.sh/v1alpha1 + kind: Remote + metadata: + name: example + spec: + authorizedKeys: + - `ssh-rsa AAA...` + manifests: + path: "." + ``` + + (replace the `ssh-rsa AAA...` section with the content of `~/.ssh/id_rsa.pub`) + +--- + +## Pushing to our remote + +- Get the `gitkubed` IP address: + ``` + kubectl -n kube-system get svc gitkubed + IP=$(kubectl -n kube-system get svc gitkubed -o json | + jq -r .spec.clusterIP) + ``` + +- Get ourselves a sample repository with resource YAML files: + ``` + git clone git://github.com/jpetazzo/kubercoins + cd kubercoins + ``` + +- Add the remote and push to it: + ``` + git remote add k8s ssh://default-example@$IP/~/git/default-example + git push k8s master + ``` + +--- + +## Making changes + +- Edit a local file + +- Commit + +- Push! + +- Make sure that you push to the `k8s` remote + +--- + +## Other features + +- Gitkube can also build container images for us + + (see the [documentation](https://github.com/hasura/gitkube/blob/master/docs/remote.md) for more details) + +- Gitkube can also deploy Helm Charts + + (instead of raw YAML files) diff --git a/slides/k8s/healthchecks.md b/slides/k8s/healthchecks.md new file mode 100644 index 00000000..2563da68 --- /dev/null +++ b/slides/k8s/healthchecks.md @@ -0,0 +1,178 @@ +# Healthchecks + +- Kubernetes provides two kinds of healthchecks: liveness and readiness + +- Healthchecks are *probes* that apply to *containers* (not to pods) + +- Each container can have two (optional) probes: + + - liveness = is this container dead or alive? + + - readiness = is this container ready to serve traffic? + +- Different probes are available (HTTP, TCP, program execution) + +- Let's see the difference and how to use them! + +--- + +## Liveness probe + +- Indicates if the container is dead or alive + +- A dead container cannot come back to life + +- If the liveness probe fails, the container is killed + + (to make really sure that it's really dead; no zombies or undeads!) + +- What happens next depends on the pod's `restartPolicy`: + + - `Never`: the container is not restarted + + - `OnFailure` or `Always`: the container is restarted + +--- + +## When to use a liveness probe + +- To indicate failures that can't be recovered + + - deadlocks (causing all requests to time out) + + - internal corruption (causing all requests to error) + +- If the liveness probe fails *N* consecutive times, the container is killed + +- *N* is the `failureThreshold` (3 by default) + +--- + +## Readiness probe + +- Indicates if the container is ready to serve traffic + +- If a container becomes "unready" (let's say busy!) it might be ready again soon + +- If the readiness probe fails: + + - the container is *not* killed + + - if the pod is a member of a service, it is temporarily removed + + - it is re-added as soon as the readiness probe passes again + +--- + +## When to use a readiness probe + +- To indicate temporary failures + + - the application can only service *N* parallel connections + + - the runtime is busy doing garbage collection or initial data load + +- The container is marked as "not ready" after `failureThreshold` failed attempts + + (3 by default) + +- It is marked again as "ready" after `successThreshold` successful attempts + + (1 by default) + +--- + +## Different types of probes + +- HTTP request + + - specify URL of the request (and optional headers) + + - any status code between 200 and 399 indicates success + +- TCP connection + + - the probe succeeds if the TCP port is open + +- arbitrary exec + + - a command is executed in the container + + - exit status of zero indicates success + +--- + +## Benefits of using probes + +- Rolling updates proceed when containers are *actually ready* + + (as opposed to merely started) + +- Containers in a broken state gets killed and restarted + + (instead of serving errors or timeouts) + +- Overloaded backends get removed from load balancer rotation + + (thus improving response times across the board) + +--- + +## Example: HTTP probe + +Here is a pod template for the `rng` web service of the DockerCoins app: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: rng-with-liveness +spec: + containers: + - name: rng + image: dockercoins/rng:v0.1 + livenessProbe: + httpGet: + path: / + port: 80 + initialDelaySeconds: 10 + periodSeconds: 1 +``` + +If the backend serves an error, or takes longer than 1s, 3 times in a row, it gets killed. + +--- + +## Example: exec probe + +Here is a pod template for a Redis server: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: redis-with-liveness +spec: + containers: + - name: redis + image: redis + livenessProbe: + exec: + command: ["redis-cli", "ping"] +``` + +If the Redis process becomes unresponsive, it will be killed. + +--- + +## Details about liveness and readiness probes + +- Probes are executed at intervals of `periodSeconds` (default: 10) + +- The timeout for a probe is set with `timeoutSeconds` (default: 1) + +- A probe is considered successful after `successThreshold` successes (default: 1) + +- A probe is considered failing after `failureThreshold` failures (default: 3) + +- If a probe is not defined, it's as if there was an "always successful" probe diff --git a/slides/k8s/ingress.md b/slides/k8s/ingress.md new file mode 100644 index 00000000..b7113e42 --- /dev/null +++ b/slides/k8s/ingress.md @@ -0,0 +1,524 @@ +# Exposing HTTP services with Ingress resources + +- *Services* give us a way to access a pod or a set of pods + +- Services can be exposed to the outside world: + + - with type `NodePort` (on a port >30000) + + - with type `LoadBalancer` (allocating an external load balancer) + +- What about HTTP services? + + - how can we expose `webui`, `rng`, `hasher`? + + - the Kubernetes dashboard? + + - a new version of `webui`? + +--- + +## Exposing HTTP services + +- If we use `NodePort` services, clients have to specify port numbers + + (i.e. http://xxxxx:31234 instead of just http://xxxxx) + +- `LoadBalancer` services are nice, but: + + - they are not available in all environments + + - they often carry an additional cost (e.g. they provision an ELB) + + - they require one extra step for DNS integration +
+ (waiting for the `LoadBalancer` to be provisioned; then adding it to DNS) + +- We could build our own reverse proxy + +--- + +## Building a custom reverse proxy + +- There are many options available: + + Apache, HAProxy, Hipache, NGINX, Traefik, ... + + (look at [jpetazzo/aiguillage](https://github.com/jpetazzo/aiguillage) for a minimal reverse proxy configuration using NGINX) + +- Most of these options require us to update/edit configuration files after each change + +- Some of them can pick up virtual hosts and backends from a configuration store + +- Wouldn't it be nice if this configuration could be managed with the Kubernetes API? + +-- + +- Enter.red[¹] *Ingress* resources! + +.footnote[.red[¹] Pun maybe intended.] + +--- + +## Ingress resources + +- Kubernetes API resource (`kubectl get ingress`/`ingresses`/`ing`) + +- Designed to expose HTTP services + +- Basic features: + + - load balancing + - SSL termination + - name-based virtual hosting + +- Can also route to different services depending on: + + - URI path (e.g. `/api`→`api-service`, `/static`→`assets-service`) + - Client headers, including cookies (for A/B testing, canary deployment...) + - and more! + +--- + +## Principle of operation + +- Step 1: deploy an *ingress controller* + + - ingress controller = load balancer + control loop + + - the control loop watches over ingress resources, and configures the LB accordingly + +- Step 2: setup DNS + + - associate DNS entries with the load balancer address + +- Step 3: create *ingress resources* + + - the ingress controller picks up these resources and configures the LB + +- Step 4: profit! + +--- + +## Ingress in action + +- We will deploy the Traefik ingress controller + + - this is an arbitrary choice + + - maybe motivated by the fact that Traefik releases are named after cheeses + +- For DNS, we will use [nip.io](http://nip.io/) + + - `*.1.2.3.4.nip.io` resolves to `1.2.3.4` + +- We will create ingress resources for various HTTP services + +--- + +## Deploying pods listening on port 80 + +- We want our ingress load balancer to be available on port 80 + +- We could do that with a `LoadBalancer` service + + ... but it requires support from the underlying infrastructure + +- We could use pods specifying `hostPort: 80` + + ... but with most CNI plugins, this [doesn't work or require additional setup](https://github.com/kubernetes/kubernetes/issues/23920) + +- We could use a `NodePort` service + + ... but that requires [changing the `--service-node-port-range` flag in the API server](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/) + +- Last resort: the `hostNetwork` mode + +--- + +## Without `hostNetwork` + +- Normally, each pod gets its own *network namespace* + + (sometimes called sandbox or network sandbox) + +- An IP address is associated to the pod + +- This IP address is routed/connected to the cluster network + +- All containers of that pod are sharing that network namespace + + (and therefore using the same IP address) + +--- + +## With `hostNetwork: true` + +- No network namespace gets created + +- The pod is using the network namespace of the host + +- It "sees" (and can use) the interfaces (and IP addresses) of the host + +- The pod can receive outside traffic directly, on any port + +- Downside: with most network plugins, network policies won't work for that pod + + - most network policies work at the IP address level + + - filtering that pod = filtering traffic from the node + +--- + +## Running Traefik + +- The [Traefik documentation](https://docs.traefik.io/user-guide/kubernetes/#deploy-trfik-using-a-deployment-or-daemonset) tells us to pick between Deployment and Daemon Set + +- We are going to use a Daemon Set so that each node can accept connections + +- We will do two minor changes to the [YAML provided by Traefik](https://github.com/containous/traefik/blob/master/examples/k8s/traefik-ds.yaml): + + - enable `hostNetwork` + + - add a *toleration* so that Traefik also runs on `node1` + +--- + +## Taints and tolerations + +- A *taint* is an attribute added to a node + +- It prevents pods from running on the node + +- ... Unless they have a matching *toleration* + +- When deploying with `kubeadm`: + + - a taint is placed on the node dedicated the control plane + + - the pods running the control plane have a matching toleration + +--- + +class: extra-details + +## Checking taints on our nodes + +.exercise[ + +- Check our nodes specs: + ```bash + kubectl get node node1 -o json | jq .spec + kubectl get node node2 -o json | jq .spec + ``` + +] + +We should see a result only for `node1` (the one with the control plane): + +```json + "taints": [ + { + "effect": "NoSchedule", + "key": "node-role.kubernetes.io/master" + } + ] +``` + +--- + +class: extra-details + +## Understanding a taint + +- The `key` can be interpreted as: + + - a reservation for a special set of pods +
+ (here, this means "this node is reserved for the control plane") + + - an error condition on the node +
+ (for instance: "disk full", do not start new pods here!) + +- The `effect` can be: + + - `NoSchedule` (don't run new pods here) + + - `PreferNoSchedule` (try not to run new pods here) + + - `NoExecute` (don't run new pods and evict running pods) + +--- + +class: extra-details + +## Checking tolerations on the control plane + +.exercise[ + +- Check tolerations for CoreDNS: + ```bash + kubectl -n kube-system get deployments coredns -o json | + jq .spec.template.spec.tolerations + ``` + +] + +The result should include: +```json + { + "effect": "NoSchedule", + "key": "node-role.kubernetes.io/master" + } +``` + +It means: "bypass the exact taint that we saw earlier on `node1`." + +--- + +class: extra-details + +## Special tolerations + +.exercise[ + +- Check tolerations on `kube-proxy`: + ```bash + kubectl -n kube-system get ds kube-proxy -o json | + jq .spec.template.spec.tolerations + ``` + +] + +The result should include: +```json + { + "operator": "Exists" + } +``` + +This one is a special case that means "ignore all taints and run anyway." + +--- + +## Running Traefik on our cluster + +- We provide a YAML file (`k8s/traefik.yaml`) which is essentially the sum of: + + - [Traefik's Daemon Set resources](https://github.com/containous/traefik/blob/master/examples/k8s/traefik-ds.yaml) (patched with `hostNetwork` and tolerations) + + - [Traefik's RBAC rules](https://github.com/containous/traefik/blob/master/examples/k8s/traefik-rbac.yaml) allowing it to watch necessary API objects + +.exercise[ + +- Apply the YAML: + ```bash + kubectl apply -f ~/container.training/k8s/traefik.yaml + ``` + +] + +--- + +## Checking that Traefik runs correctly + +- If Traefik started correctly, we now have a web server listening on each node + +.exercise[ + +- Check that Traefik is serving 80/tcp: + ```bash + curl localhost + ``` + +] + +We should get a `404 page not found` error. + +This is normal: we haven't provided any ingress rule yet. + +--- + +## Setting up DNS + +- To make our lives easier, we will use [nip.io](http://nip.io) + +- Check out `http://cheddar.A.B.C.D.mip.io` + + (replacing A.B.C.D with the IP address of `node1`) + +- We should get the same `404 page not found` error + + (meaning that our DNS is "set up properly", so to speak!) + +--- + +## Traefik web UI + +- Traefik provides a web dashboard + +- With the current install method, it's listening on port 8080 + +.exercise[ + +- Go to `http://node1:8080` (replacing `node1` with its IP address) + +] + +--- + +## Setting up host-based routing ingress rules + +- We are going to use `errm/cheese` images + + (there are [3 tags available](https://hub.docker.com/r/errm/cheese/tags/): wensleydale, cheddar, stilton) + +- These images contain a simple static HTTP server sending a picture of cheese + +- We will run 3 deployments (one for each cheese) + +- We will create 3 services (one for each deployment) + +- Then we will create 3 ingress rules (one for each service) + +- We will route `.A.B.C.D.nip.io` to the corresponding deployment + +--- + +## Running cheesy web servers + +.exercise[ + +- Run all three deployments: + ```bash + kubectl run cheddar --image=errm/cheese:cheddar + kubectl run stilton --image=errm/cheese:stilton + kubectl run wensleydale --image=errm/cheese:wensleydale + ``` + +- Create a service for each of them: + ```bash + kubectl expose deployment cheddar --port=80 + kubectl expose deployment stilton --port=80 + kubectl expose deployment wensleydale --port=80 + ``` + +] + +--- + +## What does an ingress resource look like? + +Here is a minimal host-based ingress resource: + +```yaml +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: cheddar +spec: + rules: + - host: cheddar.`A.B.C.D`.nip.io + http: + paths: + - path: / + backend: + serviceName: cheddar + servicePort: 80 + +``` + +(It is in `k8s/ingress.yaml`.) + +--- + +## Creating our first ingress resources + +.exercise[ + +- Edit the file `~/container.training/k8s/ingress.yaml` + +- Replace A.B.C.D with the IP address of `node1` + +- Apply the file + +- Open http://cheddar.A.B.C.D.nip.io + +] + +(An image of a piece of cheese should show up.) + +--- + +## Creating the other ingress resources + +.exercise[ + +- Edit the file `~/container.training/k8s/ingress.yaml` + +- Replace `cheddar` with `stilton` (in `name`, `host`, `serviceName`) + +- Apply the file + +- Check that `stilton.A.B.C.D.nip.io` works correctly + +- Repeat for `wensleydale` + +] + +--- + +## Using multiple ingress controllers + +- You can have multiple ingress controllers active simultaneously + + (e.g. Traefik and NGINX) + +- You can even have multiple instances of the same controller + + (e.g. one for internal, another for external traffic) + +- The `kubernetes.io/ingress.class` annotation can be used to tell which one to use + +- It's OK if multiple ingress controllers configure the same resource + + (it just means that the service will be accessible through multiple paths) + +--- + +## Ingress: the good + +- The traffic flows directly from the ingress load balancer to the backends + + - it doesn't need to go through the `ClusterIP` + + - in fact, we don't even need a `ClusterIP` (we can use a headless service) + +- The load balancer can be outside of Kubernetes + + (as long as it has access to the cluster subnet) + +- This allows to use external (hardware, physical machines...) load balancers + +- Annotations can encode special features + + (rate-limiting, A/B testing, session stickiness, etc.) + +--- + +## Ingress: the bad + +- Aforementioned "special features" are not standardized yet + +- Some controllers will support them; some won't + +- Even relatively common features (stripping a path prefix) can differ: + + - [traefik.ingress.kubernetes.io/rule-type: PathPrefixStrip](https://docs.traefik.io/user-guide/kubernetes/#path-based-routing) + + - [ingress.kubernetes.io/rewrite-target: /](https://github.com/kubernetes/contrib/tree/master/ingress/controllers/nginx/examples/rewrite) + +- This should eventually stabilize + + (remember that ingresses are currently `apiVersion: extensions/v1beta1`) diff --git a/slides/k8s/owners-and-dependents.md b/slides/k8s/owners-and-dependents.md new file mode 100644 index 00000000..7a7c2fc2 --- /dev/null +++ b/slides/k8s/owners-and-dependents.md @@ -0,0 +1,177 @@ +# Owners and dependents + +- Some objects are created by other objects + + (example: pods created by replica sets, themselves created by deployments) + +- When an *owner* object is deleted, its *dependents* are deleted + + (this is the default behavior; it can be changed) + +- We can delete a dependent directly if we want + + (but generally, the owner will recreate another right away) + +- An object can have multiple owners + +--- + +## Finding out the owners of an object + +- The owners are recorded in the field `ownerReferences` in the `metadata` block + +.exercise[ + +- Let's start a replicated `nginx` deployment: + ```bash + kubectl run yanginx --image=nginx --replicas=3 + ``` + +- Once it's up, check the corresponding pods: + ```bash + kuebectl get pods -l run=yanginx -o yaml | head -n 25 + ``` + +] + +These pods are owned by a ReplicaSet named yanginx-xxxxxxxxxx. + +--- + +## Listing objects with their owners + +- This is a good opportunity to try the `custom-columns` output! + +.exercise[ + +- Show all pods with their owners: + ```bash + kubectl get pod -o custom-columns=\ + NAME:.metadata.name,\ + OWNER-KIND:.metadata.ownerReferences[0].kind,\ + OWNER-NAME:.metadata.ownerReferences[0].name + ``` + +] + +Note: the `custom-columns` option should be one long option (without spaces), +so the lines should not be indented (otherwise the indentation will insert spaces). + +--- + +## Deletion policy + +- When deleting an object through the API, three policies are available: + + - foreground (API call returns after all dependents are deleted) + + - background (API call returns immediately; dependents are scheduled for deletion) + + - orphan (the dependents are not deleted) + +- When deleting an object with `kubectl`, this is selected with `--cascade`: + + - `--cascade=true` deletes all dependent objects (default) + + - `--cascade=false` orphans dependent objects + +--- + +## What happens when an object is deleted + +- It is removed from the list of owners of its dependents + +- If, for one of these dependents, the list of owners becomes empty ... + + - if the policy is "orphan", the object stays + + - otherwise, the object is deleted + +--- + +## Orphaning pods + +- We are going to delete the Deployment and Replica Set that we created + +- ... without deleting the corresponding pods! + +.exercise[ + +- Delete the Deployment: + ```bash + kubectl delete deployment -l run=yanginx --cascade=false + ``` + +- Delete the Replica Set: + ```bash + kubectl delete replicaset -l run=yanginx --cascade=false + ``` + +- Check that the pods are still here: + ```bash + kubectl get pods + ``` + +] + +--- + +class: extra-details + +## When and why would we have orphans? + +- If we remove an owner and explicitly instruct the API to orphan dependents + + (like on the previous slide) + +- If we change the labels on a dependent, so that it's not selected anymore + + (e.g. change the `run: yanginx` in the pods of the previous example) + +- If a deployment tool that we're using does these things for us + +- If there is a serious problem within API machinery or other components + + (i.e. "this should not happen") + +--- + +## Finding orphan objects + +- We're going to output all pods in JSON format + +- Then we will use `jq` to keep only the ones *without* an owner + +- And we will display their name + +.exercise[ + +- List all pods that *do not* have an owner: + ```bash + kubectl get pod -o json | jq -r " + .items[] + | select(.metadata.ownerReferences|not) + | .metadata.name" + ``` + +] + +--- + +## Deleting orphan pods + +- Now that we can list orphan pods, deleting them is easy + +.exercise[ + +- Add `| xargs kubectl delete pod` to the previous command: + ```bash + kubectl get pod -o json | jq -r " + .items[] + | select(.metadata.ownerReferences|not) + | .metadata.name" | xargs kubectl delete pod + ``` + +] + +As always, the [documentation](https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/) has useful extra information and pointers. \ No newline at end of file diff --git a/slides/k8s/portworx.md b/slides/k8s/portworx.md new file mode 100644 index 00000000..001ea848 --- /dev/null +++ b/slides/k8s/portworx.md @@ -0,0 +1,638 @@ +# Highly available Persistent Volumes + +- How can we achieve true durability? + +- How can we store data that would survive the loss of a node? + +-- + +- We need to use Persistent Volumes backed by highly available storage systems + +- There are many ways to achieve that: + + - leveraging our cloud's storage APIs + + - using NAS/SAN systems or file servers + + - distributed storage systems + +-- + +- We are going to see one distributed storage system in action + +--- + +## Our test scenario + +- We will set up a distributed storage system on our cluster + +- We will use it to deploy a SQL database (PostgreSQL) + +- We will insert some test data in the database + +- We will disrupt the node running the database + +- We will see how it recovers + +--- + +## Portworx + +- Portworx is a *commercial* persistent storage solution for containers + +- It works with Kubernetes, but also Mesos, Swarm ... + +- It provides [hyper-converged](https://en.wikipedia.org/wiki/Hyper-converged_infrastructure) storage + + (=storage is provided by regular compute nodes) + +- We're going to use it here because it can be deployed on any Kubernetes cluster + + (it doesn't require any particular infrastructure) + +- We don't endorse or support Portworx in any particular way + + (but we appreciate that it's super easy to install!) + +--- + +## A useful reminder + +- We're installing Portworx because we need a storage system + +- If you are using AKS, EKS, GKE ... you already have a storage system + + (but you might want another one, e.g. to leverage local storage) + +- If you have setup Kubernetes yourself, there are other solutions available too + + - on premises, you can use a good old SAN/NAS + + - on a private cloud like OpenStack, you can use e.g. Cinder + + - everywhere, you can use other systems, e.g. Gluster, StorageOS + +--- + +## Portworx requirements + +- Kubernetes cluster ✔️ + +- Optional key/value store (etcd or Consul) ❌ + +- At least one available block device ❌ + +--- + +## The key-value store + +- In the current version of Portworx (1.4) it is recommended to use etcd or Consul + +- But Portworx also has beta support for an embedded key/value store + +- For simplicity, we are going to use the latter option + + (but if we have deployed Consul or etcd, we can use that, too) + +--- + +## One available block device + +- Block device = disk or partition on a disk + +- We can see block devices with `lsblk` + + (or `cat /proc/partitions` if we're old school like that!) + +- If we don't have a spare disk or partition, we can use a *loop device* + +- A loop device is a block device actually backed by a file + +- These are frequently used to mount ISO (CD/DVD) images or VM disk images + +--- + +## Setting up a loop device + +- We are going to create a 10 GB (empty) file on each node + +- Then make a loop device from it, to be used by Portworx + +.exercise[ + +- Create a 10 GB file on each node: + ```bash + for N in $(seq 1 5); do ssh node$N sudo truncate --size 10G /portworx.blk; done + ``` + (If SSH asks to confirm host keys, enter `yes` each time.) + +- Associate the file to a loop device on each node: + ```bash + for N in $(seq 1 5); do ssh node$N sudo losetup /dev/loop0 /portworx.blk; done + ``` + +] + +--- + +## Installing Portworx + +- To install Portworx, we need to go to https://install.portworx.com/ + +- This website will ask us a bunch of questoins about our cluster + +- Then, it will generate a YAML file that we should apply to our cluster + +-- + +- Or, we can just apply that YAML file directly (it's in `k8s/portworx.yaml`) + +.exercise[ + +- Install Portworx: + ```bash + kubectl apply -f ~/container.training/k8s/portworx.yaml + ``` + +] + +--- + +class: extra-details + +## Generating a custom YAML file + +If you want to generate a YAML file tailored to your own needs, the easiest +way is to use https://install.portworx.com/. + +FYI, this is how we obtained the YAML file used earlier: +``` +KBVER=$(kubectl version -o json | jq -r .serverVersion.gitVersion) +BLKDEV=/dev/loop0 +curl https://install.portworx.com/1.4/?kbver=$KBVER&b=true&s=$BLKDEV&c=px-workshop&stork=true&lh=true +``` +If you want to use an external key/value store, add one of the following: +``` +&k=etcd://`XXX`:2379 +&k=consul://`XXX`:8500 +``` +... where `XXX` is the name or address of your etcd or Consul server. + +--- + +## Dynamic provisioning of persistent volumes + +- We are going to run PostgreSQL in a Stateful set + +- The Stateful set will specify a `volumeClaimTemplate` + +- That `volumeClaimTemplate` will create Persistent Volume Claims + +- Kubernetes' [dynamic provisioning](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/) will satisfy these Persistent Volume Claims + + (by creating Persistent Volumes and binding them to the claims) + +- The Persistent Volumes are then available for the PostgreSQL pods + +--- + +## Storage Classes + +- It's possible that multiple storage systems are available + +- Or, that a storage system offers multiple tiers of storage + + (SSD vs. magnetic; mirrored or not; etc.) + +- We need to tell Kubernetes *which* system and tier to use + +- This is achieved by creating a Storage Class + +- A `volumeClaimTemplate` can indicate which Storage Class to use + +- It is also possible to mark a Storage Class as "default" + + (it will be used if a `volumeClaimTemplate` doesn't specify one) + +--- + +## Our default Storage Class + +This is our Storage Class (in `k8s/storage-class.yaml`): + +```yaml +kind: StorageClass +apiVersion: storage.k8s.io/v1beta1 +metadata: + name: portworx-replicated + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: kubernetes.io/portworx-volume +parameters: + repl: "2" + priority_io: "high" +``` + +- It says "use Portworx to create volumes" + +- It tells Portworx to "keep 2 replicas of these volumes" + +- It marks the Storage Class as being the default one + +--- + +## Creating our Storage Class + +- Let's apply that YAML file! + +.exercise[ + +- Create the Storage Class: + ```bash + kubectl apply -f ~/container.training/k8s/storage-class.yaml + ``` + +- Check that it is now available: + ```bash + kubectl get sc + ``` + +] + +It should show as `portworx-replicated (default)`. + +--- + +## Our Postgres Stateful set + +- The next slide shows `k8s/postgres.yaml` + +- It defines a Stateful set + +- With a `volumeClaimTemplate` requesting a 1 GB volume + +- That volume will be mounted to `/var/lib/postgresql` + +- There is another little detail: we enable the `stork` scheduler + +- The `stork` scheduler is optional (it's specific to Portworx) + +- It helps the Kubernetes scheduler to colocate the pod with its volume + + (see [this blog post](https://portworx.com/stork-storage-orchestration-kubernetes/) for more details about that) + +--- + +.small[ +```yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres +spec: + selector: + matchLabels: + app: postgres + serviceName: postgres + template: + metadata: + labels: + app: postgres + spec: + schedulerName: stork + containers: + - name: postgres + image: postgres:10.5 + volumeMounts: + - mountPath: /var/lib/postgresql + name: postgres + volumeClaimTemplates: + - metadata: + name: postgres + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi +``` +] + +--- + +## Creating the Stateful set + +- Before applying the YAML, watch what's going on with `kubectl get events -w` + +.exercise[ + +- Apply that YAML: + ```bash + kubectl apply -f ~/container.training/k8s/postgres.yaml + ``` + +] + +--- + +## Testing our PostgreSQL pod + +- We will use `kubectl exec` to get a shell in the pod + +- Good to know: we need to use the `postgres` user in the pod + +.exercise[ + +- Get a shell in the pod, as the `postgres` user: + ```bash + kubectl exec -ti postgres-0 su postgres + ``` + +- Check that default databases have been created correctly: + ```bash + psql -l + ``` + +] + +(This should show us 3 lines: postgres, template0, and template1.) + +--- + +## Inserting data in PostgreSQL + +- We will create a database and populate it with `pgbench` + +.exercise[ + +- Create a database named `demo`: + ```bash + createdb demo + ``` + +- Populate it with `pgbench`: + ```bash + pgbench -i -s 10 demo + ``` + +] + +- The `-i` flag means "create tables" + +- The `-s 10` flag means "create 10 x 100,000 rows" + +--- + +## Checking how much data we have now + +- The `pgbench` tool inserts rows in table `pgbench_accounts` + +.exercise[ + +- Check that the `demo` base exists: + ```bash + psql -l + ``` + +- Check how many rows we have in `pgbench_accounts`: + ```bash + psql demo -c "select count(*) from pgbench_accounts" + ``` + +] + +(We should see a count of 1,000,000 rows.) + +--- + +## Find out which node is hosting the database + +- We can find that information with `kubectl get pods -o wide` + +.exercise[ + +- Check the node running the database: + ```bash + kuebectl get pod postgres-0 -o wide + ``` + +] + +We are going to disrupt that node. + +-- + +By "disrupt" we mean: "disconnect it from the network". + +--- + +## Disconnect the node + +- We will use `iptables` to block all traffic exiting the node + + (except SSH traffic, so we can repair the node later if needed) + +.exercise[ + +- SSH to the node to disrupt: + ```bash + ssh `nodeX` + ``` + +- Allow SSH traffic leaving the node, but block all other traffic: + ```bash + sudo iptables -I OUTPUT -p tcp --sport 22 -j ACCEPT + sudo iptables -I OUTPUT 2 -j DROP + ``` + +] + +--- + +## Check that the node is disconnected + +.exercise[ + +- Check that the node can't communicate with other nodes: + ```bash + ping -c 3 node1 + ``` + +- Logout to go back on `node1` + +- Watch the events unfolding with `kubectl get events -w` and `kubectl get pods -w` + +] + +- It will take some time for Kubernetes to mark the node as unhealthy + +- Then it will attempt to reschedule the pod to another node + +- In about a minute, our pod should be up and running again + +--- + +## Check that our data is still available + +- We are going to reconnect to the (new) pod and check + +.exercise[ + +- Get a shell on the pod: + ```bash + kubectl exec -ti postgres-0 su postgres + ``` + +- Check the number of rows in the `pgbench_accounts` table: + ```bash + psql demo -c "select count(*) from pgbench_accounts + ``` + +] + +--- + +## Double-check that the pod has really moved + +- Just to make sure the system is not bluffing! + +.exercise[ + +- Look at which node the pod is now running on + ```bash + kubectl get pod postgres-0 -o wide + ``` + +] + +--- + +## Re-enable the node + +- Let's fix the node that we disconnected from the network + +.exercise[ + +- SSH to the node: + ```bash + ssh `nodeX` + ``` + +- Remove the iptables rule blocking traffic: + ```bash + sudo iptables -D OUTPUT 2 + ``` + +] + +--- + +class: extra-details + +## A few words about this PostgreSQL setup + +- In a real deployment, you would want to set a password + +- This can be done by creating a `secret`: + ``` + kubectl create secret generic postgres \ + --from-literal=password=$(base64 /dev/urandom | head -c16) + ``` + +- And then passing that secret to the container: + ```yaml + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres + key: password + ``` + +--- + +class: extra-details + +## Troubleshooting Portworx + +- If we need to see what's going on with Portworx: + ``` + PXPOD=$(kubectl -n kube-system get pod -l name=portworx -o json | + jq -r .items[0].metadata.name) + kubectl -n kube-system exec $PXPOD -- /opt/pwx/bin/pxctl status + ``` + +- We can also connect to Lighthouse (a web UI) + + - check the port with `kubectl -n kube-system get svc px-lighthouse` + + - connect to that port + + - the default login/password is `admin/Password1` + + - then specify `portworx-service` as the endpoint + +--- + +class: extra-details + +## Removing Portworx + +- Portworx provides a storage driver + +- It needs to place itself "above" the Kubelet + + (it installs itself straight on the nodes) + +- To remove it, we need to do more than just deleting its Kubernetes resources + +- It is done by applying a special label: + ``` + kubectl label nodes --all px/enabled=remove --overwrite + ``` + +- Then removing a bunch of local files: + ``` + sudo chattr -i /etc/pwx/.private.json + sudo rm -rf /etc/pwx /opt/pwx + ``` + + (on each node where Portworx was running) + +--- + +class: extra-details + +## Dynamic provisioning without a provider + +- What if we want to use Stateful sets without a storage provider? + +- We will have to create volumes manually + + (by creating Persistent Volume objects) + +- These volumes will be automatically bound with matching Persistent Volume Claims + +- We can use local volumes (essentially bind mounts of host directories) + +- Of course, these volumes won't be available in case of node failure + +- Check [this blog post](https://kubernetes.io/blog/2018/04/13/local-persistent-volumes-beta/) for more information and gotchas + +--- + +## Acknowledgements + +The Portworx installation tutorial, and the PostgreSQL example, +were inspired by [Portworx examples on Katacoda](https://katacoda.com/portworx/scenarios/), in particular: + +- [installing Portworx on Kubernetes](https://www.katacoda.com/portworx/scenarios/deploy-px-k8s) + + (with adapatations to use a loop device and an embedded key/value store) + +- [persistent volumes on Kubernetes using Portworx](https://www.katacoda.com/portworx/scenarios/px-k8s-vol-basic) + + (with adapatations to specify a default Storage Class) + +- [HA PostgreSQL on Kubernetes with Portworx](https://www.katacoda.com/portworx/scenarios/px-k8s-postgres-all-in-one) + + (with adaptations to use a Stateful Set and simplify PostgreSQL's setup) diff --git a/slides/k8s/prometheus.md b/slides/k8s/prometheus.md new file mode 100644 index 00000000..7a3e12e1 --- /dev/null +++ b/slides/k8s/prometheus.md @@ -0,0 +1,486 @@ +# Collecting metrics with Prometheus + +- Prometheus is an open-source monitoring system including: + + - multiple *service discovery* backends to figure out which metrics to collect + + - a *scraper* to collect these metrics + + - an efficient *time series database* to store these metrics + + - a specific query language (PromQL) to query these time series + + - an *alert manager* to notify us according to metrics values or trends + +- We are going to deploy it on our Kubernetes cluster and see how to query it + +--- + +## Why Prometheus? + +- We don't endorse Prometheus more or less than any other system + +- It's relatively well integrated within the Cloud Native ecosystem + +- It can be self-hosted (this is useful for tutorials like this) + +- It can be used for deployments of varying complexity: + + - one binary and 10 lines of configuration to get started + + - all the way to thousands of nodes and millions of metrics + +--- + +## Exposing metrics to Prometheus + +- Prometheus obtains metrics and their values by querying *exporters* + +- An exporter serves metrics over HTTP, in plain text + +- This is was the *node exporter* looks like: + + http://demo.robustperception.io:9100/metrics + +- Prometheus itself exposes its own internal metrics, too: + + http://demo.robustperception.io:9090/metrics + +- If you want to expose custom metrics to Prometheus: + + - serve a text page like these, and you're good to go + + - libraries are available in various languages to help with quantiles etc. + +--- + +## How Prometheus gets these metrics + +- The *Prometheus server* will *scrape* URLs like these at regular intervals + + (by default: every minute; can be more/less frequent) + +- If you're worried about parsing overhead: exporters can also use protobuf + +- The list of URLs to scrape (the *scrape targets*) is defined in configuration + +--- + +## Defining scrape targets + +This is maybe the simplest configuration file for Prometheus: +```yaml +scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] +``` + +- In this configuration, Prometheus collects its own internal metrics + +- A typical configuration file will have multiple `scrape_configs` + +- In this configuration, the list of targets is fixed + +- A typical configuration file will use dynamic service discovery + +--- + +## Service discovery + +This configuration file will leverage existing DNS `A` records: +```yaml +scrape_configs: + - ... + - job_name: 'node' + dns_sd_configs: + - names: ['api-backends.dc-paris-2.enix.io'] + type: 'A' + port: 9100 +``` + +- In this configuration, Prometheus resolves the provided name(s) + + (here, `api-backends.dc-paris-2.enix.io`) + +- Each resulting IP address is added as a target on port 9100 + +--- + +## Dynamic service discovery + +- In the DNS example, the names are re-resolved at regular intervals + +- As DNS records are created/updated/removed, scrape targets change as well + +- Existing data (previously collected metrics) is not deleted + +- Other service discovery backends work in a similar fashion + +--- + +## Other service discovery mechanisms + +- Prometheus can connect to e.g. a cloud API to list instances + +- Or to the Kubernetes API to list nodes, pods, services ... + +- Or a service like Consul, Zookeeper, etcd, to list applications + +- The resulting configurations files are *way more complex* + + (but don't worry, we won't need to write them ourselves) + +--- + +## Time series database + +- We could wonder, "why do we need a specialized database?" + +- One metrics data point = metrics ID + timestamp + value + +- With a classic SQL or noSQL data store, that's at least 160 bits of data + indexes + +- Prometheus is way more efficient, without sacrificing performance + + (it will even be gentler on the I/O subsystem since it needs to write less) + +FIXME link to Goutham's talk + +--- + +## Running Prometheus on our cluster + +We need to: + +- Run the Prometheus server in a pod + + (using e.g. a Deployment to ensure that it keeps running) + +- Expose the Prometheus server web UI (e.g. with a NodePort) + +- Run the *node exporter* on each node (with a Daemon Set) + +- Setup a Service Account so that Prometheus can query the Kubernetes API + +- Configure the Prometheus server + + (storing the configuration in a Config Map for easy updates) + +--- + +## Helm Charts to the rescue + +- To make our lives easier, we are going to use a Helm Chart + +- The Helm Chart will take care of all the steps explained above + + (including some extra features that we don't need, but won't hurt) + +--- + +## Step 1: install Helm + +- If we already installed Helm earlier, these commands won't break anything + +.exercice[ + +- Install Tiller (Helm's server-side component) on our cluster: + ```bash + helm init + ``` + +- Give Tiller permission to deploy things on our cluster: + ```bash + kubectl create clusterrolebinding add-on-cluster-admin \ + --clusterrole=cluster-admin --serviceaccount=kube-system:default + ``` + +] + +--- + +## Step 2: install Prometheus + +- Skip this if we already installed Prometheus earlier + + (in doubt, check with `helm list`) + +.exercice[ + +- Install Prometheus on our cluster: + ```bash + helm install stable/prometheus \ + --set server.service.type=NodePort \ + --set server.persistentVolume.enabled=false + ``` + +] + +The provided flags: + +- expose the server web UI (and API) on a NodePort + +- use an ephemeral volume for metrics storage +
+ (instead of requesting a Persistent Volume through a Persistent Volume Claim) + +--- + +## Connecting to the Prometheus web UI + +- Let's connect to the web UI and see what we can do + +.exercise[ + +- Figure out the NodePort that was allocated to the Prometheus server: + ```bash + kubectl get svc prometheus-server + ``` + +- With your browser, connect to that port + +] + +--- + +## Querying some metrics + +- This is easy ... if you are familiar with PromQL + +.exercise[ + +- Click on "Graph", and in "expression", paste the following: + ``` + sum by (instance) ( + irate( + container_cpu_usage_seconds_total{ + pod_name=~"worker.*" + }[5m] + ) + ) + ``` + +] + +- Click on the blue "Execute" button and on the "Graph" tab just below + +- We see the cumulated CPU usage of worker pods for each node +
+ (if we just deployed Prometheus, there won't be much data to see, though) + +--- + +## Getting started with PromQL + +- We can't learn PromQL in just 5 minutes + +- But we can cover the basics to get an idea of what is possible + + (and have some keywords and pointers) + +- We are going to break down the query above + + (building it one step at a time) + +--- + +## Graphing one metric across all tags + +This query will show us CPU usage across all containers: +``` +container_cpu_usage_seconds_total +``` + +- The suffix of the metrics name tells us: + + - the unit (seconds of CPU) + + - that it's the total used since the container creation + +- Since it's a "total", it is an increasing quantity + + (we need to compute the derivative if we want e.g. CPU % over time) + +- We see that the metrics retrieved have *tags* attached to them + +--- + +## Selecting metrics with tags + +This query will show us only metrics for worker containers: +``` +container_cpu_usage_seconds_total{pod_name=~"worker.*"} +``` + +- The `=~` operator allows regex matching + +- We select all the pods with a name starting with `worker` + + (it would be better to use labels to select pods; more on that later) + +- The result is a smaller set of containers + +--- + +## Transforming counters in rates + +This query will show us CPU usage % instead of total seconds used: +``` +100*irate(container_cpu_usage_seconds_total{pod_name=~"worker.*"}[5m]) +``` + +- The [`irate`](https://prometheus.io/docs/prometheus/latest/querying/functions/#irate) operator computes the "per-second instant rate of increase" + + - `rate` is similar but allows decreasing counters and negative values + + - with `irate`, if a counter goes back to zero, we don't get a negative spike + +- The `[5m]` tells how far to look back if there is a gap in the data + +- And we multiply with `100*` to get CPU % usage + +--- + +## Aggregation operators + +This query sums the CPU usage per node: +``` +sum by (instance) ( + irate(container_cpu_usage_seconds_total{pod_name=~"worker.*"}[5m]) +) +``` + +- `instance` corresponds to the node on which the container is running + +- `sum by (instance) (...)` computes the sum for each instance + +- Note: all the other tags are collapsed + + (in other words, the resulting graph only shows the `instance` tag) + +- PromQL supports many more [aggregation operators](https://prometheus.io/docs/prometheus/latest/querying/operators/#aggregation-operators) + +--- + +## What kind of metrics can we collect? + +- Node metrics (related to physical or virtual machines) + +- Container metrics (resource usage per container) + +- Databases, message queues, load balancers, ... + + (check out this [list of exporters](https://prometheus.io/docs/instrumenting/exporters/)!) + +- Instrumentation (=deluxe `printf` for our code) + +- Business metrics (customers served, revenue, ...) + +--- + +class: extra-details + +## Node metrics + +- CPU, RAM, disk usage on the whole node + +- Total number of processes running, and their states + +- Number of open files, sockets, and their states + +- I/O activity (disk, network), per operation or volume + +- Physical/hardware (when applicable): temperature, fan speed ... + +- ... and much more! + +--- + +class: extra-details + +## Container metrics + +- Similar to node metrics, but not totally identical + +- RAM breakdown will be different + + - active vs inactive memory + - some memory is *shared* between containers, and accounted specially + +- I/O activity is also harder to track + + - async writes can cause deferred "charges" + - some page-ins are also shared between containers + +For details about container metrics, see: +
+http://jpetazzo.github.io/2013/10/08/docker-containers-metrics/ + +--- + +class: extra-details + +## Application metrics + +- Arbitrary metrics related to your application and business + +- System performance: request latency, error rate ... + +- Volume information: number of rows in database, message queue size ... + +- Business data: inventory, items sold, revenue ... + +--- + +class: extra-details + +## Detecting scrape targets + +- Prometheus can leverage Kubernetes service discovery + + (with proper configuration) + +- Services or pods can be annotated with: + + - `prometheus.io/scrape: true` to enable scraping + - `prometheus.io/port: 9090` to indicate the port number + - `prometheus.io/path: /metrics` to indicate the URI (`/metrics` by default) + +- Prometheus will detect and scrape these (without needing a restart or reload) + +--- + +## Querying labels + +- What if we want to get metrics for containers belong to pod tagged `worker`? + +- The cAdvisor exporter does not give us Kubernetes labels + +- Kubernetes labels are exposed through another exporter + +- We can see Kubernetes labels through metrics `kube_pod_labels` + + (each container appears as a time series with constant value of `1`) + +- Prometheus *kind of* supports "joins" between time series + +- But only if the names of the tags match exactly + +--- + +## Unfortunately ... + +- The cAdvisor exporter uses tag `pod_name` for the name of a pod + +- The Kubernetes service endpoints exporter uses tag `pod` instead + +- And this is why we can't have nice things + +- See [Prometheus issue #2204](https://github.com/prometheus/prometheus/issues/2204) for the rationale + + ([this comment](https://github.com/prometheus/prometheus/issues/2204#issuecomment-261515520) in particular if you want a workaround involving relabeling) + +- Then see [this blog post](https://www.robustperception.io/exposing-the-software-version-to-prometheus) or [this other one](https://www.weave.works/blog/aggregating-pod-resource-cpu-memory-usage-arbitrary-labels-prometheus/) to see how to perform "joins" + +- There is a good chance that the situation will improve in the future diff --git a/slides/k8s/statefulsets.md b/slides/k8s/statefulsets.md new file mode 100644 index 00000000..b997c1c1 --- /dev/null +++ b/slides/k8s/statefulsets.md @@ -0,0 +1,402 @@ +# Stateful sets + +- Stateful sets are a type of resource in the Kubernetes API + + (like pods, deployments, services...) + +- They offer mechanisms to deploy scaled stateful applications + +- At a first glance, they look like *deployments*: + + - a stateful set defines a pod spec and a number of replicas *R* + + - it will make sure that *R* copies of the pod are running + + - that number can be changed while the stateful set is running + + - updating the pod spec will cause a rolling update to happen + +- But they also have some significant differences + +--- + +## Stateful sets unique features + +- Pods in a stateful set are numbered (from 0 to *R-1*) and ordered + +- They are started and updated in order (from 0 to *R-1*) + +- A pod is started (or updated) only when the previous one is ready + +- They are stopped in reverse order (from *R-1* to 0) + +- Each pod know its identity (i.e. which number it is in the set) + +- Each pod can discover the IP address of the others easily + +- The pods can have persistent volumes attached to them + +🤔 Wait a minute ... Can't we already attach volumes to pods and deployments? + +--- + +## Volumes and Persistent Volumes + +- [Volumes](https://kubernetes.io/docs/concepts/storage/volumes/) are used for many purposes: + + - sharing data between containers in a pod + + - exposing configuration information and secrets to containers + + - accessing storage systems + +- The last type of volumes is known as a "Persistent Volume" + +--- + +## Persistent Volumes types + +- There are many [types of Persistent Volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#types-of-persistent-volumes) available: + + - public cloud storage (GCEPersistentDisk, AWSElasticBlockStore, AzureDisk...) + + - private cloud storage (Cinder, VsphereVolume...) + + - traditional storage systems (NFS, iSCSI, FC...) + + - distributed storage (Ceph, Glusterfs, Portworx...) + +- Using a persistent volume requires: + + - creating the volume out-of-band (outside of the Kubernetes API) + + - referencing the volume in the pod description, with all its parameters + +--- + +## Using a Persistent Volume + +Here is a pod definition using an AWS EBS volume (that has to be created first): + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: pod-using-my-ebs-volume +spec: + containers: + - image: ... + name: container-using-my-ebs-volume + volumeMounts: + - mountPath: /my-ebs + name: my-ebs-volume + volumes: + - name: my-ebs-volume + awsElasticBlockStore: + volumeID: vol-049df61146c4d7901 + fsType: ext4 +``` + +--- + +## Shortcomings of Persistent Volumes + +- Their lifecycle (creation, deletion...) is managed outside of the Kubernetes API + + (we can't just use `kubectl apply/create/delete/...` to manage them) + +- If a Deployment uses a volume, all replicas end up using the same volume + +- That volume must then support concurrent access + + - some volumes do (e.g. NFS servers support multiple read/write access) + + - some volumes support concurrent reads + + - some volumes support concurrent access for colocated pods + +- What we really need is a way for each replica to have its own volume + +--- + +## Persistent Volume Claims + +- To abstract the different types of storage, a pod can use a special volume type + +- This type is a *Persistent Volume Claim* + +- Using a Persistent Volume Claim is a two-step process: + + - creating the claim + + - using the claim in a pod (as if it were any other kind of volume) + +- Between these two steps, something will happen behind the scenes: + + - Kubernetes will associate an existing volume with the claim + + - ... or dynamically create a volume if possible and necessary + +--- + +## What's in a Persistent Volume Claim? + +- At the very least, the claim should indicate: + + - the size of the volume (e.g. "5 GiB") + + - the access mode (e.g. "read-write by a single pod") + +- It can also give extra details, like: + + - which storage system to use (e.g. Portworx, EBS...) + + - extra parameters for that storage system + + e.g.: "replicate the data 3 times, and use SSD media" + +- The extra details are provided by specifying a Storage Class + +--- + +## What's a Storage Class? + +- A Storage Class is yet another Kubernetes API resource + + (visible with e.g. `kubectl get storageclass` or `kubectl get sc`) + +- It indicates which *provisioner* to use + +- And arbitrary paramters for that provisioner + + (replication levels, type of disk ... anything relevant!) + +- It is necessary to define a Storage Class to use [dynamic provisioning](https://kubernetes.io/docs/concepts/storage/dynamic-provisioning/) + +- Conversely, it is not necessary to define one if you will create volumes manually + + (we will see dynamic provisioning in action later) + +--- + +## Defining a Persistent Volume Claim + +Here is a minimal PVC: + +```yaml +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: my-claim +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +``` + +--- + +## Using a Persistent Volume Claim + +Here is the same definition as earlier, but using a PVC: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: pod-using-a-claim +spec: + containers: + - image: ... + name: container-using-a-claim + volumeMounts: + - mountPath: /my-ebs + name: my-volume + volumes: + - name: my-volume + persistentVolumeClaim: + claimName: my-claim +``` + +--- + +## Persistent Volume Claims and Stateful sets + +- The pods in a stateful set can define a `volumeClaimTemplate` + +- A `volumeClaimTemplate` will dynamically create one Persistent Volume Claim per pod + +- Each pod will therefore have its own volume + +- These volumes are numbered (like the pods) + +- When updating the stateful set (e.g. image upgrade), each pod keeps its volume + +- When pods get rescheduled (e.g. node failure), they keep their volume + + (this requires a storage system that is not node-local) + +- These volumes are not automatically deleted + + (when the stateful set is scaled down or deleted) + +--- + +## Stateful set recap + +- A Stateful sets manages a number of identical pods + + (like a Deployment) + +- These pods are numbered, and started/upgraded/stopped in a specific order + +- These pods are aware of their number + + (e.g., #0 can decide to be the primary, and #1 can be secondary) + +- These pods can find the IP addresses of the other pods in the set + + (through a *headless service*) + +- These pods can each have their own persistent storage + + (Deployments cannot do that) + +--- + +## Stateful sets in action + +- We are going to deploy a Consul cluster with 3 nodes + +- Consul is a highly-available key/value store + + (like etcd or Zookeeper) + +- One easy way to bootstrap a cluster is to tell each node: + + - the addresses of other nodes + + - how many nodes are expected (to know when quorum is reached) + +--- + +## Bootstrapping a Consul cluster + +*After reading the Consul documentation carefully (and/or asking around), +we figure out the minimal command-line to run our Consul cluster.* + +``` +consul agent -data=dir=/consul/data -client=0.0.0.0 -server -ui \ + -bootstrap-expect=3 \ + -retry-join=`X.X.X.X` \ + -retry-join=`Y.Y.Y.Y` +``` + +- We need to replace X.X.X.X and Y.Y.Y.Y with the addresses of other nodes + +- We can specify DNS names, but then they have to be FQDN + +- It's OK for a pod to include itself in the list as well + +- We can therefore use the same command-line on all nodes (easier!) + +--- + +## Discovering the addresses of other pods + +- When a service is created for a stateful set, individual DNS entries are created + +- These entries are constructed like this: + + `-...svc.cluster.local` + +- `` is the number of the pod in the set (starting at zero) + +- If we deploy Consul in the default namespace, the names could be: + + - `consul-0.consul.default.svc.cluster.local` + - `consul-1.consul.default.svc.cluster.local` + - `consul-2.consul.default.svc.cluster.local` + +--- + +## Putting it all together + +- The file `k8s/consul.yaml` defines a service and a stateful set + +- It has a few extra touches: + + - the name of the namespace is injected through an environment variable + + - a `podAntiAffinity` prevents two pods from running on the same node + + - a `preStop` hook makes the pod leave the cluster when shutdown gracefully + +This was inspired by this [excellent tutorial](https://github.com/kelseyhightower/consul-on-kubernetes) by Kelsey Hightower. +Some features from the original tutorial (TLS authentication between +nodes and encryption of gossip traffic) were removed for simplicity. + +--- + +## Running our Consul cluster + +- We'll use the provided YAML file + +.exercise[ + +- Create the stateful set and associated service: + ```bash + kubectl apply -f ~/container.training/k8s/consul.yaml + ``` + +- Check the logs as the pods come up one after another: + ```bash + stern consul + ``` + +- Check the health of the cluster: + ```bash + kubectl exec consul-0 consul members + ``` + +] + +--- + +## Caveats + +- We haven't used a `volumeClaimTemplate` here + +- That's because we don't have a storage provider yet + + (except if you're running this on your own and your cluster has one) + +- What happens if we lose a pod? + + - a new pod gets rescheduled (with an empty state) + + - the new pod tries to connect to the two others + + - it will be accepted (after 1-2 minutes of instability) + + - and it will retrieve the data from the other pods + +--- + +## Failure modes + +- What happens if we lose two pods? + + - manual repair will be required + + - we will need to instruct the remaining one to act solo + + - then rejoin new pods + +- What happens if we lose three pods? (aka all of them) + + - we lose all the data (ouch) + +- If we run Consul without persistent storage, backups are a good idea! diff --git a/slides/k8s/volumes.md b/slides/k8s/volumes.md new file mode 100644 index 00000000..2773d185 --- /dev/null +++ b/slides/k8s/volumes.md @@ -0,0 +1,184 @@ +# Volumes + +- Volumes are special directories that are mounted in containers + +- Volumes can have many different purposes: + + - share files and directories between containers running on the same machine + + - share files and directories between containers and their host + + - centralize configuration information in Kubernetes and expose it to containers + + - manage credentials and secrets and expose them securely to containers + + - store persistent data for stateful services + + - access storage systems (like Ceph, EBS, NFS, Portworx, and many others) + +--- + +## Kubernetes volumes vs. Docker volumes + +- Kubernetes and Docker volumes are very similar + + (the [Kubernetes documentation](https://kubernetes.io/docs/concepts/storage/volumes/) says otherwise ... +
+ but it refers to Docker 1.7, which was released in 2015!) + +- Docker volumes allow to share data between containers running on the same host + +- Kubernetes volumes allow us to share data between containers in the same pod + +- Both Docker and Kubernetes volumes allow us access to storage systems + +- Kubernetes volumes are also used to expose configuration and secrets + +- Docker has specific concepts for configuration and secrets + + (but under the hood, the technical implementation is similar) + +- If you're not familiar with Docker volumes, you can safely ignore this slide! + +--- + +## A simple volume example + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: nginx-with-volume +spec: + volumes: + - name: www + containers: + - name: nginx + image: nginx + volumeMounts: + - name: www + mountPath: /usr/share/nginx/html/ +``` + +--- + +## A simple volume example, explained + +- We define a standalone `Pod` named `nginx-with-volume` + +- In that pod, there is a volume named `www` + +- No type is specified, so it will default to `emptyDir` + + (as the name implies, it will be initialized as an empty directory at pod creation) + +- In that pod, there is also a container named `nginx` + +- That container mounts the volume `www` to path `/usr/share/nginx/html/` + +--- + +## A volume shared between two containers + +.small[ +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: nginx-with-volume +spec: + volumes: + - name: www + containers: + - name: nginx + image: nginx + volumeMounts: + - name: www + mountPath: /usr/share/nginx/html/ + - name: git + image: alpine + command: [ "sh", "-c", "apk add --no-cache git && git clone https://github.com/octocat/Spoon-Knife /www" ] + volumeMounts: + - name: www + mountPath: /www/ + restartPolicy: OnFailure +``` +] + +--- + +## Sharing a volume, explained + +- We added another container to the pod + +- That container mounts the `www` volume on a different path (`/www`) + +- It uses the `alpine` image + +- When started, it installs `git` and clones the `octocat/Spoon-Knife` repository + + (that repository contains a tiny HTML website) + +- As a result, NGINX now serves this website + +--- + +## Sharing a volume, in action + +- Let's try it! + +.exercise[ + +- Create the pod by applying the YAML file: + ```bash + kubectl apply -f ~/container.training/k8s/nginx-with-volume.yaml + ``` + +- Check the IP address that was allocated to our pod: + ```bash + kubectl get pod nginx-with-volume -o wide + IP=$(kubectl get pod nginx-with-volume -o json | jq -r .status.podIP) + ``` + +- Access the web server: + ```bash + curl $IP + ``` + +] + +--- + +## The devil is in the details + +- The default `restartPolicy` is `Always` + +- This would cause our `git` container to run again ... and again ... and again + + (with an exponential back-off delay, as explained [in the documentation](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy)) + +- That's why we specified `restartPolicy: OnFailure` + +- There is a short period of time during which the website is not available + + (because the `git` container hasn't done its job yet) + +- This could be avoided by using [Init Containers](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) + + (we will see a live example in a few sections) + +--- + +## Volume lifecycle + +- The lifecycle of a volume is linked to the pod's lifecycle + +- This means that a volume is created when the pod is created + +- This is mostly relevant for `emptyDir` volumes + + (other volumes, like remote storage, are not "created" but rather "attached" ) + +- A volume survives across container restarts + +- A volume is destroyed (or, for remote storage, detached) when the pod is destroyed diff --git a/slides/kube-fullday.yml b/slides/kube-fullday.yml index 05df056d..cd13b334 100644 --- a/slides/kube-fullday.yml +++ b/slides/kube-fullday.yml @@ -22,28 +22,40 @@ chapters: - - shared/prereqs.md - k8s/versions-k8s.md - shared/sampleapp.md - #- shared/composescale.md + - shared/composescale.md - shared/composedown.md -- - k8s/concepts-k8s.md + - k8s/concepts-k8s.md - shared/declarative.md - k8s/declarative.md - - k8s/kubenet.md +- - k8s/kubenet.md - k8s/kubectlget.md - k8s/setup-k8s.md - k8s/kubectlrun.md -- - k8s/kubectlexpose.md - - k8s/ourapponkube.md + - k8s/kubectlexpose.md +- - k8s/ourapponkube.md - k8s/kubectlproxy.md - k8s/localkubeconfig.md - k8s/dashboard.md -- - k8s/kubectlscale.md + - k8s/kubectlscale.md - k8s/daemonset.md - - k8s/rollout.md +- - k8s/rollout.md + - k8s/healthchecks.md - k8s/logs-cli.md - k8s/logs-centralized.md - - k8s/helm.md +- - k8s/helm.md - k8s/namespaces.md - k8s/netpol.md - - k8s/whatsnext.md + - k8s/authn-authz.md +- - k8s/ingress.md + - k8s/gitworkflows.md + - k8s/prometheus.md +- - k8s/volumes.md + - k8s/build-with-docker.md + - k8s/build-with-kaniko.md + - k8s/configuration.md +- - k8s/owners-and-dependents.md + - k8s/statefulsets.md + - k8s/portworx.md +- - k8s/whatsnext.md - k8s/links.md - shared/thankyou.md diff --git a/slides/kube-selfpaced.yml b/slides/kube-selfpaced.yml index 598193bc..df3fc8a3 100644 --- a/slides/kube-selfpaced.yml +++ b/slides/kube-selfpaced.yml @@ -23,26 +23,38 @@ chapters: - shared/sampleapp.md - shared/composescale.md - shared/composedown.md -- - k8s/concepts-k8s.md + - k8s/concepts-k8s.md - shared/declarative.md - k8s/declarative.md - - k8s/kubenet.md +- - k8s/kubenet.md - k8s/kubectlget.md - k8s/setup-k8s.md - k8s/kubectlrun.md -- - k8s/kubectlexpose.md - - k8s/ourapponkube.md + - k8s/kubectlexpose.md +- - k8s/ourapponkube.md - k8s/kubectlproxy.md - k8s/localkubeconfig.md - k8s/dashboard.md -- - k8s/kubectlscale.md + - k8s/kubectlscale.md - k8s/daemonset.md - - k8s/rollout.md -- - k8s/logs-cli.md +- - k8s/rollout.md + - k8s/healthchecks.md + - k8s/logs-cli.md - k8s/logs-centralized.md - - k8s/helm.md +- - k8s/helm.md - k8s/namespaces.md - k8s/netpol.md - - k8s/whatsnext.md + - k8s/authn-authz.md +- - k8s/ingress.md + - k8s/gitworkflows.md + - k8s/prometheus.md +- - k8s/volumes.md + - k8s/build-with-docker.md + - k8s/build-with-kaniko.md + - k8s/configuration.md +- - k8s/owners-and-dependents.md + - k8s/statefulsets.md + - k8s/portworx.md +- - k8s/whatsnext.md - k8s/links.md - shared/thankyou.md