✨ Prepare content for Thoughtworks Infrastructure

Rehaul for crd/v1; demonstrate what happens when adding
data validation a posteriori.

.gitignore

@@ -1,9 +1,13 @@
 *.pyc
 *.swp
 *~
+
 prepare-vms/tags
 prepare-vms/infra
 prepare-vms/www
+
+prepare-tf/tag-*
+
 slides/*.yml.html
 slides/autopilot/state.yaml
 slides/index.html

compose/frr-route-reflector/conf/zebra.conf

@@ -1,2 +1,3 @@
 hostname frr
+ip nht resolve-via-default
 log stdout

compose/frr-route-reflector/docker-compose.yaml

@@ -2,30 +2,36 @@ version: "3"
 
 services:
   bgpd:
-    image: ajones17/frr:662
+    image: frrouting/frr:v8.2.2
     volumes:
     - ./conf:/etc/frr
     - ./run:/var/run/frr
     network_mode: host
-    entrypoint: /usr/lib/frr/bgpd -f /etc/frr/bgpd.conf --log=stdout --log-level=debug --no_kernel
+    cap_add:
+    - NET_ADMIN
+    - SYS_ADMIN
+    entrypoint: /usr/lib/frr/bgpd -f /etc/frr/bgpd.conf --log=stdout --log-level=debug --no_kernel --no_zebra
     restart: always
 
   zebra:
-    image: ajones17/frr:662
+    image: frrouting/frr:v8.2.2
     volumes:
     - ./conf:/etc/frr
     - ./run:/var/run/frr
     network_mode: host
+    cap_add:
+    - NET_ADMIN
+    - SYS_ADMIN
     entrypoint: /usr/lib/frr/zebra -f /etc/frr/zebra.conf --log=stdout --log-level=debug
     restart: always
 
   vtysh:
-    image: ajones17/frr:662
+    image: frrouting/frr:v8.2.2
     volumes:
     - ./conf:/etc/frr
     - ./run:/var/run/frr
     network_mode: host
-    entrypoint: vtysh -c "show ip bgp"
+    entrypoint: vtysh
 
   chmod:
     image: alpine

dockercoins/Tiltfile

@@ -1,49 +1,72 @@
-k8s_yaml(blob('''
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    app: registry
-  name: registry
-spec:
-  selector:
-    matchLabels:
-      app: registry
-  template:
-    metadata:
-      labels:
-        app: registry
-    spec:
-      containers:
-      - image: registry
-        name: registry
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: registry
-  name: registry
-spec:
-  ports:
-  - port: 5000
-    protocol: TCP
-    targetPort: 5000
-    nodePort: 30555
-  selector:
-    app: registry
-  type: NodePort
-'''))
+# (1) Setting up a registry, and telling Tilt to use it.
+
+# Tilt needs a registry to store images.
+
+# The following manifest defines a Deployment to run a basic Docker registry,
+# and a NodePort Service to access it. Using a NodePort means that we don't
+# need to obtain a TLS certificate, because we will be accessing the registry
+# through localhost.
+k8s_yaml('../k8s/tilt-registry.yaml')
+
+# Tell Tilt to use the registry that we just deployed instead of whatever
+# is defined in our Kubernetes resources. Tilt will patch image names to
+# use our registry.
 default_registry('localhost:30555')
+
+# Create a port forward so that we can access the registry from our local
+# environment, too. Note that if you run Tilt directly from a Kubernetes node
+# (which is not typical, but might happen in some lab/training environments)
+# the following might cause an error because port 30555 is already taken.
+k8s_resource(workload='tilt-registry', port_forwards='30555:5000')
+
+# (2) Telling Tilt how to build and run our app.
+
+# The following two lines will use the kubectl-build plugin
+# to leverage buildkit and build the images in our Kubernetes
+# cluster. This is not enabled by default, because it requires
+# the plugin to be installed.
+# See https://github.com/vmware-tanzu/buildkit-cli-for-kubectl
+# for more information about this plugin.
+#load('ext://kubectl_build', 'kubectl_build')
+#docker_build = kubectl_build
+
+# Our Kubernetes manifests use images 'dockercoins/...' so we tell Tilt
+# how each of these images should be built. The first argument is the name
+# of the image, the second argument is the directory containing the build
+# context (i.e. the Dockerfile to build the image).
 docker_build('dockercoins/hasher', 'hasher')
 docker_build('dockercoins/rng', 'rng')
 docker_build('dockercoins/webui', 'webui')
 docker_build('dockercoins/worker', 'worker')
+
+# The following manifests defines five Deployments and four Services for
+# our application.
 k8s_yaml('../k8s/dockercoins.yaml')
 
-# Uncomment the following line to let tilt run with the default kubeadm cluster-admin context.
-#allow_k8s_contexts('kubernetes-admin@kubernetes')
+# (3) Finishing touches.
 
-# While we're here: if you're controlling a remote cluster, uncomment that line.
-# It will create a port forward so that you can access the remote registry.
-#k8s_resource(workload='registry', port_forwards='30555:5000')
+# The following line lets Tilt run with the default kubeadm cluster-admin context.
+allow_k8s_contexts('kubernetes-admin@kubernetes')
+
+# Note: the whole section below (to set up ngrok tunnels) is disabled,
+# because ngrok now requires to set up an account to serve HTML
+# content. So we can still use ngrok for e.g. webhooks and "raw" APIs,
+# but not to serve web pages like the Tilt UI.
+
+# # This will run an ngrok tunnel to expose Tilt to the outside world.
+# # This is intended to be used when Tilt runs on a remote machine.
+# local_resource(name='ngrok:tunnel', serve_cmd='ngrok http 10350')
+
+# # This will wait until the ngrok tunnel is up, and show its URL to the user.
+# # We send the output to /dev/tty so that it doesn't get intercepted by
+# # Tilt, and gets displayed to the user's terminal instead.
+# # Note: this assumes that the ngrok instance will be running on port 4040.
+# # If you have other ngrok instances running on the machine, this might not work.
+# local_resource(name='ngrok:showurl', cmd='''
+#   while sleep 1; do
+#     TUNNELS=$(curl -fsSL http://localhost:4040/api/tunnels | jq -r .tunnels[].public_url)
+#     [ "$TUNNELS" ] && break
+#   done
+#   printf "\nYou should be able to connect to the Tilt UI with the following URL(s): %s\n" "$TUNNELS" >/dev/tty
+#   '''
+# )

dockercoins/webui/Dockerfile

@@ -1,6 +1,6 @@
 FROM node:4-slim
 RUN npm install express
-RUN npm install redis
+RUN npm install redis@3
 COPY files/ /files/
 COPY webui.js /
 CMD ["node", "webui.js"]

dockercoins/webui/files/index.html

@@ -13,7 +13,7 @@
     color: royalblue;
 }
 </style>
-<script src="jquery.js"></script>
+<script src="jquery-1.11.3.min.js"></script>
 <script src="d3.min.js"></script>
 <script src="rickshaw.min.js"></script>
 <script>

dockercoins/webui/files/jquery.js

@@ -1 +0,0 @@
-jquery-1.11.3.min.js

k8s/Tiltfile.helmchart

@@ -0,0 +1,8 @@
+k8s_yaml(helm(
+    "./path-to-chart", name="blue",
+    values=[], # Example: ["./path/to/values.yaml"]
+    set=[
+        "image.repository=jpetazzo/color",
+        "image.tag=latest",
+    ]
+))

k8s/admission-configuration.yaml

@@ -0,0 +1,16 @@
+apiVersion: apiserver.config.k8s.io/v1
+kind: AdmissionConfiguration
+plugins:
+- name: PodSecurity
+  configuration:
+    apiVersion: pod-security.admission.config.k8s.io/v1alpha1
+    kind: PodSecurityConfiguration
+    defaults:
+      enforce: baseline
+      audit: baseline
+      warn: baseline
+    exemptions:
+      usernames:
+      - cluster-admin
+      namespaces:
+      - kube-system

k8s/certbot.yaml

@@ -7,7 +7,7 @@ spec:
   - port: 80
     protocol: TCP
 ---
-apiVersion: networking.k8s.io/v1beta1
+apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: certbot
@@ -16,9 +16,12 @@ spec:
   - http:
       paths:
       - path: /.well-known/acme-challenge/
+        pathType: Prefix
         backend:
-          serviceName: certbot
-          servicePort: 80
+          service:
+            name: certbot
+            port:
+              number: 80
 ---
 apiVersion: v1
 kind: Endpoints

k8s/consul-1.yaml

@@ -3,6 +3,12 @@
 # - no actual persistence
 # - scaling down to 1 will break the cluster
 # - pods may be colocated
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: consul
+---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
@@ -28,11 +34,6 @@ subjects:
     name: consul
 ---
 apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: consul
----
-apiVersion: v1
 kind: Service
 metadata:
   name: consul
@@ -61,7 +62,7 @@ spec:
       serviceAccountName: consul
       containers:
         - name: consul
-          image: "consul:1.8"
+          image: "consul:1.11"
           env:
             - name: NAMESPACE
               valueFrom:

k8s/consul-2.yaml

@@ -2,6 +2,12 @@
 # There is still no actual persistence, but:
 # - podAntiaffinity prevents pod colocation
 # - clusters works when scaling down to 1 (thanks to lifecycle hook)
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: consul
+---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
@@ -27,11 +33,6 @@ subjects:
     name: consul
 ---
 apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: consul
----
-apiVersion: v1
 kind: Service
 metadata:
   name: consul
@@ -68,7 +69,7 @@ spec:
       terminationGracePeriodSeconds: 10
       containers:
         - name: consul
-          image: "consul:1.8"
+          image: "consul:1.11"
           env:
             - name: NAMESPACE
               valueFrom:

k8s/consul-3.yaml

@@ -1,5 +1,11 @@
 # Even better Consul cluster.
 # That one uses a volumeClaimTemplate to achieve true persistence.
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: consul
+---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
@@ -25,11 +31,6 @@ subjects:
     name: consul
 ---
 apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: consul
----
-apiVersion: v1
 kind: Service
 metadata:
   name: consul
@@ -75,7 +76,7 @@ spec:
       terminationGracePeriodSeconds: 10
       containers:
         - name: consul
-          image: "consul:1.8"
+          image: "consul:1.11"
           volumeMounts:
             - name: data
               mountPath: /consul/data

k8s/dashboard-insecure.yaml

@@ -1,151 +1,189 @@
-# This file is based on the following manifest:
-# https://github.com/kubernetes/dashboard/blob/master/aio/deploy/recommended.yaml
-# It adds the "skip login" flag, as well as an insecure hack to defeat SSL.
-# As its name implies, it is INSECURE and you should not use it in production,
-# or on clusters that contain any kind of important or sensitive data, or on
-# clusters that have a life span of more than a few hours.
-
-# Copyright 2017 The Kubernetes Authors.
+# This file was generated with the script ./update-dashboard-yaml.sh.
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
+---
 apiVersion: v1
 kind: Namespace
 metadata:
+  creationTimestamp: null
   name: kubernetes-dashboard
-
+spec: {}
+status: {}
 ---
-
 apiVersion: v1
 kind: ServiceAccount
 metadata:
+  annotations: null
   labels:
-    k8s-app: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
   name: kubernetes-dashboard
   namespace: kubernetes-dashboard
-
 ---
-
-kind: Service
-apiVersion: v1
-metadata:
-  labels:
-    k8s-app: kubernetes-dashboard
-  name: kubernetes-dashboard
-  namespace: kubernetes-dashboard
-spec:
-  ports:
-    - port: 443
-      targetPort: 8443
-  selector:
-    k8s-app: kubernetes-dashboard
-
----
-
 apiVersion: v1
 kind: Secret
 metadata:
+  annotations: null
   labels:
-    k8s-app: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
   name: kubernetes-dashboard-certs
   namespace: kubernetes-dashboard
 type: Opaque
-
 ---
-
 apiVersion: v1
 kind: Secret
 metadata:
   labels:
-    k8s-app: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
   name: kubernetes-dashboard-csrf
   namespace: kubernetes-dashboard
 type: Opaque
-data:
-  csrf: ""
-
 ---
-
 apiVersion: v1
 kind: Secret
 metadata:
   labels:
-    k8s-app: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
   name: kubernetes-dashboard-key-holder
   namespace: kubernetes-dashboard
 type: Opaque
-
 ---
-
-kind: ConfigMap
 apiVersion: v1
+data: null
+kind: ConfigMap
 metadata:
+  annotations: null
   labels:
-    k8s-app: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
   name: kubernetes-dashboard-settings
   namespace: kubernetes-dashboard
-
 ---
-
-kind: Role
 apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
 metadata:
+  annotations: null
   labels:
-    k8s-app: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
+  name: kubernetes-dashboard-metrics
+rules:
+- apiGroups:
+  - metrics.k8s.io
+  resources:
+  - pods
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  annotations: null
+  labels:
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
+  name: kubernetes-dashboard-metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubernetes-dashboard-metrics
+subjects:
+- kind: ServiceAccount
+  name: kubernetes-dashboard
+  namespace: kubernetes-dashboard
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  annotations: null
+  labels:
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
   name: kubernetes-dashboard
   namespace: kubernetes-dashboard
 rules:
-  # Allow Dashboard to get, update and delete Dashboard exclusive secrets.
-  - apiGroups: [""]
-    resources: ["secrets"]
-    resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]
-    verbs: ["get", "update", "delete"]
-    # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
-  - apiGroups: [""]
-    resources: ["configmaps"]
-    resourceNames: ["kubernetes-dashboard-settings"]
-    verbs: ["get", "update"]
-    # Allow Dashboard to get metrics.
-  - apiGroups: [""]
-    resources: ["services"]
-    resourceNames: ["heapster", "dashboard-metrics-scraper"]
-    verbs: ["proxy"]
-  - apiGroups: [""]
-    resources: ["services/proxy"]
-    resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]
-    verbs: ["get"]
-
+- apiGroups:
+  - ""
+  resourceNames:
+  - kubernetes-dashboard-key-holder
+  - kubernetes-dashboard-certs
+  - kubernetes-dashboard-csrf
+  resources:
+  - secrets
+  verbs:
+  - get
+  - update
+  - delete
+- apiGroups:
+  - ""
+  resourceNames:
+  - kubernetes-dashboard-settings
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - update
+- apiGroups:
+  - ""
+  resourceNames:
+  - heapster
+  - dashboard-metrics-scraper
+  resources:
+  - services
+  verbs:
+  - proxy
+- apiGroups:
+  - ""
+  resourceNames:
+  - heapster
+  - 'http:heapster:'
+  - 'https:heapster:'
+  - dashboard-metrics-scraper
+  - http:dashboard-metrics-scraper
+  resources:
+  - services/proxy
+  verbs:
+  - get
 ---
-
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  labels:
-    k8s-app: kubernetes-dashboard
-  name: kubernetes-dashboard
-rules:
-  # Allow Metrics Scraper to get metrics from the Metrics server
-  - apiGroups: ["metrics.k8s.io"]
-    resources: ["pods", "nodes"]
-    verbs: ["get", "list", "watch"]
-
----
-
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
+  annotations: null
   labels:
-    k8s-app: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
   name: kubernetes-dashboard
   namespace: kubernetes-dashboard
 roleRef:
@@ -153,210 +191,145 @@ roleRef:
   kind: Role
   name: kubernetes-dashboard
 subjects:
-  - kind: ServiceAccount
-    name: kubernetes-dashboard
-    namespace: kubernetes-dashboard
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: kubernetes-dashboard
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: kubernetes-dashboard
-subjects:
-  - kind: ServiceAccount
-    name: kubernetes-dashboard
-    namespace: kubernetes-dashboard
-
----
-
-kind: Deployment
-apiVersion: apps/v1
-metadata:
-  labels:
-    k8s-app: kubernetes-dashboard
+- kind: ServiceAccount
   name: kubernetes-dashboard
   namespace: kubernetes-dashboard
-spec:
-  replicas: 1
-  revisionHistoryLimit: 10
-  selector:
-    matchLabels:
-      k8s-app: kubernetes-dashboard
-  template:
-    metadata:
-      labels:
-        k8s-app: kubernetes-dashboard
-    spec:
-      containers:
-        - name: kubernetes-dashboard
-          image: kubernetesui/dashboard:v2.0.0
-          imagePullPolicy: Always
-          ports:
-            - containerPort: 8443
-              protocol: TCP
-          args:
-            - --auto-generate-certificates
-            - --namespace=kubernetes-dashboard
-            # Uncomment the following line to manually specify Kubernetes API server Host
-            # If not specified, Dashboard will attempt to auto discover the API server and connect
-            # to it. Uncomment only if the default does not work.
-            # - --apiserver-host=http://my-address:port
-            - --enable-skip-login
-          volumeMounts:
-            - name: kubernetes-dashboard-certs
-              mountPath: /certs
-              # Create on-disk volume to store exec logs
-            - mountPath: /tmp
-              name: tmp-volume
-          livenessProbe:
-            httpGet:
-              scheme: HTTPS
-              path: /
-              port: 8443
-            initialDelaySeconds: 30
-            timeoutSeconds: 30
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            runAsUser: 1001
-            runAsGroup: 2001
-      volumes:
-        - name: kubernetes-dashboard-certs
-          secret:
-            secretName: kubernetes-dashboard-certs
-        - name: tmp-volume
-          emptyDir: {}
-      serviceAccountName: kubernetes-dashboard
-      nodeSelector:
-        "kubernetes.io/os": linux
-      # Comment the following tolerations if Dashboard must not be deployed on master
-      tolerations:
-        - key: node-role.kubernetes.io/master
-          effect: NoSchedule
-
 ---
-
-kind: Service
 apiVersion: v1
+kind: Service
 metadata:
+  annotations: null
   labels:
-    k8s-app: dashboard-metrics-scraper
-  name: dashboard-metrics-scraper
+    app.kubernetes.io/component: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
+    kubernetes.io/cluster-service: "true"
+  name: kubernetes-dashboard
   namespace: kubernetes-dashboard
 spec:
   ports:
-    - port: 8000
-      targetPort: 8000
+  - name: http
+    port: 443
+    targetPort: http
   selector:
-    k8s-app: dashboard-metrics-scraper
-
+    app.kubernetes.io/component: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/name: kubernetes-dashboard
+  type: NodePort
 ---
-
-kind: Deployment
 apiVersion: apps/v1
+kind: Deployment
 metadata:
+  annotations: null
   labels:
-    k8s-app: dashboard-metrics-scraper
-  name: dashboard-metrics-scraper
+    app.kubernetes.io/component: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
+  name: kubernetes-dashboard
   namespace: kubernetes-dashboard
 spec:
   replicas: 1
-  revisionHistoryLimit: 10
   selector:
     matchLabels:
-      k8s-app: dashboard-metrics-scraper
+      app.kubernetes.io/component: kubernetes-dashboard
+      app.kubernetes.io/instance: kubernetes-dashboard
+      app.kubernetes.io/name: kubernetes-dashboard
+  strategy:
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+    type: RollingUpdate
   template:
     metadata:
+      annotations: null
       labels:
-        k8s-app: dashboard-metrics-scraper
-      annotations:
-        seccomp.security.alpha.kubernetes.io/pod: 'runtime/default'
-    spec:
-      containers:
-        - name: dashboard-metrics-scraper
-          image: kubernetesui/metrics-scraper:v1.0.4
-          ports:
-            - containerPort: 8000
-              protocol: TCP
-          livenessProbe:
-            httpGet:
-              scheme: HTTP
-              path: /
-              port: 8000
-            initialDelaySeconds: 30
-            timeoutSeconds: 30
-          volumeMounts:
-          - mountPath: /tmp
-            name: tmp-volume
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            runAsUser: 1001
-            runAsGroup: 2001
-      serviceAccountName: kubernetes-dashboard
-      nodeSelector:
-        "kubernetes.io/os": linux
-      # Comment the following tolerations if Dashboard must not be deployed on master
-      tolerations:
-        - key: node-role.kubernetes.io/master
-          effect: NoSchedule
-      volumes:
-        - name: tmp-volume
-          emptyDir: {}
-
----
-
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    app: dashboard
-  name: dashboard
-spec:
-  selector:
-    matchLabels:
-      app: dashboard
-  template:
-    metadata:
-      labels:
-        app: dashboard
+        app.kubernetes.io/component: kubernetes-dashboard
+        app.kubernetes.io/instance: kubernetes-dashboard
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/name: kubernetes-dashboard
+        app.kubernetes.io/version: 2.5.0
+        helm.sh/chart: kubernetes-dashboard-5.2.0
     spec:
       containers:
       - args:
-        - sh
-        - -c
-        - apk add --no-cache socat && socat TCP-LISTEN:80,fork,reuseaddr OPENSSL:kubernetes-dashboard.kubernetes-dashboard:443,verify=0
-        image: alpine
-        name: dashboard
-
+        - --namespace=kubernetes-dashboard
+        - --sidecar-host=http://127.0.0.1:8000
+        - --enable-skip-login
+        - --enable-insecure-login
+        image: kubernetesui/dashboard:v2.5.0
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 9090
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 30
+        name: kubernetes-dashboard
+        ports:
+        - containerPort: 9090
+          name: http
+          protocol: TCP
+        resources:
+          limits:
+            cpu: 2
+            memory: 200Mi
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsGroup: 2001
+          runAsUser: 1001
+        volumeMounts:
+        - mountPath: /certs
+          name: kubernetes-dashboard-certs
+        - mountPath: /tmp
+          name: tmp-volume
+      - image: kubernetesui/metrics-scraper:v1.0.7
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 8000
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 30
+        name: dashboard-metrics-scraper
+        ports:
+        - containerPort: 8000
+          protocol: TCP
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsGroup: 2001
+          runAsUser: 1001
+        volumeMounts:
+        - mountPath: /tmp
+          name: tmp-volume
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      serviceAccountName: kubernetes-dashboard
+      volumes:
+      - name: kubernetes-dashboard-certs
+        secret:
+          secretName: kubernetes-dashboard-certs
+      - emptyDir: {}
+        name: tmp-volume
 ---
-
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: dashboard
-  name: dashboard
-spec:
-  ports:
-  - port: 80
-    protocol: TCP
-    targetPort: 80
-  selector:
-    app: dashboard
-  type: NodePort
-
----
-
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: insecure-dashboard
+  creationTimestamp: null
+  name: kubernetes-dashboard:insecure
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole

k8s/dashboard-recommended.yaml

@@ -1,147 +1,189 @@
-# This is a copy of the following file:
-# https://github.com/kubernetes/dashboard/blob/master/aio/deploy/recommended.yaml
-
-# Copyright 2017 The Kubernetes Authors.
+# This file was generated with the script ./update-dashboard-yaml.sh.
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
+---
 apiVersion: v1
 kind: Namespace
 metadata:
+  creationTimestamp: null
   name: kubernetes-dashboard
-
+spec: {}
+status: {}
 ---
-
 apiVersion: v1
 kind: ServiceAccount
 metadata:
+  annotations: null
   labels:
-    k8s-app: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
   name: kubernetes-dashboard
   namespace: kubernetes-dashboard
-
 ---
-
-kind: Service
-apiVersion: v1
-metadata:
-  labels:
-    k8s-app: kubernetes-dashboard
-  name: kubernetes-dashboard
-  namespace: kubernetes-dashboard
-spec:
-  ports:
-    - port: 443
-      targetPort: 8443
-  selector:
-    k8s-app: kubernetes-dashboard
-
----
-
 apiVersion: v1
 kind: Secret
 metadata:
+  annotations: null
   labels:
-    k8s-app: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
   name: kubernetes-dashboard-certs
   namespace: kubernetes-dashboard
 type: Opaque
-
 ---
-
 apiVersion: v1
 kind: Secret
 metadata:
   labels:
-    k8s-app: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
   name: kubernetes-dashboard-csrf
   namespace: kubernetes-dashboard
 type: Opaque
-data:
-  csrf: ""
-
 ---
-
 apiVersion: v1
 kind: Secret
 metadata:
   labels:
-    k8s-app: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
   name: kubernetes-dashboard-key-holder
   namespace: kubernetes-dashboard
 type: Opaque
-
 ---
-
-kind: ConfigMap
 apiVersion: v1
+data: null
+kind: ConfigMap
 metadata:
+  annotations: null
   labels:
-    k8s-app: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
   name: kubernetes-dashboard-settings
   namespace: kubernetes-dashboard
-
 ---
-
-kind: Role
 apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
 metadata:
+  annotations: null
   labels:
-    k8s-app: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
+  name: kubernetes-dashboard-metrics
+rules:
+- apiGroups:
+  - metrics.k8s.io
+  resources:
+  - pods
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  annotations: null
+  labels:
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
+  name: kubernetes-dashboard-metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubernetes-dashboard-metrics
+subjects:
+- kind: ServiceAccount
+  name: kubernetes-dashboard
+  namespace: kubernetes-dashboard
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  annotations: null
+  labels:
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
   name: kubernetes-dashboard
   namespace: kubernetes-dashboard
 rules:
-  # Allow Dashboard to get, update and delete Dashboard exclusive secrets.
-  - apiGroups: [""]
-    resources: ["secrets"]
-    resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]
-    verbs: ["get", "update", "delete"]
-    # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
-  - apiGroups: [""]
-    resources: ["configmaps"]
-    resourceNames: ["kubernetes-dashboard-settings"]
-    verbs: ["get", "update"]
-    # Allow Dashboard to get metrics.
-  - apiGroups: [""]
-    resources: ["services"]
-    resourceNames: ["heapster", "dashboard-metrics-scraper"]
-    verbs: ["proxy"]
-  - apiGroups: [""]
-    resources: ["services/proxy"]
-    resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]
-    verbs: ["get"]
-
+- apiGroups:
+  - ""
+  resourceNames:
+  - kubernetes-dashboard-key-holder
+  - kubernetes-dashboard-certs
+  - kubernetes-dashboard-csrf
+  resources:
+  - secrets
+  verbs:
+  - get
+  - update
+  - delete
+- apiGroups:
+  - ""
+  resourceNames:
+  - kubernetes-dashboard-settings
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - update
+- apiGroups:
+  - ""
+  resourceNames:
+  - heapster
+  - dashboard-metrics-scraper
+  resources:
+  - services
+  verbs:
+  - proxy
+- apiGroups:
+  - ""
+  resourceNames:
+  - heapster
+  - 'http:heapster:'
+  - 'https:heapster:'
+  - dashboard-metrics-scraper
+  - http:dashboard-metrics-scraper
+  resources:
+  - services/proxy
+  verbs:
+  - get
 ---
-
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  labels:
-    k8s-app: kubernetes-dashboard
-  name: kubernetes-dashboard
-rules:
-  # Allow Metrics Scraper to get metrics from the Metrics server
-  - apiGroups: ["metrics.k8s.io"]
-    resources: ["pods", "nodes"]
-    verbs: ["get", "list", "watch"]
-
----
-
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
+  annotations: null
   labels:
-    k8s-app: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
   name: kubernetes-dashboard
   namespace: kubernetes-dashboard
 roleRef:
@@ -149,157 +191,135 @@ roleRef:
   kind: Role
   name: kubernetes-dashboard
 subjects:
-  - kind: ServiceAccount
-    name: kubernetes-dashboard
-    namespace: kubernetes-dashboard
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: kubernetes-dashboard
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: kubernetes-dashboard
-subjects:
-  - kind: ServiceAccount
-    name: kubernetes-dashboard
-    namespace: kubernetes-dashboard
-
----
-
-kind: Deployment
-apiVersion: apps/v1
-metadata:
-  labels:
-    k8s-app: kubernetes-dashboard
+- kind: ServiceAccount
   name: kubernetes-dashboard
   namespace: kubernetes-dashboard
-spec:
-  replicas: 1
-  revisionHistoryLimit: 10
-  selector:
-    matchLabels:
-      k8s-app: kubernetes-dashboard
-  template:
-    metadata:
-      labels:
-        k8s-app: kubernetes-dashboard
-    spec:
-      containers:
-        - name: kubernetes-dashboard
-          image: kubernetesui/dashboard:v2.0.0
-          imagePullPolicy: Always
-          ports:
-            - containerPort: 8443
-              protocol: TCP
-          args:
-            - --auto-generate-certificates
-            - --namespace=kubernetes-dashboard
-            # Uncomment the following line to manually specify Kubernetes API server Host
-            # If not specified, Dashboard will attempt to auto discover the API server and connect
-            # to it. Uncomment only if the default does not work.
-            # - --apiserver-host=http://my-address:port
-          volumeMounts:
-            - name: kubernetes-dashboard-certs
-              mountPath: /certs
-              # Create on-disk volume to store exec logs
-            - mountPath: /tmp
-              name: tmp-volume
-          livenessProbe:
-            httpGet:
-              scheme: HTTPS
-              path: /
-              port: 8443
-            initialDelaySeconds: 30
-            timeoutSeconds: 30
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            runAsUser: 1001
-            runAsGroup: 2001
-      volumes:
-        - name: kubernetes-dashboard-certs
-          secret:
-            secretName: kubernetes-dashboard-certs
-        - name: tmp-volume
-          emptyDir: {}
-      serviceAccountName: kubernetes-dashboard
-      nodeSelector:
-        "kubernetes.io/os": linux
-      # Comment the following tolerations if Dashboard must not be deployed on master
-      tolerations:
-        - key: node-role.kubernetes.io/master
-          effect: NoSchedule
-
 ---
-
-kind: Service
 apiVersion: v1
+kind: Service
 metadata:
+  annotations: null
   labels:
-    k8s-app: dashboard-metrics-scraper
-  name: dashboard-metrics-scraper
+    app.kubernetes.io/component: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
+    kubernetes.io/cluster-service: "true"
+  name: kubernetes-dashboard
   namespace: kubernetes-dashboard
 spec:
   ports:
-    - port: 8000
-      targetPort: 8000
+  - name: https
+    port: 443
+    targetPort: https
   selector:
-    k8s-app: dashboard-metrics-scraper
-
+    app.kubernetes.io/component: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/name: kubernetes-dashboard
+  type: ClusterIP
 ---
-
-kind: Deployment
 apiVersion: apps/v1
+kind: Deployment
 metadata:
+  annotations: null
   labels:
-    k8s-app: dashboard-metrics-scraper
-  name: dashboard-metrics-scraper
+    app.kubernetes.io/component: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
+  name: kubernetes-dashboard
   namespace: kubernetes-dashboard
 spec:
   replicas: 1
-  revisionHistoryLimit: 10
   selector:
     matchLabels:
-      k8s-app: dashboard-metrics-scraper
+      app.kubernetes.io/component: kubernetes-dashboard
+      app.kubernetes.io/instance: kubernetes-dashboard
+      app.kubernetes.io/name: kubernetes-dashboard
+  strategy:
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+    type: RollingUpdate
   template:
     metadata:
+      annotations: null
       labels:
-        k8s-app: dashboard-metrics-scraper
-      annotations:
-        seccomp.security.alpha.kubernetes.io/pod: 'runtime/default'
+        app.kubernetes.io/component: kubernetes-dashboard
+        app.kubernetes.io/instance: kubernetes-dashboard
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/name: kubernetes-dashboard
+        app.kubernetes.io/version: 2.5.0
+        helm.sh/chart: kubernetes-dashboard-5.2.0
     spec:
       containers:
-        - name: dashboard-metrics-scraper
-          image: kubernetesui/metrics-scraper:v1.0.4
-          ports:
-            - containerPort: 8000
-              protocol: TCP
-          livenessProbe:
-            httpGet:
-              scheme: HTTP
-              path: /
-              port: 8000
-            initialDelaySeconds: 30
-            timeoutSeconds: 30
-          volumeMounts:
-          - mountPath: /tmp
-            name: tmp-volume
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            runAsUser: 1001
-            runAsGroup: 2001
+      - args:
+        - --namespace=kubernetes-dashboard
+        - --auto-generate-certificates
+        - --sidecar-host=http://127.0.0.1:8000
+        image: kubernetesui/dashboard:v2.5.0
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 8443
+            scheme: HTTPS
+          initialDelaySeconds: 30
+          timeoutSeconds: 30
+        name: kubernetes-dashboard
+        ports:
+        - containerPort: 8443
+          name: https
+          protocol: TCP
+        resources:
+          limits:
+            cpu: 2
+            memory: 200Mi
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsGroup: 2001
+          runAsUser: 1001
+        volumeMounts:
+        - mountPath: /certs
+          name: kubernetes-dashboard-certs
+        - mountPath: /tmp
+          name: tmp-volume
+      - image: kubernetesui/metrics-scraper:v1.0.7
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 8000
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 30
+        name: dashboard-metrics-scraper
+        ports:
+        - containerPort: 8000
+          protocol: TCP
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsGroup: 2001
+          runAsUser: 1001
+        volumeMounts:
+        - mountPath: /tmp
+          name: tmp-volume
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
       serviceAccountName: kubernetes-dashboard
-      nodeSelector:
-        "kubernetes.io/os": linux
-      # Comment the following tolerations if Dashboard must not be deployed on master
-      tolerations:
-        - key: node-role.kubernetes.io/master
-          effect: NoSchedule
       volumes:
-        - name: tmp-volume
-          emptyDir: {}
+      - name: kubernetes-dashboard-certs
+        secret:
+          secretName: kubernetes-dashboard-certs
+      - emptyDir: {}
+        name: tmp-volume

k8s/dashboard-with-token.yaml

@@ -1,151 +1,189 @@
-# This file is based on the following manifest:
-# https://github.com/kubernetes/dashboard/blob/master/aio/deploy/recommended.yaml
-# It adds a ServiceAccount that has cluster-admin privileges on the cluster,
-# and exposes the dashboard on a NodePort. It makes it easier to do quick demos
-# of the Kubernetes dashboard, without compromising the security too much.
-
-# Copyright 2017 The Kubernetes Authors.
+# This file was generated with the script ./update-dashboard-yaml.sh.
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
+---
 apiVersion: v1
 kind: Namespace
 metadata:
+  creationTimestamp: null
   name: kubernetes-dashboard
-
+spec: {}
+status: {}
 ---
-
 apiVersion: v1
 kind: ServiceAccount
 metadata:
+  annotations: null
   labels:
-    k8s-app: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
   name: kubernetes-dashboard
   namespace: kubernetes-dashboard
-
 ---
-
-kind: Service
-apiVersion: v1
-metadata:
-  labels:
-    k8s-app: kubernetes-dashboard
-  name: kubernetes-dashboard
-  namespace: kubernetes-dashboard
-spec:
-  type: NodePort
-  ports:
-    - port: 443
-      targetPort: 8443
-  selector:
-    k8s-app: kubernetes-dashboard
-
----
-
 apiVersion: v1
 kind: Secret
 metadata:
+  annotations: null
   labels:
-    k8s-app: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
   name: kubernetes-dashboard-certs
   namespace: kubernetes-dashboard
 type: Opaque
-
 ---
-
 apiVersion: v1
 kind: Secret
 metadata:
   labels:
-    k8s-app: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
   name: kubernetes-dashboard-csrf
   namespace: kubernetes-dashboard
 type: Opaque
-data:
-  csrf: ""
-
 ---
-
 apiVersion: v1
 kind: Secret
 metadata:
   labels:
-    k8s-app: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
   name: kubernetes-dashboard-key-holder
   namespace: kubernetes-dashboard
 type: Opaque
-
 ---
-
-kind: ConfigMap
 apiVersion: v1
+data: null
+kind: ConfigMap
 metadata:
+  annotations: null
   labels:
-    k8s-app: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
   name: kubernetes-dashboard-settings
   namespace: kubernetes-dashboard
-
 ---
-
-kind: Role
 apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
 metadata:
+  annotations: null
   labels:
-    k8s-app: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
+  name: kubernetes-dashboard-metrics
+rules:
+- apiGroups:
+  - metrics.k8s.io
+  resources:
+  - pods
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  annotations: null
+  labels:
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
+  name: kubernetes-dashboard-metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubernetes-dashboard-metrics
+subjects:
+- kind: ServiceAccount
+  name: kubernetes-dashboard
+  namespace: kubernetes-dashboard
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  annotations: null
+  labels:
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
   name: kubernetes-dashboard
   namespace: kubernetes-dashboard
 rules:
-  # Allow Dashboard to get, update and delete Dashboard exclusive secrets.
-  - apiGroups: [""]
-    resources: ["secrets"]
-    resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]
-    verbs: ["get", "update", "delete"]
-    # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
-  - apiGroups: [""]
-    resources: ["configmaps"]
-    resourceNames: ["kubernetes-dashboard-settings"]
-    verbs: ["get", "update"]
-    # Allow Dashboard to get metrics.
-  - apiGroups: [""]
-    resources: ["services"]
-    resourceNames: ["heapster", "dashboard-metrics-scraper"]
-    verbs: ["proxy"]
-  - apiGroups: [""]
-    resources: ["services/proxy"]
-    resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]
-    verbs: ["get"]
-
+- apiGroups:
+  - ""
+  resourceNames:
+  - kubernetes-dashboard-key-holder
+  - kubernetes-dashboard-certs
+  - kubernetes-dashboard-csrf
+  resources:
+  - secrets
+  verbs:
+  - get
+  - update
+  - delete
+- apiGroups:
+  - ""
+  resourceNames:
+  - kubernetes-dashboard-settings
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - update
+- apiGroups:
+  - ""
+  resourceNames:
+  - heapster
+  - dashboard-metrics-scraper
+  resources:
+  - services
+  verbs:
+  - proxy
+- apiGroups:
+  - ""
+  resourceNames:
+  - heapster
+  - 'http:heapster:'
+  - 'https:heapster:'
+  - dashboard-metrics-scraper
+  - http:dashboard-metrics-scraper
+  resources:
+  - services/proxy
+  verbs:
+  - get
 ---
-
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  labels:
-    k8s-app: kubernetes-dashboard
-  name: kubernetes-dashboard
-rules:
-  # Allow Metrics Scraper to get metrics from the Metrics server
-  - apiGroups: ["metrics.k8s.io"]
-    resources: ["pods", "nodes"]
-    verbs: ["get", "list", "watch"]
-
----
-
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
+  annotations: null
   labels:
-    k8s-app: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
   name: kubernetes-dashboard
   namespace: kubernetes-dashboard
 roleRef:
@@ -153,179 +191,144 @@ roleRef:
   kind: Role
   name: kubernetes-dashboard
 subjects:
-  - kind: ServiceAccount
-    name: kubernetes-dashboard
-    namespace: kubernetes-dashboard
-
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: kubernetes-dashboard
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: kubernetes-dashboard
-subjects:
-  - kind: ServiceAccount
-    name: kubernetes-dashboard
-    namespace: kubernetes-dashboard
-
----
-
-kind: Deployment
-apiVersion: apps/v1
-metadata:
-  labels:
-    k8s-app: kubernetes-dashboard
+- kind: ServiceAccount
   name: kubernetes-dashboard
   namespace: kubernetes-dashboard
-spec:
-  replicas: 1
-  revisionHistoryLimit: 10
-  selector:
-    matchLabels:
-      k8s-app: kubernetes-dashboard
-  template:
-    metadata:
-      labels:
-        k8s-app: kubernetes-dashboard
-    spec:
-      containers:
-        - name: kubernetes-dashboard
-          image: kubernetesui/dashboard:v2.0.0
-          imagePullPolicy: Always
-          ports:
-            - containerPort: 8443
-              protocol: TCP
-          args:
-            - --auto-generate-certificates
-            - --namespace=kubernetes-dashboard
-            # Uncomment the following line to manually specify Kubernetes API server Host
-            # If not specified, Dashboard will attempt to auto discover the API server and connect
-            # to it. Uncomment only if the default does not work.
-            # - --apiserver-host=http://my-address:port
-          volumeMounts:
-            - name: kubernetes-dashboard-certs
-              mountPath: /certs
-              # Create on-disk volume to store exec logs
-            - mountPath: /tmp
-              name: tmp-volume
-          livenessProbe:
-            httpGet:
-              scheme: HTTPS
-              path: /
-              port: 8443
-            initialDelaySeconds: 30
-            timeoutSeconds: 30
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            runAsUser: 1001
-            runAsGroup: 2001
-      volumes:
-        - name: kubernetes-dashboard-certs
-          secret:
-            secretName: kubernetes-dashboard-certs
-        - name: tmp-volume
-          emptyDir: {}
-      serviceAccountName: kubernetes-dashboard
-      nodeSelector:
-        "kubernetes.io/os": linux
-      # Comment the following tolerations if Dashboard must not be deployed on master
-      tolerations:
-        - key: node-role.kubernetes.io/master
-          effect: NoSchedule
-
 ---
-
-kind: Service
 apiVersion: v1
+kind: Service
 metadata:
+  annotations: null
   labels:
-    k8s-app: dashboard-metrics-scraper
-  name: dashboard-metrics-scraper
+    app.kubernetes.io/component: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
+    kubernetes.io/cluster-service: "true"
+  name: kubernetes-dashboard
   namespace: kubernetes-dashboard
 spec:
   ports:
-    - port: 8000
-      targetPort: 8000
+  - name: https
+    port: 443
+    targetPort: https
   selector:
-    k8s-app: dashboard-metrics-scraper
-
+    app.kubernetes.io/component: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/name: kubernetes-dashboard
+  type: NodePort
 ---
-
-kind: Deployment
 apiVersion: apps/v1
+kind: Deployment
 metadata:
+  annotations: null
   labels:
-    k8s-app: dashboard-metrics-scraper
-  name: dashboard-metrics-scraper
+    app.kubernetes.io/component: kubernetes-dashboard
+    app.kubernetes.io/instance: kubernetes-dashboard
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kubernetes-dashboard
+    app.kubernetes.io/version: 2.5.0
+    helm.sh/chart: kubernetes-dashboard-5.2.0
+  name: kubernetes-dashboard
   namespace: kubernetes-dashboard
 spec:
   replicas: 1
-  revisionHistoryLimit: 10
   selector:
     matchLabels:
-      k8s-app: dashboard-metrics-scraper
+      app.kubernetes.io/component: kubernetes-dashboard
+      app.kubernetes.io/instance: kubernetes-dashboard
+      app.kubernetes.io/name: kubernetes-dashboard
+  strategy:
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+    type: RollingUpdate
   template:
     metadata:
+      annotations: null
       labels:
-        k8s-app: dashboard-metrics-scraper
-      annotations:
-        seccomp.security.alpha.kubernetes.io/pod: 'runtime/default'
+        app.kubernetes.io/component: kubernetes-dashboard
+        app.kubernetes.io/instance: kubernetes-dashboard
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/name: kubernetes-dashboard
+        app.kubernetes.io/version: 2.5.0
+        helm.sh/chart: kubernetes-dashboard-5.2.0
     spec:
       containers:
-        - name: dashboard-metrics-scraper
-          image: kubernetesui/metrics-scraper:v1.0.4
-          ports:
-            - containerPort: 8000
-              protocol: TCP
-          livenessProbe:
-            httpGet:
-              scheme: HTTP
-              path: /
-              port: 8000
-            initialDelaySeconds: 30
-            timeoutSeconds: 30
-          volumeMounts:
-          - mountPath: /tmp
-            name: tmp-volume
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            runAsUser: 1001
-            runAsGroup: 2001
+      - args:
+        - --namespace=kubernetes-dashboard
+        - --auto-generate-certificates
+        - --sidecar-host=http://127.0.0.1:8000
+        image: kubernetesui/dashboard:v2.5.0
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 8443
+            scheme: HTTPS
+          initialDelaySeconds: 30
+          timeoutSeconds: 30
+        name: kubernetes-dashboard
+        ports:
+        - containerPort: 8443
+          name: https
+          protocol: TCP
+        resources:
+          limits:
+            cpu: 2
+            memory: 200Mi
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsGroup: 2001
+          runAsUser: 1001
+        volumeMounts:
+        - mountPath: /certs
+          name: kubernetes-dashboard-certs
+        - mountPath: /tmp
+          name: tmp-volume
+      - image: kubernetesui/metrics-scraper:v1.0.7
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 8000
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 30
+        name: dashboard-metrics-scraper
+        ports:
+        - containerPort: 8000
+          protocol: TCP
+        securityContext:
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          runAsGroup: 2001
+          runAsUser: 1001
+        volumeMounts:
+        - mountPath: /tmp
+          name: tmp-volume
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
       serviceAccountName: kubernetes-dashboard
-      nodeSelector:
-        "kubernetes.io/os": linux
-      # Comment the following tolerations if Dashboard must not be deployed on master
-      tolerations:
-        - key: node-role.kubernetes.io/master
-          effect: NoSchedule
       volumes:
-        - name: tmp-volume
-          emptyDir: {}
-
+      - name: kubernetes-dashboard-certs
+        secret:
+          secretName: kubernetes-dashboard-certs
+      - emptyDir: {}
+        name: tmp-volume
 ---
-
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  labels:
-    k8s-app: kubernetes-dashboard
-  name: cluster-admin
-  namespace: kubernetes-dashboard
-
----
-
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  labels:
-    k8s-app: kubernetes-dashboard
-  name: kubernetes-dashboard-cluster-admin
+  creationTimestamp: null
+  name: kubernetes-dashboard:cluster-admin
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
@@ -334,3 +337,10 @@ subjects:
 - kind: ServiceAccount
   name: cluster-admin
   namespace: kubernetes-dashboard
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  name: cluster-admin
+  namespace: kubernetes-dashboard

k8s/haproxy.cfg

@@ -1,18 +1,16 @@
 global
   daemon
-  maxconn 256
 
 defaults
   mode tcp
-  timeout connect 5000ms
-  timeout client 50000ms
-  timeout server 50000ms
+  timeout connect 5s
+  timeout client 50s
+  timeout server 50s
 
-frontend the-frontend
+listen very-basic-load-balancer
   bind *:80
-  default_backend the-backend
-
-backend the-backend
-  server google.com-80 google.com:80 maxconn 32 check
-  server ibm.fr-80 ibm.fr:80 maxconn 32 check
+  server blue color.blue.svc:80
+  server green color.green.svc:80
 
+# Note: the services above must exist,
+# otherwise HAproxy won't start.

k8s/kyverno-ingress-domain-name-1.yaml

@@ -0,0 +1,28 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: ingress-domain-name
+spec:
+  rules:
+  - name: create-ingress
+    match:
+      resources: 
+        kinds:
+        - Service
+    generate: 
+      kind: Ingress
+      name: "{{request.object.metadata.name}}"
+      namespace: "{{request.object.metadata.namespace}}"
+      data:
+        spec:
+          rules:
+          - host: "{{request.object.metadata.name}}.{{request.object.metadata.namespace}}.A.B.C.D.nip.io"
+            http:
+              paths:
+              - backend:
+                  service:
+                    name: "{{request.object.metadata.name}}"
+                    port:
+                      number: 80
+                path: /
+                pathType: Prefix

k8s/kyverno-ingress-domain-name-2a.yaml

@@ -0,0 +1,32 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: ingress-domain-name
+spec:
+  rules:
+  - name: create-ingress
+    match:
+      resources: 
+        kinds:
+        - Service
+    preconditions:
+    - key: "{{request.object.spec.ports[0].name}}"
+      operator: Equals
+      value: http
+    generate: 
+      kind: Ingress
+      name: "{{request.object.metadata.name}}"
+      namespace: "{{request.object.metadata.namespace}}"
+      data:
+        spec:
+          rules:
+          - host: "{{request.object.metadata.name}}.{{request.object.metadata.namespace}}.A.B.C.D.nip.io"
+            http:
+              paths:
+              - backend:
+                  service:
+                    name: "{{request.object.metadata.name}}"
+                    port:
+                      name: http
+                path: /
+                pathType: Prefix

k8s/kyverno-ingress-domain-name-2b.yaml

@@ -0,0 +1,32 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: ingress-domain-name
+spec:
+  rules:
+  - name: create-ingress
+    match:
+      resources: 
+        kinds:
+        - Service
+    preconditions:
+      - key: http
+        operator: In
+        value: "{{request.object.spec.ports[*].name}}"
+    generate: 
+      kind: Ingress
+      name: "{{request.object.metadata.name}}"
+      namespace: "{{request.object.metadata.namespace}}"
+      data:
+        spec:
+          rules:
+          - host: "{{request.object.metadata.name}}.{{request.object.metadata.namespace}}.A.B.C.D.nip.io"
+            http:
+              paths:
+              - backend:
+                  service:
+                    name: "{{request.object.metadata.name}}"
+                    port:
+                      name: http
+                path: /
+                pathType: Prefix

k8s/kyverno-ingress-domain-name-2c.yaml

@@ -0,0 +1,34 @@
+# Note: this policy uses the operator "AnyIn", which was introduced in Kyverno 1.6.
+# (This policy won't work with Kyverno 1.5!)
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: ingress-domain-name
+spec:
+  rules:
+  - name: create-ingress
+    match:
+      resources: 
+        kinds:
+        - Service
+    preconditions:
+      - key: "{{request.object.spec.ports[*].port}}"
+        operator: AnyIn
+        value: [ 80 ]
+    generate: 
+      kind: Ingress
+      name: "{{request.object.metadata.name}}"
+      namespace: "{{request.object.metadata.namespace}}"
+      data:
+        spec:
+          rules:
+          - host: "{{request.object.metadata.name}}.{{request.object.metadata.namespace}}.A.B.C.D.nip.io"
+            http:
+              paths:
+              - backend:
+                  service:
+                    name: "{{request.object.metadata.name}}"
+                    port:
+                      name: http
+                path: /
+                pathType: Prefix

k8s/kyverno-ingress-domain-name-3.yaml

@@ -0,0 +1,37 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: ingress-domain-name
+spec:
+  rules:
+  - name: create-ingress
+    context:
+    - name: configmap
+      configMap:
+        name: ingress-domain-name
+        namespace: "{{request.object.metadata.namespace}}"
+    match:
+      resources: 
+        kinds:
+        - Service
+    preconditions:
+    - key: "{{request.object.spec.ports[0].name}}"
+      operator: Equals
+      value: http
+    generate: 
+      kind: Ingress
+      name: "{{request.object.metadata.name}}"
+      namespace: "{{request.object.metadata.namespace}}"
+      data:
+        spec:
+          rules:
+          - host: "{{request.object.metadata.name}}.{{request.object.metadata.namespace}}.{{configmap.data.domain}}"
+            http:
+              paths:
+              - backend:
+                  service:
+                    name: "{{request.object.metadata.name}}"
+                    port:
+                      name: http
+                path: /
+                pathType: Prefix

k8s/kyverno-pod-color-2.yaml

@@ -11,11 +11,21 @@ spec:
       resources:
         kinds:
         - Pod
+    preconditions:
+    - key: "{{ request.operation }}"
+      operator: Equals
+      value: UPDATE
+    - key: "{{ request.oldObject.metadata.labels.color }}"
+      operator: NotEquals
+      value: ""
+    - key: "{{ request.object.metadata.labels.color }}"
+      operator: NotEquals
+      value: ""
     validate:
       message: "Once label color has been added, it cannot be changed."
       deny:
         conditions:
-        - key: "{{ request.oldObject.metadata.labels.color }}"
-          operator: NotEqual
-          value: "{{ request.object.metadata.labels.color }}"
+        - key: "{{ request.object.metadata.labels.color }}"
+          operator: NotEquals
+          value: "{{ request.oldObject.metadata.labels.color }}"
 

k8s/kyverno-pod-color-3.yaml

@@ -6,20 +6,23 @@ spec:
   validationFailureAction: enforce
   background: false
   rules:
-  - name: prevent-color-removal
+  - name: prevent-color-change
     match:
       resources:
         kinds:
         - Pod
-        selector:
-          matchExpressions:
-          - key: color
-            operator: DoesNotExist
+    preconditions:
+    - key: "{{ request.operation }}"
+      operator: Equals
+      value: UPDATE
+    - key: "{{ request.oldObject.metadata.labels.color }}"
+      operator: NotEquals
+      value: ""
+    - key: "{{ request.object.metadata.labels.color }}"
+      operator: Equals
+      value: ""
     validate:
       message: "Once label color has been added, it cannot be removed."
       deny:
         conditions:
-        - key: "{{ request.oldObject.metadata.labels.color }}"
-          operator: NotIn
-          value: []
 

k8s/kyverno-tls-for-ingress.yaml

@@ -0,0 +1,46 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: tls-for-ingress
+spec:
+  rules:
+  - name: create-role
+    match:
+      resources: 
+        kinds:
+        - Certificate
+    generate: 
+      kind: Role
+      apiVersion: rbac.authorization.k8s.io/v1
+      name: "{{request.object.metadata.name}}"
+      namespace: "{{request.object.metadata.namespace}}"
+      data:
+        rules:
+        - verbs:
+          - get
+          apiGroups:
+          - ""
+          resources:
+          - secrets
+          resourceNames:
+          - "{{request.object.metadata.name}}"
+  - name: create-rolebinding
+    match:
+      resources: 
+        kinds:
+        - Certificate
+    generate: 
+      kind: RoleBinding
+      apiVersion: rbac.authorization.k8s.io/v1
+      name: "{{request.object.metadata.name}}"
+      namespace: "{{request.object.metadata.namespace}}"
+      data:
+        roleRef:
+          apiGroup: rbac.authorization.k8s.io
+          kind: Role
+          name: "{{request.object.metadata.name}}"
+        subjects:
+        - kind: ServiceAccount
+          name: default
+          namespace: "{{request.object.metadata.namespace}}"
+

k8s/mounter.yaml

@@ -0,0 +1,20 @@
+kind: Pod
+apiVersion: v1
+metadata:
+  generateName: mounter-
+  labels:
+    container.training/mounter: ""
+spec:
+  volumes:
+  - name: pvc
+    persistentVolumeClaim:
+      claimName: my-pvc-XYZ45
+  containers:
+  - name: mounter
+    image: alpine
+    stdin: true
+    tty: true
+    volumeMounts:
+    - name: pvc
+      mountPath: /pvc
+    workingDir: /pvc

k8s/netpol-dockercoins.yaml

@@ -3,8 +3,7 @@ apiVersion: networking.k8s.io/v1
 metadata:
   name: deny-from-other-namespaces
 spec:
-  podSelector:
-    matchLabels:
+  podSelector: {}
   ingress:
   - from:
     - podSelector: {}

k8s/pizza-1.yaml

@@ -0,0 +1,14 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: pizzas.container.training
+spec:
+  group: container.training
+  version: v1alpha1
+  scope: Namespaced
+  names:
+    plural: pizzas
+    singular: pizza
+    kind: Pizza
+    shortNames:
+    - piz

k8s/pizza-2.yaml

@@ -0,0 +1,20 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: pizzas.container.training
+spec:
+  group: container.training
+  scope: Namespaced
+  names:
+    plural: pizzas
+    singular: pizza
+    kind: Pizza
+    shortNames:
+    - piz
+  versions:
+  - name: v1alpha1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object

k8s/pizza-3.yaml

@@ -0,0 +1,32 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: pizzas.container.training
+spec:
+  group: container.training
+  scope: Namespaced
+  names:
+    plural: pizzas
+    singular: pizza
+    kind: Pizza
+    shortNames:
+    - piz
+  versions:
+  - name: v1alpha1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        required: [ spec ]
+        properties:
+          spec:
+            type: object
+            required: [ sauce, toppings ]
+            properties:
+              sauce:
+                type: string
+              toppings:
+                type: array
+                items:
+                  type: string

k8s/pizza-4.yaml

@@ -0,0 +1,39 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: pizzas.container.training
+spec:
+  group: container.training
+  scope: Namespaced
+  names:
+    plural: pizzas
+    singular: pizza
+    kind: Pizza
+    shortNames:
+    - piz
+  versions:
+  - name: v1alpha1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        required: [ spec ]
+        properties:
+          spec:
+            type: object
+            required: [ sauce, toppings ]
+            properties:
+              sauce:
+                type: string
+              toppings:
+                type: array
+                items:
+                  type: string
+    additionalPrinterColumns:
+    - jsonPath: .spec.sauce
+      name: Sauce
+      type: string
+    - jsonPath: .spec.toppings
+      name: Toppings
+      type: string

k8s/pizza-5.yaml

@@ -0,0 +1,40 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: pizzas.container.training
+spec:
+  group: container.training
+  scope: Namespaced
+  names:
+    plural: pizzas
+    singular: pizza
+    kind: Pizza
+    shortNames:
+    - piz
+  versions:
+  - name: v1alpha1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        required: [ spec ]
+        properties:
+          spec:
+            type: object
+            required: [ sauce, toppings ]
+            properties:
+              sauce:
+                type: string
+                enum: [ red, white ]
+              toppings:
+                type: array
+                items:
+                  type: string
+    additionalPrinterColumns:
+    - jsonPath: .spec.sauce
+      name: Sauce
+      type: string
+    - jsonPath: .spec.toppings
+      name: Toppings
+      type: string

k8s/pizzas.yaml

@@ -0,0 +1,45 @@
+---
+apiVersion: container.training/v1alpha1
+kind: Pizza
+metadata:
+  name: margherita
+spec:
+  sauce: red
+  toppings:
+    - mozarella
+    - basil
+---
+apiVersion: container.training/v1alpha1
+kind: Pizza
+metadata:
+  name: quatrostagioni
+spec:
+  sauce: red
+  toppings:
+    - artichoke
+    - basil
+    - mushrooms
+    - prosciutto
+---
+apiVersion: container.training/v1alpha1
+kind: Pizza
+metadata:
+  name: mehl31
+spec:
+  sauce: white
+  toppings:
+    - goatcheese
+    - pear
+    - walnuts
+    - mozzarella
+    - rosemary
+    - honey
+---
+apiVersion: container.training/v1alpha1
+kind: Pizza
+metadata:
+  name: brownie
+spec:
+  sauce: chocolate
+  toppings:
+    - nuts

k8s/pv.yaml

@@ -0,0 +1,20 @@
+kind: PersistentVolume
+apiVersion: v1
+metadata:
+  generateName: my-pv-
+  labels:
+    container.training/pv: ""
+spec:
+  accessModes:
+  - ReadWriteOnce
+  - ReadWriteMany
+  capacity:
+    storage: 1G
+  hostPath:
+    path: /tmp/my-pv
+  #storageClassName: my-sc
+  #claimRef:
+  #  kind: PersistentVolumeClaim
+  #  apiVersion: v1
+  #  namespace: default
+  #  name: my-pvc-XYZ45

k8s/pvc.yaml

@@ -0,0 +1,13 @@
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  generateName: my-pvc-
+  labels:
+    container.training/pvc: ""
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1G
+  #storageClassName: my-sc

k8s/rainbow.yaml

@@ -0,0 +1,147 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: blue
+  labels:
+    app: rainbow
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: rainbow
+    color: blue
+  name: color
+  namespace: blue
+spec:
+  selector:
+    matchLabels:
+      app: rainbow
+      color: blue
+  template:
+    metadata:
+      labels:
+        app: rainbow
+        color: blue
+    spec:
+      containers:
+      - image: jpetazzo/color
+        name: color
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: rainbow
+    color: blue
+  name: color
+  namespace: blue
+spec:
+  ports:
+  - name: http
+    port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    app: rainbow
+    color: blue
+  type: ClusterIP
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: green
+  labels:
+    app: rainbow
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: rainbow
+    color: green
+  name: color
+  namespace: green
+spec:
+  selector:
+    matchLabels:
+      app: rainbow
+      color: green
+  template:
+    metadata:
+      labels:
+        app: rainbow
+        color: green
+    spec:
+      containers:
+      - image: jpetazzo/color
+        name: color
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: rainbow
+    color: green
+  name: color
+  namespace: green
+spec:
+  ports:
+  - name: http
+    port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    app: rainbow
+    color: green
+  type: ClusterIP
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: red
+  labels:
+    app: rainbow
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: rainbow
+    color: red
+  name: color
+  namespace: red
+spec:
+  selector:
+    matchLabels:
+      app: rainbow
+      color: red
+  template:
+    metadata:
+      labels:
+        app: rainbow
+        color: red
+    spec:
+      containers:
+      - image: jpetazzo/color
+        name: color
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: rainbow
+    color: red
+  name: color
+  namespace: red
+spec:
+  ports:
+  - name: http
+    port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    app: rainbow
+    color: red
+  type: ClusterIP

k8s/test.yaml

@@ -1,17 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: Ingress
-metadata:
-  name: whatever
-spec:
-  #tls:
-  #- secretName: whatever.A.B.C.D.nip.io
-  #  hosts:
-  #  - whatever.A.B.C.D.nip.io
-  rules:
-  - host: whatever.nip.io
-    http:
-      paths:
-      - path: /
-        backend:
-          serviceName: whatever
-          servicePort: 1234

k8s/tilt-registry.yaml

@@ -0,0 +1,42 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: tilt-registry
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: tilt-registry
+  name: tilt-registry
+  namespace: tilt-registry
+spec:
+  selector:
+    matchLabels:
+      app: tilt-registry
+  template:
+    metadata:
+      labels:
+        app: tilt-registry
+    spec:
+      containers:
+      - image: registry
+        name: registry
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: tilt-registry
+  name: tilt-registry
+  namespace: tilt-registry
+spec:
+  ports:
+  - port: 5000
+    protocol: TCP
+    targetPort: 5000
+    nodePort: 30555
+  selector:
+    app: tilt-registry
+  type: NodePort

k8s/traefik-v2.yaml

@@ -35,6 +35,9 @@ spec:
         - name: http
           containerPort: 80
           hostPort: 80
+        - name: https
+          containerPort: 443
+          hostPort: 443
         - name: admin
           containerPort: 8080
           hostPort: 8080
@@ -100,3 +103,12 @@ subjects:
 - kind: ServiceAccount
   name: traefik-ingress-controller
   namespace: kube-system
+---
+kind: IngressClass
+apiVersion: networking.k8s.io/v1
+metadata:
+  name: traefik
+  annotations:
+    ingressclass.kubernetes.io/is-default-class: "true"
+spec:
+  controller: traefik.io/ingress-controller

k8s/update-dashboard-yaml.sh

@@ -0,0 +1,73 @@
+#!/bin/sh
+
+banner() {
+  echo "# This file was generated with the script $0."
+  echo "#"
+}
+
+create_namespace() {
+  # 'helm template --namespace ... --create-namespace'
+  # doesn't create the namespace, so we need to create it.
+  # https://github.com/helm/helm/issues/9813
+  echo ---
+  kubectl create namespace kubernetes-dashboard \
+          -o yaml --dry-run=client
+  echo ---
+}
+
+add_namespace() {
+  # 'helm template --namespace ...' doesn't add namespace information,
+  # so we do it with this convenient filter instead.
+  # https://github.com/helm/helm/issues/10737
+  kubectl create -f- -o yaml --dry-run=client --namespace kubernetes-dashboard
+}
+
+(
+  banner
+  create_namespace
+  helm template kubernetes-dashboard kubernetes-dashboard \
+       --repo https://kubernetes.github.io/dashboard/ \
+       --create-namespace --namespace kubernetes-dashboard \
+       --set "extraArgs={--enable-skip-login,--enable-insecure-login}" \
+       --set metricsScraper.enabled=true \
+       --set protocolHttp=true \
+       --set service.type=NodePort \
+       | add_namespace
+  echo ---
+  kubectl create clusterrolebinding kubernetes-dashboard:insecure \
+          --clusterrole=cluster-admin \
+          --serviceaccount=kubernetes-dashboard:kubernetes-dashboard \
+          -o yaml --dry-run=client \
+          #
+) > dashboard-insecure.yaml
+
+(
+  banner
+  create_namespace
+  helm template kubernetes-dashboard kubernetes-dashboard \
+       --repo https://kubernetes.github.io/dashboard/ \
+       --create-namespace --namespace kubernetes-dashboard \
+       --set metricsScraper.enabled=true \
+       | add_namespace
+) > dashboard-recommended.yaml
+
+(
+  banner
+  create_namespace
+  helm template kubernetes-dashboard kubernetes-dashboard \
+       --repo https://kubernetes.github.io/dashboard/ \
+       --create-namespace --namespace kubernetes-dashboard \
+       --set metricsScraper.enabled=true \
+       --set service.type=NodePort \
+       | add_namespace
+  echo ---
+  kubectl create clusterrolebinding kubernetes-dashboard:cluster-admin \
+          --clusterrole=cluster-admin \
+          --serviceaccount=kubernetes-dashboard:cluster-admin \
+          -o yaml --dry-run=client \
+          #
+  echo ---
+  kubectl create serviceaccount -n kubernetes-dashboard cluster-admin \
+          -o yaml --dry-run=client \
+          # 
+) > dashboard-with-token.yaml

k8s/ytt/1-variables/app.yaml

@@ -0,0 +1,164 @@
+#! Define and use variables.
+---
+#@ repository = "dockercoins"
+#@ tag = "v0.1"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: hasher
+  name: hasher
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: hasher
+  template:
+    metadata:
+      labels:
+        app: hasher
+    spec:
+      containers:
+      - image: #@ "{}/hasher:{}".format(repository, tag)
+        name: hasher
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: hasher
+  name: hasher
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    app: hasher
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: redis
+  name: redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      containers:
+      - image: redis
+        name: redis
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: redis
+  name: redis
+spec:
+  ports:
+  - port: 6379
+    protocol: TCP
+    targetPort: 6379
+  selector:
+    app: redis
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: rng
+  name: rng
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: rng
+  template:
+    metadata:
+      labels:
+        app: rng
+    spec:
+      containers:
+      - image: #@ "{}/rng:{}".format(repository, tag)
+        name: rng
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: rng
+  name: rng
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    app: rng
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: webui
+  name: webui
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: webui
+  template:
+    metadata:
+      labels:
+        app: webui
+    spec:
+      containers:
+      - image: #@ "{}/webui:{}".format(repository, tag)
+        name: webui
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: webui
+  name: webui
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    app: webui
+  type: NodePort
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: worker
+  name: worker
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: worker
+  template:
+    metadata:
+      labels:
+        app: worker
+    spec:
+      containers:
+      - image: #@ "{}/worker:{}".format(repository, tag)
+        name: worker

k8s/ytt/2-functions/app.yaml

@@ -0,0 +1,167 @@
+#! Define and use a function to set the deployment image.
+---
+#@ repository = "dockercoins"
+#@ tag = "v0.1"
+#@ def image(component):
+#@   return "{}/{}:{}".format(repository, component, tag)
+#@ end
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: hasher
+  name: hasher
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: hasher
+  template:
+    metadata:
+      labels:
+        app: hasher
+    spec:
+      containers:
+      - image: #@ image("hasher")
+        name: hasher
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: hasher
+  name: hasher
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    app: hasher
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: redis
+  name: redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      containers:
+      - image: redis
+        name: redis
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: redis
+  name: redis
+spec:
+  ports:
+  - port: 6379
+    protocol: TCP
+    targetPort: 6379
+  selector:
+    app: redis
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: rng
+  name: rng
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: rng
+  template:
+    metadata:
+      labels:
+        app: rng
+    spec:
+      containers:
+      - image: #@ image("rng")
+        name: rng
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: rng
+  name: rng
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    app: rng
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: webui
+  name: webui
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: webui
+  template:
+    metadata:
+      labels:
+        app: webui
+    spec:
+      containers:
+      - image: #@ image("webui")
+        name: webui
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: webui
+  name: webui
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    app: webui
+  type: NodePort
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: worker
+  name: worker
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: worker
+  template:
+    metadata:
+      labels:
+        app: worker
+    spec:
+      containers:
+      - image: #@ image("worker")
+        name: worker

k8s/ytt/3-labels/app.yaml

@@ -0,0 +1,164 @@
+#! Define and use functions, demonstrating how to generate labels.
+---
+#@ repository = "dockercoins"
+#@ tag = "v0.1"
+#@ def image(component):
+#@   return "{}/{}:{}".format(repository, component, tag)
+#@ end
+#@ def labels(component):
+#@   return {
+#@     "app": component,
+#@     "container.training/generated-by": "ytt",
+#@   }
+#@ end
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels: #@ labels("hasher")
+  name: hasher
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: hasher
+  template:
+    metadata:
+      labels:
+        app: hasher
+    spec:
+      containers:
+      - image: #@ image("hasher")
+        name: hasher
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels: #@ labels("hasher")
+  name: hasher
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    app: hasher
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels: #@ labels("redis")
+  name: redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      containers:
+      - image: redis
+        name: redis
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels: #@ labels("redis")
+  name: redis
+spec:
+  ports:
+  - port: 6379
+    protocol: TCP
+    targetPort: 6379
+  selector:
+    app: redis
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels: #@ labels("rng")
+  name: rng
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: rng
+  template:
+    metadata:
+      labels:
+        app: rng
+    spec:
+      containers:
+      - image: #@ image("rng")
+        name: rng
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels: #@ labels("rng")
+  name: rng
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    app: rng
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels: #@ labels("webui")
+  name: webui
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: webui
+  template:
+    metadata:
+      labels:
+        app: webui
+    spec:
+      containers:
+      - image: #@ image("webui")
+        name: webui
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels: #@ labels("webui")
+  name: webui
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    app: webui
+  type: NodePort
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels: #@ labels("worker")
+  name: worker
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: worker
+  template:
+    metadata:
+      labels:
+        app: worker
+    spec:
+      containers:
+      - image: #@ image("worker")
+        name: worker

k8s/ytt/4-data/app.yaml

@@ -0,0 +1,162 @@
+---
+#@ load("@ytt:data", "data")
+#@ def image(component):
+#@   return "{}/{}:{}".format(data.values.repository, component, data.values.tag)
+#@ end
+#@ def labels(component):
+#@   return {
+#@     "app": component,
+#@     "container.training/generated-by": "ytt",
+#@   }
+#@ end
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels: #@ labels("hasher")
+  name: hasher
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: hasher
+  template:
+    metadata:
+      labels:
+        app: hasher
+    spec:
+      containers:
+      - image: #@ image("hasher")
+        name: hasher
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels: #@ labels("hasher")
+  name: hasher
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    app: hasher
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels: #@ labels("redis")
+  name: redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      containers:
+      - image: redis
+        name: redis
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels: #@ labels("redis")
+  name: redis
+spec:
+  ports:
+  - port: 6379
+    protocol: TCP
+    targetPort: 6379
+  selector:
+    app: redis
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels: #@ labels("rng")
+  name: rng
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: rng
+  template:
+    metadata:
+      labels:
+        app: rng
+    spec:
+      containers:
+      - image: #@ image("rng")
+        name: rng
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels: #@ labels("rng")
+  name: rng
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    app: rng
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels: #@ labels("webui")
+  name: webui
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: webui
+  template:
+    metadata:
+      labels:
+        app: webui
+    spec:
+      containers:
+      - image: #@ image("webui")
+        name: webui
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels: #@ labels("webui")
+  name: webui
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    app: webui
+  type: NodePort
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels: #@ labels("worker")
+  name: worker
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: worker
+  template:
+    metadata:
+      labels:
+        app: worker
+    spec:
+      containers:
+      - image: #@ image("worker")
+        name: worker

k8s/ytt/4-data/schema.yaml

@@ -0,0 +1,4 @@
+#@data/values-schema
+---
+repository: dockercoins
+tag: v0.1

k8s/ytt/5-factor/app.yaml

@@ -0,0 +1,54 @@
+---
+#@ load("@ytt:data", "data")
+---
+#@ def Deployment(component, repository=data.values.repository, tag=data.values.tag):
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: #@ component
+    container.training/generated-by: ytt
+  name: #@ component
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: #@ component
+  template:
+    metadata:
+      labels:
+        app: #@ component
+    spec:
+      containers:
+      - image: #@ repository + "/" + component + ":" + tag
+        name: #@ component
+#@ end
+---
+#@ def Service(component, port=80, type="ClusterIP"):
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: #@ component
+    container.training/generated-by: ytt
+  name: #@ component
+spec:
+  ports:
+  - port: #@ port
+    protocol: TCP
+    targetPort: #@ port
+  selector:
+    app: #@ component
+  type: #@ type
+#@ end
+---
+--- #@ Deployment("hasher")
+--- #@ Service("hasher")
+--- #@ Deployment("redis", repository="library", tag="latest")
+--- #@ Service("redis", port=6379)
+--- #@ Deployment("rng")
+--- #@ Service("rng")
+--- #@ Deployment("webui")
+--- #@ Service("webui", type="NodePort")
+--- #@ Deployment("worker")
+---

k8s/ytt/5-factor/schema.yaml

@@ -0,0 +1,4 @@
+#@data/values-schema
+---
+repository: dockercoins
+tag: v0.1

k8s/ytt/6-template/app.yaml

@@ -0,0 +1,56 @@
+---
+#@ load("@ytt:data", "data")
+#@ load("@ytt:template", "template")
+---
+#@ def component(name, repository=data.values.repository, tag=data.values.tag, port=None, type="ClusterIP"):
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: #@ name
+    container.training/generated-by: ytt
+  name: #@ name
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: #@ name
+  template:
+    metadata:
+      labels:
+        app: #@ name
+    spec:
+      containers:
+      - image: #@ repository + "/" + name + ":" + tag
+        name: #@ name
+        #@ if/end port==80:
+        readinessProbe:
+          httpGet:
+            port: #@ port
+#@ if port != None:
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: #@ name
+    container.training/generated-by: ytt
+  name: #@ name
+spec:
+  ports:
+  - port: #@ port
+    protocol: TCP
+    targetPort: #@ port
+  selector:
+    app: #@ name
+  type: #@ type
+#@ end
+#@ end
+---
+--- #@ template.replace(component("hasher", port=80))
+--- #@ template.replace(component("redis", repository="library", tag="latest", port=6379))
+--- #@ template.replace(component("rng", port=80))
+--- #@ template.replace(component("webui", port=80, type="NodePort"))
+--- #@ template.replace(component("worker"))
+---

k8s/ytt/6-template/schema.yaml

@@ -0,0 +1,4 @@
+#@data/values-schema
+---
+repository: dockercoins
+tag: v0.1

k8s/ytt/7-morevalues/app.yaml

@@ -0,0 +1,65 @@
+---
+#@ load("@ytt:data", "data")
+#@ load("@ytt:template", "template")
+---
+#@ def component(name, repository, tag, port=None, type="ClusterIP"):
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: #@ name
+    container.training/generated-by: ytt
+  name: #@ name
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: #@ name
+  template:
+    metadata:
+      labels:
+        app: #@ name
+    spec:
+      containers:
+      - image: #@ repository + "/" + name + ":" + tag
+        name: #@ name
+        #@ if/end port==80:
+        readinessProbe:
+          httpGet:
+            port: #@ port
+#@ if port != None:
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: #@ name
+    container.training/generated-by: ytt
+  name: #@ name
+spec:
+  ports:
+  - port: #@ port
+    protocol: TCP
+    targetPort: #@ port
+  selector:
+    app: #@ name
+  type: #@ type
+#@ end
+#@ end
+---
+#@ defaults = {}
+#@ for name in data.values:
+#@   if name.startswith("_"):
+#@     defaults.update(data.values[name])
+#@   end
+#@ end
+---
+#@ for name in data.values:
+#@   if not name.startswith("_"):
+#@     values = dict(name=name)
+#@     values.update(defaults)
+#@     values.update(data.values[name])
+--- #@ template.replace(component(**values))
+#@   end
+#@ end

k8s/ytt/7-morevalues/schema.yaml

@@ -0,0 +1,19 @@
+#@data/values-schema
+#! Entries starting with an underscore will hold default values.
+#! Entires NOT starting with an underscore will generate a Deployment
+#! (and a Service if a port number is set).
+---
+_default_:
+  repository: dockercoins
+  tag: v0.1
+hasher:
+  port: 80
+redis:
+  repository: library
+  tag: latest
+rng:
+  port: 80
+webui:
+  port: 80
+  type: NodePort
+worker: {}

k8s/ytt/8-library/_ytt_lib/component/deployment.yaml

@@ -0,0 +1,26 @@
+#@ load("@ytt:data", "data")
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: #@ data.values.name
+    container.training/generated-by: ytt
+  name: #@ data.values.name
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: #@ data.values.name
+  template:
+    metadata:
+      labels:
+        app: #@ data.values.name
+    spec:
+      containers:
+      - image: #@ data.values.repository + "/" + data.values.name + ":" + data.values.tag
+        name: #@ data.values.name
+        #@ if/end data.values.port==80:
+        readinessProbe:
+          httpGet:
+            port: #@ data.values.port

k8s/ytt/8-library/_ytt_lib/component/schema.yaml

@@ -0,0 +1,7 @@
+#@data/values-schema
+---
+name: component
+repository: dockercoins
+tag: v0.1
+port: 0
+type: ClusterIP

k8s/ytt/8-library/_ytt_lib/component/service.yaml

@@ -0,0 +1,19 @@
+#@ load("@ytt:data", "data")
+#@ if data.values.port > 0:
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: #@ data.values.name
+    container.training/generated-by: ytt
+  name: #@ data.values.name
+spec:
+  ports:
+  - port: #@ data.values.port
+    protocol: TCP
+    targetPort: #@ data.values.port
+  selector:
+    app: #@ data.values.name
+  type: #@ data.values.type
+#@ end

k8s/ytt/8-library/app.yaml

@@ -0,0 +1,20 @@
+#@ load("@ytt:data", "data")
+#@ load("@ytt:library", "library")
+#@ load("@ytt:template", "template")
+#@ 
+#@ component = library.get("component")
+#@ 
+#@ defaults = {}
+#@ for name in data.values:
+#@   if name.startswith("_"):
+#@     defaults.update(data.values[name])
+#@   end
+#@ end
+#@ for name in data.values:
+#@   if not name.startswith("_"):
+#@     values = dict(name=name)
+#@     values.update(defaults)
+#@     values.update(data.values[name])
+--- #@ template.replace(component.with_data_values(values).eval())
+#@   end
+#@ end

k8s/ytt/8-library/schema.yaml

@@ -0,0 +1,19 @@
+#@data/values-schema
+#! Entries starting with an underscore will hold default values.
+#! Entires NOT starting with an underscore will generate a Deployment
+#! (and a Service if a port number is set).
+---
+_default_:
+  repository: dockercoins
+  tag: v0.1
+hasher:
+  port: 80
+redis:
+  repository: library
+  tag: latest
+rng:
+  port: 80
+webui:
+  port: 80
+  type: NodePort
+worker: {}

k8s/ytt/9-overlay/_ytt_lib/component/deployment.yaml

@@ -0,0 +1,26 @@
+#@ load("@ytt:data", "data")
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: #@ data.values.name
+    container.training/generated-by: ytt
+  name: #@ data.values.name
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: #@ data.values.name
+  template:
+    metadata:
+      labels:
+        app: #@ data.values.name
+    spec:
+      containers:
+      - image: #@ data.values.repository + "/" + data.values.name + ":" + data.values.tag
+        name: #@ data.values.name
+        #@ if/end data.values.port==80:
+        readinessProbe:
+          httpGet:
+            port: #@ data.values.port

k8s/ytt/9-overlay/_ytt_lib/component/schema.yaml

@@ -0,0 +1,7 @@
+#@data/values-schema
+---
+name: component
+repository: dockercoins
+tag: v0.1
+port: 0
+type: ClusterIP

k8s/ytt/9-overlay/_ytt_lib/component/service.yaml

@@ -0,0 +1,19 @@
+#@ load("@ytt:data", "data")
+#@ if data.values.port > 0:
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: #@ data.values.name
+    container.training/generated-by: ytt
+  name: #@ data.values.name
+spec:
+  ports:
+  - port: #@ data.values.port
+    protocol: TCP
+    targetPort: #@ data.values.port
+  selector:
+    app: #@ data.values.name
+  type: #@ data.values.type
+#@ end

k8s/ytt/9-overlay/app.yaml

@@ -0,0 +1,20 @@
+#@ load("@ytt:data", "data")
+#@ load("@ytt:library", "library")
+#@ load("@ytt:template", "template")
+#@ 
+#@ component = library.get("component")
+#@ 
+#@ defaults = {}
+#@ for name in data.values:
+#@   if name.startswith("_"):
+#@     defaults.update(data.values[name])
+#@   end
+#@ end
+#@ for name in data.values:
+#@   if not name.startswith("_"):
+#@     values = dict(name=name)
+#@     values.update(defaults)
+#@     values.update(data.values[name])
+--- #@ template.replace(component.with_data_values(values).eval())
+#@   end
+#@ end

k8s/ytt/9-overlay/rng-healthcheck.yaml

@@ -0,0 +1,20 @@
+#@ load("@ytt:overlay", "overlay")
+
+#@ def match():
+kind: Deployment
+metadata:
+  name: rng
+#@ end
+
+#@overlay/match by=overlay.subset(match())
+---
+spec:
+  template:
+    spec:
+      containers:
+      #@overlay/match by="name"
+      - name: rng
+        readinessProbe:
+          httpGet:
+            #@overlay/match missing_ok=True
+            path: /1

k8s/ytt/9-overlay/schema.yaml

@@ -0,0 +1,19 @@
+#@data/values-schema
+#! Entries starting with an underscore will hold default values.
+#! Entires NOT starting with an underscore will generate a Deployment
+#! (and a Service if a port number is set).
+---
+_default_:
+  repository: dockercoins
+  tag: v0.1
+hasher:
+  port: 80
+redis:
+  repository: library
+  tag: latest
+rng:
+  port: 80
+webui:
+  port: 80
+  type: NodePort
+worker: {}

k8s/ytt/9-overlay/worker-scaling.yaml

@@ -0,0 +1,25 @@
+#@ load("@ytt:overlay", "overlay")
+
+#@ def match():
+kind: Deployment
+metadata:
+  name: worker
+#@ end
+
+#! This removes the number of replicas:
+#@overlay/match by=overlay.subset(match())
+---
+spec:
+  #@overlay/remove
+  replicas:
+
+#! This overrides it:
+#@overlay/match by=overlay.subset(match())
+---
+spec:
+  #@overlay/match missing_ok=True
+  replicas: 10
+
+#! Note that it's not necessary to remove the number of replicas.
+#! We're just presenting both options here (for instance, you might
+#! want to remove the number of replicas if you're using an HPA).

netlify.toml

@@ -2,4 +2,3 @@
   base = "slides"
   publish = "slides"
   command = "./build.sh once"
-

prepare-tf/README.md

@@ -0,0 +1,162 @@
+⚠️ This is work in progress. The UX needs to be improved,
+and the docs could be better.
+
+This directory contains a Terraform configuration to deploy
+a bunch of Kubernetes clusters on various cloud providers,
+using their respective managed Kubernetes products.
+
+## With shell wrapper
+
+This is the recommended use. It makes it easy to start N clusters
+on any provider. It will create a directory with a name like
+`tag-YYYY-MM-DD-HH-MM-SS-SEED-PROVIDER`, copy the Terraform configuration
+to that directory, then create the clusters using that configuration.
+
+1. One-time setup: configure provider authentication for the provider(s) that you wish to use.
+
+- Digital Ocean:
+  ```bash
+  doctl auth init
+  ```
+
+- Google Cloud Platform: you will need to create a project named `prepare-tf`
+  and enable the relevant APIs for this project (sorry, if you're new to GCP,
+  this sounds vague; but if you're familiar with it you know what to do; if you
+  want to change the project name you can edit the Terraform configuration)
+
+- Linode:
+  ```bash
+  linode-cli configure
+  ```
+
+- Oracle Cloud: FIXME
+  (set up `oci` through the `oci-cli` Python package)
+
+- Scaleway: run `scw init`
+
+2. Optional: set number of clusters, cluster size, and region.
+
+By default, 1 cluster will be configured, with 2 nodes, and auto-scaling up to 5 nodes.
+
+If you want, you can override these parameters, with the following variables.
+
+```bash
+export TF_VAR_how_many_clusters=5
+export TF_VAR_min_nodes_per_pool=2
+export TF_VAR_max_nodes_per_pool=4
+export TF_VAR_location=xxx
+```
+
+The `location` variable is optional. Each provider should have a default value.
+The value of the `location` variable is provider-specific. Examples:
+
+| Provider      | Example value     | How to see possible values
+|---------------|-------------------|---------------------------
+| Digital Ocean | `ams3`            | `doctl compute region list`
+| Google Cloud  | `europe-north1-a` | `gcloud  compute zones list`
+| Linode        | `eu-central`      | `linode-cli regions list`
+| Oracle Cloud  | `eu-stockholm-1`  | `oci iam region list`
+
+You can also specify multiple locations, and then they will be
+used in round-robin fashion.
+
+For example, with Google Cloud, since the default quotas are very
+low (my account is limited to 8 public IP addresses per zone, and
+my requests to increase that quota were denied) you can do the
+following:
+
+```bash
+export TF_VAR_location=$(gcloud compute zones list --format=json | jq -r .[].name | grep ^europe)
+```
+
+Then when you apply, clusters will be created across all available
+zones in Europe. (When I write this, there are 20+ zones in Europe,
+so even with my quota, I can create 40 clusters.)
+
+3. Run!
+
+```bash
+./run.sh <providername>
+```
+
+(If you don't specify a provider name, it will list available providers.)
+
+4. Shutting down
+
+Go to the directory that was created by the previous step (`tag-YYYY-MM...`)
+and run `terraform destroy`.
+
+You can also run `./clean.sh` which will destroy ALL clusters deployed by the previous run script.
+
+## Without shell wrapper
+
+Expert mode.
+
+Useful to run steps sperarately, and/or when working on the Terraform configurations.
+
+1. Select the provider you wish to use.
+
+Go to the `source` directory and edit `main.tf`.
+
+Change the `source` attribute of the `module "clusters"` section.
+
+Check the content of the `modules` directory to see available choices.
+
+2. Initialize the provider.
+
+```bash
+terraform init
+```
+
+3. Configure provider authentication.
+
+See steps above, and add the following extra steps:
+
+- Digital Coean:
+  ```bash
+  export DIGITALOCEAN_ACCESS_TOKEN=$(grep ^access-token ~/.config/doctl/config.yaml | cut -d: -f2 | tr -d " ")
+  ```
+
+- Linode:
+  ```bash
+  export LINODE_TOKEN=$(grep ^token ~/.config/linode-cli | cut -d= -f2 | tr -d " ")
+  ```
+
+4. Decide how many clusters and how many nodes per clusters you want.
+
+5. Provision clusters.
+
+```bash
+terraform apply
+```
+
+6. Perform second stage provisioning.
+
+This will install an SSH server on the clusters.
+
+```bash
+cd stage2
+terraform init
+terraform apply
+```
+
+7. Obtain cluster connection information.
+
+The following command shows connection information, one cluster per line, ready to copy-paste in a shared document or spreadsheet.
+
+```bash
+terraform output -json | jq -r 'to_entries[].value.value'
+```
+
+8. Destroy clusters.
+
+```bash
+cd ..
+terraform destroy
+```
+
+9. Clean up stage2.
+
+```bash
+rm stage2/terraform.tfstate*
+```

prepare-tf/cleanup.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+export LINODE_TOKEN=$(grep ^token ~/.config/linode-cli | cut -d= -f2 | tr -d " ")
+export DIGITALOCEAN_ACCESS_TOKEN=$(grep ^access-token ~/.config/doctl/config.yaml | cut -d: -f2 | tr -d " ")
+for T in  tag-*; do
+(
+  cd $T
+  terraform apply -destroy -auto-approve && mv ../$T ../deleted$T
+)
+done

prepare-tf/run.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+set -e
+
+TIME=$(which time)
+
+PROVIDER=$1
+[ "$PROVIDER" ] || {
+  echo "Please specify a provider as first argument, or 'ALL' for parallel mode."
+  echo "Available providers:"
+  ls -1 source/modules
+  exit 1
+}
+
+[ "$TAG" ] || {
+  TIMESTAMP=$(date +%Y-%m-%d-%H-%M-%S)
+  RANDOMTAG=$(base64 /dev/urandom | tr A-Z a-z | tr -d /+ | head -c5)
+  export TAG=tag-$TIMESTAMP-$RANDOMTAG
+}
+
+[ "$PROVIDER" = "ALL" ] && {
+  for PROVIDER in $(ls -1 source/modules); do
+    $TERMINAL -T $TAG-$PROVIDER -e sh -c "
+      export TAG=$TAG-$PROVIDER
+      $0 $PROVIDER
+      cd $TAG-$PROVIDER
+      bash
+      " &
+  done
+  exit 0
+}
+
+[ -d "source/modules/$PROVIDER" ] || {
+  echo "Provider '$PROVIDER' not found."
+  echo "Available providers:"
+  ls -1 source/modules
+  exit 1  
+}
+
+export LINODE_TOKEN=$(grep ^token ~/.config/linode-cli | cut -d= -f2 | tr -d " ")
+export DIGITALOCEAN_ACCESS_TOKEN=$(grep ^access-token ~/.config/doctl/config.yaml | cut -d: -f2 | tr -d " ")
+
+cp -a source $TAG
+cd $TAG
+cp -r modules/$PROVIDER modules/PROVIDER
+$TIME -o time.1.init terraform init
+$TIME -o time.2.stage1 terraform apply -auto-approve
+cd stage2
+$TIME -o ../time.3.init terraform init
+$TIME -o ../time.4.stage2 terraform apply -auto-approve

prepare-tf/source/locals.tf

@@ -0,0 +1,19 @@
+resource "random_string" "_" {
+  length  = 4
+  numeric = false
+  special = false
+  upper   = false
+}
+
+resource "time_static" "_" {}
+
+locals {
+  timestamp = formatdate("YYYY-MM-DD-hh-mm", time_static._.rfc3339)
+  tag       = random_string._.result
+  # Common tags to be assigned to all resources
+  common_tags = [
+    "created-by-terraform",
+    format("created-at-%s", local.timestamp),
+    format("created-for-%s", local.tag)
+  ]
+}

prepare-tf/source/main.tf

@@ -0,0 +1,88 @@
+module "clusters" {
+  source             = "./modules/PROVIDER"
+  for_each           = local.clusters
+  cluster_name       = each.value.cluster_name
+  min_nodes_per_pool = var.min_nodes_per_pool
+  max_nodes_per_pool = var.max_nodes_per_pool
+  enable_arm_pool    = var.enable_arm_pool
+  node_size          = var.node_size
+  common_tags        = local.common_tags
+  location           = each.value.location
+}
+
+locals {
+  clusters = {
+    for i in range(101, 101 + var.how_many_clusters) :
+    i => {
+      cluster_name     = format("%s-%03d", local.tag, i)
+      kubeconfig_path  = format("./stage2/kubeconfig.%03d", i)
+      externalips_path = format("./stage2/externalips.%03d", i)
+      flags_path       = format("./stage2/flags.%03d", i)
+      location         = local.locations[i % length(local.locations)]
+    }
+  }
+}
+
+resource "local_file" "stage2" {
+  filename        = "./stage2/main.tf"
+  file_permission = "0644"
+  content = templatefile(
+    "./stage2.tmpl",
+    { clusters = local.clusters }
+  )
+}
+
+resource "local_file" "flags" {
+  for_each        = local.clusters
+  filename        = each.value.flags_path
+  file_permission = "0600"
+  content         = <<-EOT
+    has_metrics_server: ${module.clusters[each.key].has_metrics_server}
+  EOT
+}
+
+resource "local_file" "kubeconfig" {
+  for_each        = local.clusters
+  filename        = each.value.kubeconfig_path
+  file_permission = "0600"
+  content         = module.clusters[each.key].kubeconfig
+}
+
+resource "local_file" "externalips" {
+  for_each        = local.clusters
+  filename        = each.value.externalips_path
+  file_permission = "0600"
+  content         = data.external.externalips[each.key].result.externalips
+}
+
+resource "null_resource" "wait_for_nodes" {
+  for_each = local.clusters
+  provisioner "local-exec" {
+    environment = {
+      KUBECONFIG = local_file.kubeconfig[each.key].filename
+    }
+    command = <<-EOT
+      set -e
+      kubectl get nodes --watch | grep --silent --line-buffered .
+      kubectl wait node --for=condition=Ready --all --timeout=10m
+      EOT
+  }
+}
+
+data "external" "externalips" {
+  for_each   = local.clusters
+  depends_on = [null_resource.wait_for_nodes]
+  program = [
+    "sh",
+    "-c",
+    <<-EOT
+      set -e
+      cat >/dev/null
+      export KUBECONFIG=${local_file.kubeconfig[each.key].filename}
+      echo -n '{"externalips": "'
+      kubectl get nodes \
+      -o 'jsonpath={.items[*].status.addresses[?(@.type=="ExternalIP")].address}'
+      echo -n '"}'
+      EOT
+  ]
+}

prepare-tf/source/modules/digitalocean/main.tf

@@ -0,0 +1,17 @@
+resource "digitalocean_kubernetes_cluster" "_" {
+  name = var.cluster_name
+  tags = var.common_tags
+  # Region is mandatory, so let's provide a default value.
+  region  = var.location != null ? var.location : "nyc1"
+  version = var.k8s_version
+
+  node_pool {
+    name       = "x86"
+    tags       = var.common_tags
+    size       = local.node_type
+    auto_scale = true
+    min_nodes  = var.min_nodes_per_pool
+    max_nodes  = max(var.min_nodes_per_pool, var.max_nodes_per_pool)
+  }
+
+}

prepare-tf/source/modules/digitalocean/outputs.tf

@@ -0,0 +1,11 @@
+output "kubeconfig" {
+  value = digitalocean_kubernetes_cluster._.kube_config.0.raw_config
+}
+
+output "cluster_id" {
+  value = digitalocean_kubernetes_cluster._.id
+}
+
+output "has_metrics_server" {
+  value = false
+}

prepare-tf/source/modules/digitalocean/providers.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    digitalocean = {
+      source  = "digitalocean/digitalocean"
+      version = "2.12.1"
+    }
+  }
+}

prepare-tf/source/modules/digitalocean/variables.tf

@@ -0,0 +1,57 @@
+variable "cluster_name" {
+  type    = string
+  default = "deployed-with-terraform"
+}
+
+variable "common_tags" {
+  type    = list(string)
+  default = []
+}
+
+variable "node_size" {
+  type    = string
+  default = "M"
+}
+
+variable "min_nodes_per_pool" {
+  type    = number
+  default = 2
+}
+
+variable "max_nodes_per_pool" {
+  type    = number
+  default = 5
+}
+
+# FIXME
+variable "enable_arm_pool" {
+  type    = bool
+  default = false
+}
+
+variable "node_types" {
+  type = map(string)
+  default = {
+    "S" = "s-1vcpu-2gb"
+    "M" = "s-2vcpu-4gb"
+    "L" = "s-4vcpu-8gb"
+  }
+}
+
+locals {
+  node_type = var.node_types[var.node_size]
+}
+
+# To view supported regions, run:
+# doctl compute region list
+variable "location" {
+  type    = string
+  default = null
+}
+
+# To view supported versions, run:
+# doctl kubernetes options versions -o json | jq -r .[].slug
+variable "k8s_version" {
+  type    = string
+  default = "1.22.8-do.1"
+}

prepare-tf/source/modules/googlecloud/main.tf

@@ -0,0 +1,65 @@
+resource "google_container_cluster" "_" {
+  name               = var.cluster_name
+  project            = local.project
+  location           = local.location
+  min_master_version = var.k8s_version
+
+  # To deploy private clusters, uncomment the section below,
+  # and uncomment the block in network.tf.
+  # Private clusters require extra resources (Cloud NAT,
+  # router, network, subnet) and the quota for some of these
+  # resources is fairly low on GCP; so if you want to deploy
+  # a lot of private clusters (more than 10), you can use these
+  # blocks as a base but you will probably have to refactor
+  # things quite a bit (you will at least need to define a single
+  # shared router and use it across all the clusters).
+  /*
+  network    = google_compute_network._.name
+  subnetwork = google_compute_subnetwork._.name
+
+  private_cluster_config {
+    enable_private_nodes = true
+    # This must be set to "false".
+    # (Otherwise, access to the public endpoint is disabled.)
+    enable_private_endpoint = false
+    # This must be set to a /28.
+    # I think it shouldn't collide with the pod network subnet.
+    master_ipv4_cidr_block = "10.255.255.0/28"
+  }
+  # Private clusters require "VPC_NATIVE" networking mode
+  # (as opposed to the legacy "ROUTES").
+  networking_mode = "VPC_NATIVE"
+  # ip_allocation_policy is required for VPC_NATIVE clusters.
+  ip_allocation_policy {
+    # This is the block that will be used for pods.
+    cluster_ipv4_cidr_block = "10.0.0.0/12"
+    # The services block is optional
+    # (GKE will pick one automatically).
+    #services_ipv4_cidr_block = ""
+  }
+  */
+
+  node_pool {
+    name = "x86"
+    node_config {
+      tags         = var.common_tags
+      machine_type = local.node_type
+    }
+    initial_node_count = var.min_nodes_per_pool
+    autoscaling {
+      min_node_count = var.min_nodes_per_pool
+      max_node_count = max(var.min_nodes_per_pool, var.max_nodes_per_pool)
+    }
+  }
+
+  # This is not strictly necessary.
+  # We'll see if we end up using it.
+  # (If it is removed, make sure to also remove the corresponding
+  # key+cert variables from outputs.tf!)
+  master_auth {
+    client_certificate_config {
+      issue_client_certificate = true
+    }
+  }
+}
+

prepare-tf/source/modules/googlecloud/network.tf

@@ -0,0 +1,38 @@
+/*
+resource "google_compute_network" "_" {
+  name    = var.cluster_name
+  project = local.project
+  # The default is to create subnets automatically.
+  # However, this creates one subnet per zone in all regions,
+  # which causes a quick exhaustion of the subnet quota.
+  auto_create_subnetworks = false
+}
+
+resource "google_compute_subnetwork" "_" {
+  name          = var.cluster_name
+  ip_cidr_range = "10.254.0.0/16"
+  region        = local.region
+  network       = google_compute_network._.id
+  project       = local.project
+}
+
+resource "google_compute_router" "_" {
+  name    = var.cluster_name
+  region  = local.region
+  network = google_compute_network._.name
+  project = local.project
+}
+
+resource "google_compute_router_nat" "_" {
+  name    = var.cluster_name
+  router  = google_compute_router._.name
+  region  = local.region
+  project = local.project
+  # Everyone in the network is allowed to NAT out.
+  # (We would change this if we only wanted to allow specific subnets to NAT out.)
+  source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES"
+  # Pick NAT addresses automatically.
+  # (We would change this if we wanted to use specific addresses to NAT out.)
+  nat_ip_allocate_option = "AUTO_ONLY"
+}
+*/

prepare-tf/source/modules/googlecloud/outputs.tf

@@ -0,0 +1,35 @@
+data "google_client_config" "_" {}
+
+output "kubeconfig" {
+  value = <<-EOT
+    apiVersion: v1
+    kind: Config
+    current-context: ${google_container_cluster._.name}
+    clusters:
+    - name: ${google_container_cluster._.name}
+      cluster:
+        server: https://${google_container_cluster._.endpoint}
+        certificate-authority-data: ${google_container_cluster._.master_auth[0].cluster_ca_certificate}
+    contexts:
+    - name: ${google_container_cluster._.name}
+      context:
+        cluster: ${google_container_cluster._.name}
+        user: client-token
+    users:
+    - name: client-cert
+      user:
+        client-key-data: ${google_container_cluster._.master_auth[0].client_key}
+        client-certificate-data: ${google_container_cluster._.master_auth[0].client_certificate}
+    - name: client-token
+      user:
+        token: ${data.google_client_config._.access_token}
+    EOT
+}
+
+output "cluster_id" {
+  value = google_container_cluster._.id
+}
+
+output "has_metrics_server" {
+  value = true
+}

prepare-tf/source/modules/googlecloud/providers.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "4.5.0"
+    }
+  }
+}

prepare-tf/source/modules/googlecloud/variables.tf

@@ -0,0 +1,68 @@
+variable "cluster_name" {
+  type    = string
+  default = "deployed-with-terraform"
+}
+
+variable "common_tags" {
+  type    = list(string)
+  default = []
+}
+
+variable "node_size" {
+  type    = string
+  default = "M"
+}
+
+variable "min_nodes_per_pool" {
+  type    = number
+  default = 2
+}
+
+variable "max_nodes_per_pool" {
+  type    = number
+  default = 5
+}
+
+# FIXME
+variable "enable_arm_pool" {
+  type    = bool
+  default = false
+}
+
+variable "node_types" {
+  type = map(string)
+  default = {
+    "S" = "e2-small"
+    "M" = "e2-medium"
+    "L" = "e2-standard-2"
+  }
+}
+
+locals {
+  node_type = var.node_types[var.node_size]
+}
+
+# To view supported locations, run:
+# gcloud compute zones list
+variable "location" {
+  type    = string
+  default = null
+}
+
+# To view supported versions, run:
+# gcloud container get-server-config --region=europe-north1 '--format=flattened(channels)'
+# But it's also possible to just specify e.g. "1.20" and it figures it out.
+variable "k8s_version" {
+  type    = string
+  default = "1.21"
+}
+
+locals {
+  location = var.location != null ? var.location : "europe-north1-a"
+  region   = replace(local.location, "/-[a-z]$/", "")
+  # Unfortunately, the following line doesn't work
+  # (that attribute just returns an empty string)
+  # so we have to hard-code the project name.
+  #project = data.google_client_config._.project
+  project = "prepare-tf"
+}

prepare-tf/source/modules/linode/main.tf

@@ -0,0 +1,17 @@
+resource "linode_lke_cluster" "_" {
+  label = var.cluster_name
+  tags  = var.common_tags
+  # "region" is mandatory, so let's provide a default value if none was given.
+  region      = var.location != null ? var.location : "eu-central"
+  k8s_version = local.k8s_version
+
+  pool {
+    type  = local.node_type
+    count = var.min_nodes_per_pool
+    autoscaler {
+      min = var.min_nodes_per_pool
+      max = max(var.min_nodes_per_pool, var.max_nodes_per_pool)
+    }
+  }
+
+}

prepare-tf/source/modules/linode/outputs.tf

@@ -0,0 +1,11 @@
+output "kubeconfig" {
+  value = base64decode(linode_lke_cluster._.kubeconfig)
+}
+
+output "cluster_id" {
+  value = linode_lke_cluster._.id
+}
+
+output "has_metrics_server" {
+  value = false
+}

prepare-tf/source/modules/linode/providers.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    linode = {
+      source  = "linode/linode"
+      version = "1.22.0"
+    }
+  }
+}

prepare-tf/source/modules/linode/variables.tf

@@ -0,0 +1,72 @@
+variable "cluster_name" {
+  type    = string
+  default = "deployed-with-terraform"
+}
+
+variable "common_tags" {
+  type    = list(string)
+  default = []
+}
+
+variable "node_size" {
+  type    = string
+  default = "M"
+}
+
+variable "min_nodes_per_pool" {
+  type    = number
+  default = 2
+}
+
+variable "max_nodes_per_pool" {
+  type    = number
+  default = 5
+}
+
+# FIXME
+variable "enable_arm_pool" {
+  type    = bool
+  default = false
+}
+
+variable "node_types" {
+  type = map(string)
+  default = {
+    "S" = "g6-standard-1"
+    "M" = "g6-standard-2"
+    "L" = "g6-standard-4"
+  }
+}
+
+locals {
+  node_type = var.node_types[var.node_size]
+}
+
+# To view supported regions, run:
+# linode-cli regions list
+variable "location" {
+  type    = string
+  default = null
+}
+
+# To view supported versions, run:
+# linode-cli lke versions-list --json | jq -r .[].id
+data "external" "k8s_version" {
+  program = [
+    "sh",
+    "-c",
+    <<-EOT
+      linode-cli lke versions-list --json |
+      jq -r '{"latest": [.[].id] | sort [-1]}'
+    EOT
+  ]
+}
+
+variable "k8s_version" {
+  type    = string
+  default = ""
+}
+
+locals {
+  k8s_version = var.k8s_version != "" ? var.k8s_version : data.external.k8s_version.result.latest
+}

prepare-tf/source/modules/oraclecloud/main.tf

@@ -0,0 +1,59 @@
+resource "oci_identity_compartment" "_" {
+  name          = var.cluster_name
+  description   = var.cluster_name
+  enable_delete = true
+}
+
+locals {
+  compartment_id = oci_identity_compartment._.id
+}
+
+data "oci_identity_availability_domains" "_" {
+  compartment_id = local.compartment_id
+}
+
+data "oci_core_images" "_" {
+  for_each                 = local.pools
+  compartment_id           = local.compartment_id
+  operating_system         = "Oracle Linux"
+  operating_system_version = "7.9"
+  shape                    = each.value.shape
+}
+
+resource "oci_containerengine_cluster" "_" {
+  compartment_id     = local.compartment_id
+  kubernetes_version = var.k8s_version
+  name               = "tf-oke"
+  vcn_id             = oci_core_vcn._.id
+  options {
+    service_lb_subnet_ids = [oci_core_subnet.loadbalancers.id]
+  }
+  endpoint_config {
+    is_public_ip_enabled = true
+    subnet_id            = oci_core_subnet.controlplane.id
+  }
+}
+
+resource "oci_containerengine_node_pool" "_" {
+  for_each           = local.pools
+  cluster_id         = oci_containerengine_cluster._.id
+  compartment_id     = local.compartment_id
+  kubernetes_version = var.k8s_version
+  name               = each.key
+  node_shape         = each.value.shape
+  node_shape_config {
+    memory_in_gbs = local.node_type.memory_in_gbs
+    ocpus         = local.node_type.ocpus
+  }
+  node_config_details {
+    size = var.min_nodes_per_pool
+    placement_configs {
+      availability_domain = data.oci_identity_availability_domains._.availability_domains[0].name
+      subnet_id           = oci_core_subnet.nodes.id
+    }
+  }
+  node_source_details {
+    image_id    = data.oci_core_images._[each.key].images[0].id
+    source_type = "image"
+  }
+}

prepare-tf/source/modules/oraclecloud/network.tf

@@ -0,0 +1,81 @@
+resource "oci_core_vcn" "_" {
+  compartment_id = local.compartment_id
+  cidr_block     = "10.0.0.0/16"
+  display_name   = "tf-vcn"
+}
+
+#
+# On OCI, you can have either "public" or "private" subnets.
+# In both cases, instances get addresses in the VCN CIDR block;
+# but instances in "public" subnets also get a public address.
+#
+# Then, to enable communication to the outside world, you need:
+# - for public subnets, an "internet gateway"
+#   (will allow inbound and outbound traffic)
+# - for private subnets, a "NAT gateway"
+#   (will only allow outbound traffic)
+# - optionally, for private subnets, a "service gateway"
+#   (to access other OCI services, e.g. object store)
+#
+# In this configuration, we use public subnets, and since we
+# need outside access, we add an internet gateway.
+#
+# Note that the default routing table in a VCN is empty, so we
+# add the internet gateway to the default routing table.
+# Similarly, the default security group in a VCN blocks almost
+# everything, so we add a blanket rule in that security group.
+#
+
+resource "oci_core_internet_gateway" "_" {
+  compartment_id = local.compartment_id
+  display_name   = "tf-igw"
+  vcn_id         = oci_core_vcn._.id
+}
+
+resource "oci_core_default_route_table" "_" {
+  manage_default_resource_id = oci_core_vcn._.default_route_table_id
+  route_rules {
+    destination       = "0.0.0.0/0"
+    destination_type  = "CIDR_BLOCK"
+    network_entity_id = oci_core_internet_gateway._.id
+  }
+}
+
+resource "oci_core_default_security_list" "_" {
+  manage_default_resource_id = oci_core_vcn._.default_security_list_id
+  ingress_security_rules {
+    protocol = "all"
+    source   = "0.0.0.0/0"
+  }
+  egress_security_rules {
+    protocol    = "all"
+    destination = "0.0.0.0/0"
+  }
+}
+
+resource "oci_core_subnet" "controlplane" {
+  compartment_id    = local.compartment_id
+  cidr_block        = "10.0.254.0/24"
+  vcn_id            = oci_core_vcn._.id
+  display_name      = "tf-controlplane"
+  route_table_id    = oci_core_default_route_table._.id
+  security_list_ids = [oci_core_default_security_list._.id]
+}
+
+resource "oci_core_subnet" "nodes" {
+  compartment_id    = local.compartment_id
+  cidr_block        = "10.0.0.0/20"
+  vcn_id            = oci_core_vcn._.id
+  display_name      = "tf-nodes"
+  route_table_id    = oci_core_default_route_table._.id
+  security_list_ids = [oci_core_default_security_list._.id]
+}
+
+resource "oci_core_subnet" "loadbalancers" {
+  compartment_id    = local.compartment_id
+  cidr_block        = "10.0.96.0/20"
+  vcn_id            = oci_core_vcn._.id
+  display_name      = "tf-loadbalancers"
+  route_table_id    = oci_core_default_route_table._.id
+  security_list_ids = [oci_core_default_security_list._.id]
+}

prepare-tf/source/modules/oraclecloud/outputs.tf

@@ -0,0 +1,15 @@
+data "oci_containerengine_cluster_kube_config" "_" {
+  cluster_id = oci_containerengine_cluster._.id
+}
+
+output "kubeconfig" {
+  value = data.oci_containerengine_cluster_kube_config._.content
+}
+
+output "cluster_id" {
+  value = oci_containerengine_cluster._.id
+}
+
+output "has_metrics_server" {
+  value = false
+}

prepare-tf/source/modules/oraclecloud/providers.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    oci = {
+      source  = "hashicorp/oci"
+      version = "4.48.0"
+    }
+  }
+}

prepare-tf/source/modules/oraclecloud/variables.tf

@@ -0,0 +1,85 @@
+variable "cluster_name" {
+  type    = string
+  default = "deployed-with-terraform"
+}
+
+variable "common_tags" {
+  type    = list(string)
+  default = []
+}
+
+variable "node_size" {
+  type    = string
+  default = "M"
+}
+
+variable "min_nodes_per_pool" {
+  type    = number
+  default = 2
+}
+
+variable "max_nodes_per_pool" {
+  type    = number
+  default = 5
+}
+
+variable "enable_arm_pool" {
+  type    = bool
+  default = true
+}
+
+locals {
+  arm_pool = {
+    shape = "VM.Standard.A1.Flex"
+  }
+  x86_pool = {
+    shape = "VM.Standard.E4.Flex"
+  }
+  pools = var.enable_arm_pool ? {
+    "oke-arm" = local.arm_pool
+    "oke-x86" = local.x86_pool
+    } : {
+    "oke-x86" = local.x86_pool
+  }
+}
+
+output "pool" {
+  value = local.pools
+}
+
+variable "node_types" {
+  # FIXME put better typing here
+  type = map(map(number))
+  default = {
+    "S" = {
+      memory_in_gbs = 2
+      ocpus         = 1
+    }
+    "M" = {
+      memory_in_gbs = 4
+      ocpus         = 1
+    }
+    "L" = {
+      memory_in_gbs = 8
+      ocpus         = 2
+    }
+  }
+}
+
+locals {
+  node_type = var.node_types[var.node_size]
+}
+
+# To view supported regions, run:
+# oci iam region list | jq .data[].name
+variable "location" {
+  type    = string
+  default = null
+}
+
+# To view supported versions, run:
+# oci ce cluster-options get --cluster-option-id all | jq -r '.data["kubernetes-versions"][]'
+variable "k8s_version" {
+  type    = string
+  default = "v1.20.11"
+}

prepare-tf/source/modules/scaleway/main.tf

@@ -0,0 +1,20 @@
+resource "scaleway_k8s_cluster" "_" {
+  name                        = var.cluster_name
+  region                      = var.location
+  tags                        = var.common_tags
+  version                     = var.k8s_version
+  cni                         = var.cni
+  delete_additional_resources = true
+}
+
+resource "scaleway_k8s_pool" "_" {
+  cluster_id  = scaleway_k8s_cluster._.id
+  name        = "x86"
+  tags        = var.common_tags
+  node_type   = local.node_type
+  size        = var.min_nodes_per_pool
+  min_size    = var.min_nodes_per_pool
+  max_size    = max(var.min_nodes_per_pool, var.max_nodes_per_pool)
+  autoscaling = true
+  autohealing = true
+}

prepare-tf/source/modules/scaleway/outputs.tf

@@ -0,0 +1,11 @@
+output "kubeconfig" {
+  value = scaleway_k8s_cluster._.kubeconfig.0.config_file
+}
+
+output "cluster_id" {
+  value = scaleway_k8s_cluster._.id
+}
+
+output "has_metrics_server" {
+  value = sort([var.k8s_version, "1.22"])[0] == "1.22"
+}

prepare-tf/source/modules/scaleway/providers.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    scaleway = {
+      source  = "scaleway/scaleway"
+      version = "2.1.0"
+    }
+  }
+}

prepare-tf/source/modules/scaleway/variables.tf

@@ -0,0 +1,60 @@
+variable "cluster_name" {
+  type    = string
+  default = "deployed-with-terraform"
+}
+
+variable "common_tags" {
+  type    = list(string)
+  default = []
+}
+
+variable "node_size" {
+  type    = string
+  default = "M"
+}
+
+variable "min_nodes_per_pool" {
+  type    = number
+  default = 2
+}
+
+variable "max_nodes_per_pool" {
+  type    = number
+  default = 5
+}
+
+# FIXME
+variable "enable_arm_pool" {
+  type    = bool
+  default = false
+}
+
+variable "node_types" {
+  type = map(string)
+  default = {
+    "S" = "DEV1-S"
+    "M" = "DEV1-M"
+    "L" = "DEV1-L"
+  }
+}
+
+locals {
+  node_type = var.node_types[var.node_size]
+}
+
+variable "cni" {
+  type    = string
+  default = "cilium"
+}
+
+variable "location" {
+  type    = string
+  default = null
+}
+
+# To view supported versions, run:
+# scw k8s version list -o json | jq -r .[].name
+variable "k8s_version" {
+  type    = string
+  default = "1.23.6"
+}

prepare-tf/source/providers.tf

@@ -0,0 +1,3 @@
+terraform {
+  required_version = ">= 1.0"
+}

prepare-tf/source/stage2.tmpl

@@ -0,0 +1,230 @@
+terraform {
+  required_providers {
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "2.7.1"
+    }
+  }
+}
+
+%{ for index, cluster in clusters ~}
+
+provider "kubernetes" {
+  alias = "cluster_${index}"
+  config_path = "./kubeconfig.${index}"
+}
+
+resource "kubernetes_namespace" "shpod_${index}" {
+  provider = kubernetes.cluster_${index}
+  metadata {
+    name = "shpod"
+  }
+}
+
+resource "kubernetes_deployment" "shpod_${index}" {
+  provider = kubernetes.cluster_${index}
+  metadata {
+    name = "shpod"
+    namespace = kubernetes_namespace.shpod_${index}.metadata.0.name
+  }
+  spec {
+    selector {
+      match_labels = {
+        app = "shpod"
+      }
+    }
+    template {
+      metadata {
+        labels = {
+          app = "shpod"
+        }
+      }
+      spec {
+        service_account_name = "shpod"
+        container {
+          image = "jpetazzo/shpod"
+          name = "shpod"
+          env {
+            name = "PASSWORD"
+            value = random_string.shpod_${index}.result
+          }
+          lifecycle {
+            post_start {
+              exec {
+                command = [ "sh", "-c", "curl http://myip.enix.org/REMOTE_ADDR > /etc/HOSTIP || true" ]
+              }
+            }
+          }
+          resources {
+            limits = {
+              cpu    = "2"
+              memory = "500M"
+            }
+            requests = {
+              cpu    = "100m"
+              memory = "250M"
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_service" "shpod_${index}" {
+  provider = kubernetes.cluster_${index}
+  lifecycle {
+    # Folks might alter their shpod Service to expose extra ports.
+    # Don't reset their changes.
+    ignore_changes = [ spec ]
+  }
+  metadata {
+    name = "shpod"
+    namespace = kubernetes_namespace.shpod_${index}.metadata.0.name
+  }
+  spec {
+    selector = {
+      app = "shpod"
+    }
+    port {
+      name = "ssh"
+      port = 22
+      target_port = 22
+      node_port = 32222
+    }
+    type = "NodePort"
+  }
+}
+
+resource "kubernetes_service_account" "shpod_${index}" {
+  provider = kubernetes.cluster_${index}
+  metadata {
+    name = "shpod"
+    namespace = kubernetes_namespace.shpod_${index}.metadata.0.name
+  }
+}
+
+resource "kubernetes_cluster_role_binding" "shpod_${index}" {
+  provider = kubernetes.cluster_${index}
+  metadata {
+    name = "shpod"
+  }
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "ClusterRole"
+    name      = "cluster-admin"
+  }
+  subject {
+    kind      = "ServiceAccount"
+    name      = "shpod"
+    namespace = "shpod"
+  }
+  subject {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "Group"
+    name      = "shpod-cluster-admins"
+  }
+}
+
+resource "random_string" "shpod_${index}" {
+  length  = 6
+  special = false
+  upper   = false
+}
+
+provider "helm" {
+  alias = "cluster_${index}"
+  kubernetes {
+    config_path = "./kubeconfig.${index}"
+  }
+}
+
+resource "helm_release" "metrics_server_${index}" {
+  # Some providers pre-install metrics-server.
+  # Some don't. Let's install metrics-server,
+  # but only if it's not already installed.
+  count = yamldecode(file("./flags.${index}"))["has_metrics_server"] ? 0 : 1
+  provider = helm.cluster_${index}
+  repository = "https://kubernetes-sigs.github.io/metrics-server/"
+  chart = "metrics-server"
+  version = "3.8.2"
+  name = "metrics-server"
+  namespace = "metrics-server"
+  create_namespace = true
+  set {
+    name = "args"
+    value = "{--kubelet-insecure-tls}"
+  }
+}
+
+resource "kubernetes_config_map" "kubeconfig_${index}" {
+  provider = kubernetes.cluster_${index}
+  metadata {
+    name = "kubeconfig"
+    namespace = kubernetes_namespace.shpod_${index}.metadata.0.name
+  }
+  data = {
+    kubeconfig_from_provider = file("./kubeconfig.${index}")
+    kubeconfig_cluster_admin = <<-EOT
+      kind: Config
+      apiVersion: v1
+      current-context: cluster-admin@k8s-${index}
+      clusters:
+      - name: k8s-${index}
+        cluster:
+          certificate-authority-data: $${yamldecode(file("./kubeconfig.${index}")).clusters.0.cluster.certificate-authority-data}
+          server: $${yamldecode(file("./kubeconfig.${index}")).clusters.0.cluster.server}
+      contexts:
+      - name: cluster-admin@k8s-${index}
+        context:
+          cluster: k8s-${index}
+          user: cluster-admin
+      users:
+      - name: cluster-admin
+        user:
+          client-key-data: $${base64encode(tls_private_key.cluster_admin_${index}.private_key_pem)}
+          client-certificate-data: $${base64encode(kubernetes_certificate_signing_request_v1.cluster_admin_${index}.certificate)}
+    EOT
+  }
+}
+
+resource "tls_private_key" "cluster_admin_${index}" {
+  algorithm = "RSA"
+}
+
+resource "tls_cert_request" "cluster_admin_${index}" {
+  private_key_pem = tls_private_key.cluster_admin_${index}.private_key_pem
+  subject {
+    common_name = "cluster-admin"
+    # Note: CSR API v1 doesn't allow issuing certs with "system:masters" anymore.
+    #organization = "system:masters"
+    # We'll use this custom group name instead.cluster-admin user.
+    organization = "shpod-cluster-admins"
+  }
+}
+
+resource "kubernetes_certificate_signing_request_v1" "cluster_admin_${index}" {
+  provider = kubernetes.cluster_${index}
+  metadata {
+    name = "cluster-admin"
+  }
+  spec {
+    usages = ["client auth"]
+    request = tls_cert_request.cluster_admin_${index}.cert_request_pem
+    signer_name = "kubernetes.io/kube-apiserver-client"
+  }
+  auto_approve = true
+}
+
+%{ endfor ~}
+
+output "ip_addresses_of_nodes" {
+  value = join("\n", [
+  %{ for index, cluster in clusters ~}
+    join("\t", concat(
+      [ random_string.shpod_${index}.result, "ssh -l k8s -p 32222" ],
+      split(" ", file("./externalips.${index}"))
+    )),
+  %{ endfor ~}
+  ])
+}

prepare-tf/source/variables.tf

@@ -0,0 +1,40 @@
+variable "how_many_clusters" {
+  type    = number
+  default = 1
+}
+
+variable "node_size" {
+  type    = string
+  default = "M"
+  # Can be S, M, L.
+  # We map these values to different specific instance types for each provider,
+  # but the idea is that they shoudl correspond to the following sizes:
+  # S = 2 GB RAM
+  # M = 4 GB RAM
+  # L = 8 GB RAM
+}
+
+variable "min_nodes_per_pool" {
+  type    = number
+  default = 1
+}
+
+variable "max_nodes_per_pool" {
+  type    = number
+  default = 0
+}
+
+variable "enable_arm_pool" {
+  type    = bool
+  default = false
+}
+
+variable "location" {
+  type    = string
+  default = null
+}
+
+# TODO: perhaps handle if it's space-separated instead of newline?
+locals {
+  locations = var.location == null ? [null] : split("\n", var.location)
+}

prepare-vms/README.md

@@ -14,7 +14,10 @@ These tools can help you to create VMs on:
 
 - [Docker](https://docs.docker.com/engine/installation/)
 - [Docker Compose](https://docs.docker.com/compose/install/)
-- [Parallel SSH](https://code.google.com/archive/p/parallel-ssh/) (on a Mac: `brew install pssh`) 
+- [Parallel SSH](https://github.com/lilydjwg/pssh)
+  (should be installable with `pip install git+https://github.com/lilydjwg/pssh`;
+  on a Mac, try `brew install pssh`)
+- [yq](https://github.com/kislyuk/yq)
 
 Depending on the infrastructure that you want to use, you also need to install
 the CLI that is specific to that cloud. For OpenStack deployments, you will

prepare-vms/infra/example.terraform.openstack

@@ -1,4 +1,5 @@
-INFRACLASS=openstack-tf
+INFRACLASS=terraform
+TERRAFORM=openstack
 
 # If you are using OpenStack, copy this file (e.g. to "openstack" or "enix")
 # and customize the variables below.

prepare-vms/lib/cli.sh

@@ -88,3 +88,8 @@ need_settings() {
         die "Settings file $1 doesn't exist."
     fi
 }
+
+need_login_password() {
+    USER_LOGIN=$(yq -r .user_login < tags/$TAG/settings.yaml)
+    USER_PASSWORD=$(yq -r .user_password < tags/$TAG/settings.yaml)
+}

prepare-vms/lib/clusterize.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+import os
+import sys
+import time
+import yaml
+
+#################################
+
+config = yaml.load(open("/tmp/settings.yaml"))
+CLUSTER_SIZE = config["clustersize"]
+CLUSTER_PREFIX = config["clusterprefix"]
+
+#################################
+
+# This script will be run as ubuntu user, which has root privileges.
+
+STEP = 0
+
+def bold(msg):
+    return "{} {} {}".format("$(tput smso)", msg, "$(tput rmso)")
+
+def system(cmd):
+    global STEP
+    with open("/tmp/pp.status", "a") as f:
+        t1 = time.time()
+        f.write(bold("--- RUNNING [step {}] ---> {}...".format(STEP, cmd)))
+        retcode = os.system(cmd)
+        t2 = time.time()
+        td = str(t2-t1)[:5]
+        f.write(bold("[{}] in {}s\n".format(retcode, td)))
+        STEP += 1
+    with open(os.environ["HOME"] + "/.bash_history", "a") as f:
+        f.write("{}\n".format(cmd))
+    if retcode != 0:
+        msg = "The following command failed with exit code {}:\n".format(retcode)
+        msg+= cmd
+        raise(Exception(msg))
+
+# Get our public IP address
+# ipv4_retrieval_endpoint = "http://169.254.169.254/latest/meta-data/public-ipv4"
+ipv4_retrieval_endpoint = "http://myip.enix.org/REMOTE_ADDR"
+system("curl --silent {} > /tmp/ipv4".format(ipv4_retrieval_endpoint))
+ipv4 = open("/tmp/ipv4").read()
+system("echo HOSTIP={} | sudo tee -a /etc/environment".format(ipv4))
+
+### BEGIN CLUSTERING ###
+
+addresses = list(l.strip() for l in sys.stdin)
+
+assert ipv4 in addresses
+
+def makenames(addrs):
+    return [ "%s%s"%(CLUSTER_PREFIX, i+1) for i in range(len(addrs)) ]
+
+while addresses:
+    cluster = addresses[:CLUSTER_SIZE]
+    addresses = addresses[CLUSTER_SIZE:]
+    if ipv4 not in cluster:
+        continue
+    names = makenames(cluster)
+    for ipaddr, name in zip(cluster, names):
+        system("grep ^{} /etc/hosts || echo {} {} | sudo tee -a /etc/hosts"
+                    .format(ipaddr, ipaddr, name))
+    print(cluster)
+
+    mynode = cluster.index(ipv4) + 1
+    system("echo {}{} | sudo tee /etc/hostname".format(CLUSTER_PREFIX, mynode))
+    system("sudo hostname {}{}".format(CLUSTER_PREFIX, mynode))
+
+    # Record the IPV4 and name of the first node
+    system("echo {} | sudo tee /etc/ipv4_of_first_node".format(cluster[0]))
+    system("echo {} | sudo tee /etc/name_of_first_node".format(names[0]))
+
+    # Create a convenience file to easily check if we're the first node
+    if ipv4 == cluster[0]:
+        system("sudo ln -sf /bin/true /usr/local/bin/i_am_first_node")
+    else:
+        system("sudo ln -sf /bin/false /usr/local/bin/i_am_first_node")

prepare-vms/lib/commands.sh

@@ -57,30 +57,105 @@ _cmd_clean() {
 	done	
 }
 
-_cmd deploy "Install Docker on a bunch of running VMs"
-_cmd_deploy() {
+_cmd createuser "Create the user that students will use"
+_cmd_createuser() {
     TAG=$1
     need_tag
+    need_login_password
 
-    # wait until all hosts are reachable before trying to deploy
-    info "Trying to reach $TAG instances..."
-    while ! tag_is_reachable; do
-        >/dev/stderr echo -n "."
-        sleep 2
-    done
-    >/dev/stderr echo ""
-
-    echo deploying > tags/$TAG/status
-    sep "Deploying tag $TAG"
-
-    # If this VM image is using cloud-init,
-    # wait for cloud-init to be done
     pssh "
-    if [ -d /var/lib/cloud ]; then
-        while [ ! -f /var/lib/cloud/instance/boot-finished ]; do
-            sleep 1
-        done
-    fi"
+    set -e
+    # Create the user if it doesn't exist yet.
+    id $USER_LOGIN || sudo useradd -d /home/$USER_LOGIN -g users -m -s /bin/bash $USER_LOGIN
+    # Add them to the docker group, if there is one.
+    grep ^docker: /etc/group && sudo usermod -aG docker $USER_LOGIN
+    # Set their password.
+    echo $USER_LOGIN:$USER_PASSWORD | sudo chpasswd
+    # Add them to sudoers and allow passwordless authentication.
+    echo '$USER_LOGIN ALL=(ALL) NOPASSWD:ALL' | sudo tee /etc/sudoers.d/$USER_LOGIN
+    "
+
+    # The MaxAuthTries is here to help with folks who have many SSH keys.
+    pssh "
+    set -e
+    sudo sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config
+    sudo sed -i 's/#MaxAuthTries 6/MaxAuthTries 42/' /etc/ssh/sshd_config
+    sudo service ssh restart
+    "
+
+    pssh "
+    set -e
+    cd /home/$USER_LOGIN
+    sudo -u $USER_LOGIN mkdir -p .ssh
+    if i_am_first_node; then
+      # Generate a key pair with an empty passphrase.
+      if ! sudo -u $USER_LOGIN [ -f .ssh/id_rsa ]; then
+        sudo -u $USER_LOGIN ssh-keygen -t rsa -f .ssh/id_rsa -P ''
+        sudo -u $USER_LOGIN cp .ssh/id_rsa.pub .ssh/authorized_keys
+      fi
+    fi
+    "
+
+    pssh "
+    set -e
+    cd /home/$USER_LOGIN
+    if ! i_am_first_node; then
+      # Copy keys from the first node.
+      ssh $SSHOPTS \$(cat /etc/name_of_first_node) sudo -u $USER_LOGIN tar -C /home/$USER_LOGIN -cvf- .ssh |
+      sudo -u $USER_LOGIN tar -xf-
+    fi
+    "
+
+    # FIXME do this only once.
+    pssh -I "sudo -u $USER_LOGIN tee -a /home/$USER_LOGIN/.bashrc" <<"SQRL"
+
+# Fancy prompt courtesy of @soulshake.
+export PS1='\e[1m\e[31m[$HOSTIP] \e[32m($(docker-prompt)) \e[34m\u@\h\e[35m \w\e[0m\n$ '
+
+# Bigger history, in a different file, and saved before executing each command.
+export HISTSIZE=9999
+export HISTFILESIZE=9999
+shopt -s histappend
+trap 'history -a' DEBUG
+export HISTFILE=~/.history
+SQRL
+
+    pssh -I "sudo -u $USER_LOGIN tee /home/$USER_LOGIN/.vimrc" <<SQRL
+syntax on
+set autoindent
+set expandtab
+set number
+set shiftwidth=2
+set softtabstop=2
+set nowrap
+SQRL
+
+    pssh -I "sudo -u $USER_LOGIN tee /home/$USER_LOGIN/.tmux.conf" <<SQRL
+bind h select-pane -L
+bind j select-pane -D
+bind k select-pane -U
+bind l select-pane -R
+
+# Allow using mouse to switch panes
+set -g mouse on
+
+# Make scrolling with wheels work
+
+bind -n WheelUpPane if-shell -F -t = "#{mouse_any_flag}" "send-keys -M" "if -Ft= '#{pane_in_mode}' 'send-keys -M' 'select-pane -t=; copy-mode -e; send-keys -M'"
+bind -n WheelDownPane select-pane -t= \; send-keys -M
+SQRL
+
+    # Install docker-prompt script
+    pssh -I sudo tee /usr/local/bin/docker-prompt <lib/docker-prompt
+    pssh sudo chmod +x /usr/local/bin/docker-prompt
+
+    echo user_ok > tags/$TAG/status
+}
+
+_cmd clusterize "Group VMs in clusters"
+_cmd_clusterize() {
+    TAG=$1
+    need_tag
 
     # Special case for scaleway since it doesn't come with sudo
     if [ "$INFRACLASS" = "scaleway" ]; then
@@ -103,6 +178,13 @@ _cmd_deploy() {
     #    install --owner=ubuntu --mode=600 /root/.ssh/authorized_keys --target-directory /home/ubuntu/.ssh"
     #fi
 
+    # Special case for oracle since their iptables blocks everything but SSH
+    pssh "
+    if [ -f /etc/iptables/rules.v4 ]; then
+        sudo sed -i 's/-A INPUT -j REJECT --reject-with icmp-host-prohibited//' /etc/iptables/rules.v4
+        sudo netfilter-persistent start
+    fi"
+
     # Copy settings and install Python YAML parser
     pssh -I tee /tmp/settings.yaml <tags/$TAG/settings.yaml
     pssh "
@@ -110,46 +192,25 @@ _cmd_deploy() {
     sudo apt-get install -y python-yaml"
 
     # If there is no "python" binary, symlink to python3
-    #pssh "
-    #if ! which python; then
-    #    ln -s $(which python3) /usr/local/bin/python
-    #fi"
+    pssh "
+    if ! which python; then
+        sudo ln -s $(which python3) /usr/local/bin/python
+    fi"
 
     # Copy postprep.py to the remote machines, and execute it, feeding it the list of IP addresses
-    pssh -I tee /tmp/postprep.py <lib/postprep.py
-    pssh --timeout 900 --send-input "python /tmp/postprep.py >>/tmp/pp.out 2>>/tmp/pp.err" <tags/$TAG/ips.txt
-
-    # Install docker-prompt script
-    pssh -I sudo tee /usr/local/bin/docker-prompt <lib/docker-prompt
-    pssh sudo chmod +x /usr/local/bin/docker-prompt
-
-    # If /home/docker/.ssh/id_rsa doesn't exist, copy it from the first node
-    pssh "
-    sudo -u docker [ -f /home/docker/.ssh/id_rsa ] ||
-    ssh $SSHOPTS \$(cat /etc/name_of_first_node) sudo -u docker tar -C /home/docker -cvf- .ssh |
-    sudo -u docker tar -C /home/docker -xf-"
-
-    # if 'docker@' doesn't appear in /home/docker/.ssh/authorized_keys, copy it there
-    pssh "
-    grep docker@ /home/docker/.ssh/authorized_keys ||
-    cat /home/docker/.ssh/id_rsa.pub |
-    sudo -u docker tee -a /home/docker/.ssh/authorized_keys"
+    pssh -I tee /tmp/clusterize.py <lib/clusterize.py
+    pssh --timeout 900 --send-input "python /tmp/clusterize.py >>/tmp/pp.out 2>>/tmp/pp.err" <tags/$TAG/ips.txt
 
     # On the first node, create and deploy TLS certs using Docker Machine
     # (Currently disabled.)
     true || pssh "
     if i_am_first_node; then
         grep '[0-9]\$' /etc/hosts |
-        xargs -n2 sudo -H -u docker \
-        docker-machine create -d generic --generic-ssh-user docker --generic-ip-address
+        xargs -n2 sudo -H -u $USER_LOGIN \
+        docker-machine create -d generic --generic-ssh-user $USER_LOGIN --generic-ip-address
     fi"
 
-    sep "Deployed tag $TAG"
-    echo deployed > tags/$TAG/status
-    info "You may want to run one of the following commands:"
-    info "$0 kube $TAG"
-    info "$0 pull_images $TAG"
-    info "$0 cards $TAG"
+    echo cluster_ok > tags/$TAG/status
 }
 
 _cmd disabledocker "Stop Docker Engine and don't restart it automatically"
@@ -158,10 +219,71 @@ _cmd_disabledocker() {
     need_tag
 
     pssh "
-    sudo systemctl disable docker.service
-    sudo systemctl disable docker.socket
-    sudo systemctl stop docker
-    sudo killall containerd
+    sudo systemctl disable docker.socket --now
+    sudo systemctl disable docker.service --now
+    sudo systemctl disable containerd.service --now
+    "
+}
+
+_cmd docker "Install and start Docker"
+_cmd_docker() {
+    TAG=$1
+    need_tag
+
+    pssh "
+    set -e
+    # On EC2, the ephemeral disk might be mounted on /mnt.
+    # If /mnt is a mountpoint, place Docker workspace on it.
+    if mountpoint -q /mnt; then
+      sudo mkdir -p /mnt/docker
+      sudo ln -sfn /mnt/docker /var/lib/docker
+    fi
+
+    # containerd 1.6 breaks Weave.
+    # See https://github.com/containerd/containerd/issues/6921
+    sudo tee /etc/apt/preferences.d/containerd <<EOF
+Package: containerd.io
+Pin: version 1.5.*
+Pin-Priority: 1000
+EOF
+
+    # This will install the latest Docker.
+    sudo apt-get -qy install apt-transport-https ca-certificates curl software-properties-common
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+    sudo add-apt-repository 'deb https://download.docker.com/linux/ubuntu bionic stable'
+    sudo apt-get -q update
+    sudo apt-get -qy install docker-ce
+
+    # Add registry mirror configuration.
+    if ! [ -f /etc/docker/daemon.json ]; then
+        echo '{\"registry-mirrors\": [\"https://mirror.gcr.io\"]}' | sudo tee /etc/docker/daemon.json
+        sudo systemctl restart docker
+    fi
+    "
+
+    ##VERSION## https://github.com/docker/compose/releases
+    if [ "$ARCHITECTURE" ]; then
+        COMPOSE_VERSION=v2.2.3
+        COMPOSE_PLATFORM='linux-$(uname -m)'
+    else
+        COMPOSE_VERSION=1.29.2
+        COMPOSE_PLATFORM='Linux-$(uname -m)'
+    fi
+    pssh "
+    set -e
+    ### Install docker-compose.
+    sudo curl -fsSL -o /usr/local/bin/docker-compose \
+      https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-$COMPOSE_PLATFORM
+    sudo chmod +x /usr/local/bin/docker-compose
+    docker-compose version
+
+    ### Install docker-machine.
+    ##VERSION## https://github.com/docker/machine/releases
+    MACHINE_VERSION=v0.16.2
+    sudo curl -fsSL -o /usr/local/bin/docker-machine \
+      https://github.com/docker/machine/releases/download/\$MACHINE_VERSION/docker-machine-\$(uname -s)-\$(uname -m)
+    sudo chmod +x /usr/local/bin/docker-machine
+    docker-machine version
     "
 }
 
@@ -174,23 +296,24 @@ _cmd_kubebins() {
     ETCD_VERSION=v3.4.13
     K8SBIN_VERSION=v1.19.11 # Can't go to 1.20 because it requires a serviceaccount signing key.
     CNI_VERSION=v0.8.7
+    ARCH=${ARCHITECTURE-amd64}
     pssh --timeout 300 "
     set -e
     cd /usr/local/bin
     if ! [ -x etcd ]; then
-        curl -L https://github.com/etcd-io/etcd/releases/download/$ETCD_VERSION/etcd-$ETCD_VERSION-linux-amd64.tar.gz \
+        curl -L https://github.com/etcd-io/etcd/releases/download/$ETCD_VERSION/etcd-$ETCD_VERSION-linux-$ARCH.tar.gz \
         | sudo tar --strip-components=1 --wildcards -zx '*/etcd' '*/etcdctl'
     fi
     if ! [ -x hyperkube ]; then
         ##VERSION##
-        curl -L https://dl.k8s.io/$K8SBIN_VERSION/kubernetes-server-linux-amd64.tar.gz \
+        curl -L https://dl.k8s.io/$K8SBIN_VERSION/kubernetes-server-linux-$ARCH.tar.gz \
         | sudo tar --strip-components=3 -zx \
           kubernetes/server/bin/kube{ctl,let,-proxy,-apiserver,-scheduler,-controller-manager}
     fi
     sudo mkdir -p /opt/cni/bin
     cd /opt/cni/bin
     if ! [ -x bridge ]; then
-        curl -L https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION/cni-plugins-linux-amd64-$CNI_VERSION.tgz \
+        curl -L https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION/cni-plugins-linux-$ARCH-$CNI_VERSION.tgz \
         | sudo tar -zx
     fi
     "
@@ -200,15 +323,18 @@ _cmd kube "Setup kubernetes clusters with kubeadm (must be run AFTER deploy)"
 _cmd_kube() {
     TAG=$1
     need_tag
+    need_login_password
 
     # Optional version, e.g. 1.13.5
-    KUBEVERSION=$2
+    SETTINGS=tags/$TAG/settings.yaml
+    KUBEVERSION=$(awk '/^kubernetes_version:/ {print $2}' $SETTINGS)
     if [ "$KUBEVERSION" ]; then
-        EXTRA_APTGET="=$KUBEVERSION-00"
-        EXTRA_KUBEADM="kubernetesVersion: v$KUBEVERSION"
-    else
-        EXTRA_APTGET=""
-        EXTRA_KUBEADM=""
+        pssh "
+        sudo tee /etc/apt/preferences.d/kubernetes <<EOF
+Package: kubectl kubeadm kubelet
+Pin: version $KUBEVERSION*
+Pin-Priority: 1000
+EOF"
     fi
 
     # Install packages
@@ -219,7 +345,8 @@ _cmd_kube() {
     sudo tee /etc/apt/sources.list.d/kubernetes.list"
     pssh --timeout 200 "
     sudo apt-get update -q &&
-    sudo apt-get install -qy kubelet$EXTRA_APTGET kubeadm$EXTRA_APTGET kubectl$EXTRA_APTGET &&
+    sudo apt-get install -qy kubelet kubeadm kubectl &&
+    sudo apt-mark hold kubelet kubeadm kubectl
     kubectl completion bash | sudo tee /etc/bash_completion.d/kubectl &&
     echo 'alias k=kubectl' | sudo tee /etc/bash_completion.d/k &&
     echo 'complete -F __start_kubectl k' | sudo tee -a /etc/bash_completion.d/k"
@@ -231,6 +358,11 @@ _cmd_kube() {
         sudo swapoff -a"
     fi
 
+    # Re-enable CRI interface in containerd
+    pssh "
+    echo '# Use default parameters for containerd.' | sudo tee /etc/containerd/config.toml
+    sudo systemctl restart containerd"
+
     # Initialize kube control plane
     pssh --timeout 200 "
     if i_am_first_node && [ ! -f /etc/kubernetes/admin.conf ]; then
@@ -240,29 +372,48 @@ kind: InitConfiguration
 apiVersion: kubeadm.k8s.io/v1beta2
 bootstrapTokens:
 - token: \$(cat /tmp/token)
+nodeRegistration:
+  # Comment out the next line to switch back to Docker.
+  criSocket: /run/containerd/containerd.sock
+  ignorePreflightErrors:
+  - NumCPU
+---
+kind: JoinConfiguration
+apiVersion: kubeadm.k8s.io/v1beta2
+discovery:
+  bootstrapToken:
+    apiServerEndpoint: \$(cat /etc/name_of_first_node):6443
+    token: \$(cat /tmp/token)
+    unsafeSkipCAVerification: true
+nodeRegistration:
+  # Comment out the next line to switch back to Docker.
+  criSocket: /run/containerd/containerd.sock
+  ignorePreflightErrors:
+  - NumCPU
 ---
 kind: KubeletConfiguration
 apiVersion: kubelet.config.k8s.io/v1beta1
-cgroupDriver: cgroupfs
+# The following line is necessary when using Docker.
+# It doesn't seem necessary when using containerd.
+#cgroupDriver: cgroupfs
 ---
 kind: ClusterConfiguration
 apiVersion: kubeadm.k8s.io/v1beta2
 apiServer:
   certSANs:
   - \$(cat /tmp/ipv4)
-$EXTRA_KUBEADM
 EOF
-	sudo kubeadm init --config=/tmp/kubeadm-config.yaml --ignore-preflight-errors=NumCPU
+	sudo kubeadm init --config=/tmp/kubeadm-config.yaml
     fi"
 
-    # Put kubeconfig in ubuntu's and docker's accounts
+    # Put kubeconfig in ubuntu's and $USER_LOGIN's accounts
     pssh "
     if i_am_first_node; then
-        sudo mkdir -p \$HOME/.kube /home/docker/.kube &&
+        sudo mkdir -p \$HOME/.kube /home/$USER_LOGIN/.kube &&
         sudo cp /etc/kubernetes/admin.conf \$HOME/.kube/config &&
-        sudo cp /etc/kubernetes/admin.conf /home/docker/.kube/config &&
+        sudo cp /etc/kubernetes/admin.conf /home/$USER_LOGIN/.kube/config &&
         sudo chown -R \$(id -u) \$HOME/.kube &&
-        sudo chown -R docker /home/docker/.kube
+        sudo chown -R $USER_LOGIN /home/$USER_LOGIN/.kube
     fi"
 
     # Install weave as the pod network
@@ -276,14 +427,17 @@ EOF
     pssh --timeout 200 "
     if ! i_am_first_node && [ ! -f /etc/kubernetes/kubelet.conf ]; then
         FIRSTNODE=\$(cat /etc/name_of_first_node) &&
-        TOKEN=\$(ssh $SSHOPTS \$FIRSTNODE cat /tmp/token) &&
-        sudo kubeadm join --discovery-token-unsafe-skip-ca-verification --token \$TOKEN \$FIRSTNODE:6443
+        ssh $SSHOPTS \$FIRSTNODE cat /tmp/kubeadm-config.yaml > /tmp/kubeadm-config.yaml &&
+        sudo kubeadm join --config /tmp/kubeadm-config.yaml
     fi"
 
     # Install metrics server
     pssh "
     if i_am_first_node; then
 	kubectl apply -f https://raw.githubusercontent.com/jpetazzo/container.training/master/k8s/metrics-server.yaml
+    #helm upgrade --install metrics-server \
+    #     --repo https://kubernetes-sigs.github.io/metrics-server/ metrics-server \
+    #     --namespace kube-system --set args={--kubelet-insecure-tls}
     fi"
 }
 
@@ -291,31 +445,65 @@ _cmd kubetools "Install a bunch of CLI tools for Kubernetes"
 _cmd_kubetools() {
     TAG=$1
     need_tag
+    need_login_password
+
+    ARCH=${ARCHITECTURE-amd64}
+
+    # Folks, please, be consistent!
+    # Either pick "uname -m" (on Linux, that's x86_64, aarch64, etc.)
+    # Or GOARCH (amd64, arm64, etc.)
+    # But don't mix both! Thank you ♥
+    case $ARCH in
+    amd64)
+        HERP_DERP_ARCH=x86_64
+        TILT_ARCH=x86_64
+        ;;
+    *)
+        HERP_DERP_ARCH=$ARCH
+        TILT_ARCH=${ARCH}_ALPHA
+        ;;
+    esac
 
     # Install kubectx and kubens
     pssh "
-    [ -d kubectx ] || git clone https://github.com/ahmetb/kubectx &&
-    sudo ln -sf \$HOME/kubectx/kubectx /usr/local/bin/kctx &&
-    sudo ln -sf \$HOME/kubectx/kubens /usr/local/bin/kns &&
-    sudo cp \$HOME/kubectx/completion/*.bash /etc/bash_completion.d &&
-    [ -d kube-ps1 ] || git clone https://github.com/jonmosco/kube-ps1 &&
-    sudo -u docker sed -i s/docker-prompt/kube_ps1/ /home/docker/.bashrc &&
-    sudo -u docker tee -a /home/docker/.bashrc <<EOF
-. \$HOME/kube-ps1/kube-ps1.sh
+    set -e
+    if ! [ -x /usr/local/bin/kctx ]; then
+      cd /tmp
+      git clone https://github.com/ahmetb/kubectx
+      sudo cp kubectx/kubectx /usr/local/bin/kctx
+      sudo cp kubectx/kubens /usr/local/bin/kns
+      sudo cp kubectx/completion/*.bash /etc/bash_completion.d
+    fi"
+
+    # Install kube-ps1
+    pssh "
+    set -e
+    if ! [ -f /etc/profile.d/kube-ps1.sh ]; then
+      cd /tmp
+      git clone https://github.com/jonmosco/kube-ps1
+      sudo cp kube-ps1/kube-ps1.sh /etc/profile.d/kube-ps1.sh
+      sudo -u $USER_LOGIN sed -i s/docker-prompt/kube_ps1/ /home/$USER_LOGIN/.bashrc &&
+      sudo -u $USER_LOGIN tee -a /home/$USER_LOGIN/.bashrc <<EOF
 KUBE_PS1_PREFIX=""
 KUBE_PS1_SUFFIX=""
 KUBE_PS1_SYMBOL_ENABLE="false"
 KUBE_PS1_CTX_COLOR="green"
 KUBE_PS1_NS_COLOR="green"
-EOF"
+EOF
+    fi"
 
     # Install stern
+    ##VERSION## https://github.com/stern/stern/releases
+    STERN_VERSION=1.20.1
+    FILENAME=stern_${STERN_VERSION}_linux_${ARCH}
+    URL=https://github.com/stern/stern/releases/download/v$STERN_VERSION/$FILENAME.tar.gz
     pssh "
     if [ ! -x /usr/local/bin/stern ]; then
-        ##VERSION##
-        sudo curl -L -o /usr/local/bin/stern https://github.com/wercker/stern/releases/download/1.11.0/stern_linux_amd64 &&
-        sudo chmod +x /usr/local/bin/stern &&
+        curl -fsSL $URL |
+        sudo tar -C /usr/local/bin -zx --strip-components=1 $FILENAME/stern
+        sudo chmod +x /usr/local/bin/stern
         stern --completion bash | sudo tee /etc/bash_completion.d/stern
+        stern --version
     fi"
 
     # Install helm
@@ -323,24 +511,30 @@ EOF"
     if [ ! -x /usr/local/bin/helm ]; then
         curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get-helm-3 | sudo bash &&
         helm completion bash | sudo tee /etc/bash_completion.d/helm
+        helm version
     fi"
 
     # Install kustomize
+    ##VERSION## https://github.com/kubernetes-sigs/kustomize/releases
+    KUSTOMIZE_VERSION=v4.4.0
+    URL=https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_linux_${ARCH}.tar.gz
     pssh "
     if [ ! -x /usr/local/bin/kustomize ]; then
-        ##VERSION##
-        curl -L https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v3.6.1/kustomize_v3.6.1_linux_amd64.tar.gz |
-            sudo tar -C /usr/local/bin -zx kustomize
-        echo complete -C /usr/local/bin/kustomize kustomize | sudo tee /etc/bash_completion.d/kustomize
+        curl -fsSL $URL |
+        sudo tar -C /usr/local/bin -zx kustomize
+        kustomize completion bash | sudo tee /etc/bash_completion.d/kustomize
+        kustomize version
     fi"
 
     # Install ship
     # Note: 0.51.3 is the last version that doesn't display GIN-debug messages
     # (don't want to get folks confused by that!)
+    # Only install ship on Intel platforms (no ARM 64 builds).
+    [ "$ARCH" = "amd64" ] &&
     pssh "
     if [ ! -x /usr/local/bin/ship ]; then
         ##VERSION##
-        curl -L https://github.com/replicatedhq/ship/releases/download/v0.51.3/ship_0.51.3_linux_amd64.tar.gz |
+        curl -fsSL https://github.com/replicatedhq/ship/releases/download/v0.51.3/ship_0.51.3_linux_$ARCH.tar.gz |
              sudo tar -C /usr/local/bin -zx ship
     fi"
 
@@ -348,61 +542,81 @@ EOF"
     pssh "
     if [ ! -x /usr/local/bin/aws-iam-authenticator ]; then
         ##VERSION##
-        sudo curl -o /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.12.7/2019-03-27/bin/linux/amd64/aws-iam-authenticator
+        sudo curl -fsSLo /usr/local/bin/aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.12.7/2019-03-27/bin/linux/$ARCH/aws-iam-authenticator
 	sudo chmod +x /usr/local/bin/aws-iam-authenticator
+    aws-iam-authenticator version
     fi"
 
     # Install the krew package manager
     pssh "
-    if [ ! -d /home/docker/.krew ]; then
+    if [ ! -d /home/$USER_LOGIN/.krew ]; then
         cd /tmp &&
-        curl -fsSL https://github.com/kubernetes-sigs/krew/releases/latest/download/krew.tar.gz |
+        KREW=krew-linux_$ARCH
+        curl -fsSL https://github.com/kubernetes-sigs/krew/releases/latest/download/\$KREW.tar.gz |
         tar -zxf- &&
-        sudo -u docker -H ./krew-linux_amd64 install krew &&
-        echo export PATH=/home/docker/.krew/bin:\\\$PATH | sudo -u docker tee -a /home/docker/.bashrc
+        sudo -u $USER_LOGIN -H ./\$KREW install krew &&
+        echo export PATH=/home/$USER_LOGIN/.krew/bin:\\\$PATH | sudo -u $USER_LOGIN tee -a /home/$USER_LOGIN/.bashrc
     fi"
 
     # Install k9s
     pssh "
     if [ ! -x /usr/local/bin/k9s ]; then
-        VERSION=v0.24.10 &&
-        FILENAME=k9s_\${VERSION}_\$(uname -s)_\$(uname -m).tar.gz &&
-        curl -sSL https://github.com/derailed/k9s/releases/download/\$VERSION/\$FILENAME |
+        FILENAME=k9s_Linux_$HERP_DERP_ARCH.tar.gz &&
+        curl -fsSL https://github.com/derailed/k9s/releases/latest/download/\$FILENAME |
         sudo tar -zxvf- -C /usr/local/bin k9s
+        k9s version
     fi"
 
     # Install popeye
     pssh "
     if [ ! -x /usr/local/bin/popeye ]; then
-        FILENAME=popeye_\$(uname -s)_\$(uname -m).tar.gz &&
-        curl -sSL https://github.com/derailed/popeye/releases/latest/download/\$FILENAME |
+        FILENAME=popeye_Linux_$HERP_DERP_ARCH.tar.gz &&
+        curl -fsSL https://github.com/derailed/popeye/releases/latest/download/\$FILENAME |
         sudo tar -zxvf- -C /usr/local/bin popeye
+        popeye version
     fi"
 
     # Install Tilt
+    # Official instructions:
+    # curl -fsSL https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.sh | bash
+    # But the install script is not arch-aware (see https://github.com/tilt-dev/tilt/pull/5050).
     pssh "
     if [ ! -x /usr/local/bin/tilt ]; then
-        curl -fsSL https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.sh | bash
+        TILT_VERSION=0.22.15
+        FILENAME=tilt.\$TILT_VERSION.linux.$TILT_ARCH.tar.gz
+        curl -fsSL https://github.com/tilt-dev/tilt/releases/download/v\$TILT_VERSION/\$FILENAME |
+        sudo tar -zxvf- -C /usr/local/bin tilt
+        tilt version
     fi"
 
     # Install Skaffold
     pssh "
     if [ ! -x /usr/local/bin/skaffold ]; then
-        curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64 &&
+        curl -fsSLo skaffold https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-$ARCH &&
         sudo install skaffold /usr/local/bin/
+        skaffold version
     fi"
 
     # Install Kompose
     pssh "
     if [ ! -x /usr/local/bin/kompose ]; then
-        curl -Lo kompose https://github.com/kubernetes/kompose/releases/latest/download/kompose-linux-amd64 &&
+        curl -fsSLo kompose https://github.com/kubernetes/kompose/releases/latest/download/kompose-linux-$ARCH &&
         sudo install kompose /usr/local/bin
+        kompose version
     fi"
 
+    ##VERSION## https://github.com/bitnami-labs/sealed-secrets/releases
+    KUBESEAL_VERSION=0.17.4
+    #case $ARCH in
+    #amd64) FILENAME=kubeseal-linux-amd64;;
+    #arm64) FILENAME=kubeseal-arm64;;
+    #*)     FILENAME=nope;;
+    #esac
     pssh "
     if [ ! -x /usr/local/bin/kubeseal ]; then
-        curl -Lo kubeseal https://github.com/bitnami-labs/sealed-secrets/releases/download/v0.13.1/kubeseal-linux-amd64 &&
-        sudo install kubeseal /usr/local/bin
+        curl -fsSL https://github.com/bitnami-labs/sealed-secrets/releases/download/v$KUBESEAL_VERSION/kubeseal-$KUBESEAL_VERSION-linux-$ARCH.tar.gz |
+        sudo tar -zxvf- -C /usr/local/bin kubeseal
+        kubeseal --version
     fi"
 }
 
@@ -429,6 +643,7 @@ _cmd_kubetest() {
         echo \$NODE ; kubectl get nodes | grep -w \$NODE | grep -w Ready
       done
     fi"
+    echo kube_ok > tags/$TAG/status
 }
 
 _cmd ips "Show the IP addresses for a given tag"
@@ -474,14 +689,6 @@ _cmd_maketag() {
     date +%Y-%m-%d-%H-%M-$MS-$USER
 }
 
-_cmd ping "Ping VMs in a given tag, to check that they have network access"
-_cmd_ping() {
-    TAG=$1
-    need_tag
-
-    fping < tags/$TAG/ips.txt
-}
-
 _cmd netfix "Disable GRO and run a pinger job on the VMs"
 _cmd_netfix () {
     TAG=$1
@@ -507,14 +714,28 @@ EOF
     sudo systemctl start pinger"
 }
 
+_cmd ping "Ping VMs in a given tag, to check that they have network access"
+_cmd_ping() {
+    TAG=$1
+    need_tag
+
+    fping < tags/$TAG/ips.txt
+}
+
 _cmd tailhist "Install history viewer on port 1088"
 _cmd_tailhist () {
     TAG=$1
     need_tag
+    need_login_password
+
+    ARCH=${ARCHITECTURE-amd64}
+    [ "$ARCH" = "aarch64" ] && ARCH=arm64
 
     pssh "
-    wget https://github.com/joewalnes/websocketd/releases/download/v0.3.0/websocketd-0.3.0_amd64.deb
-    sudo dpkg -i websocketd-0.3.0_amd64.deb
+    set -e
+    wget https://github.com/joewalnes/websocketd/releases/download/v0.3.0/websocketd-0.3.0-linux_$ARCH.zip
+    unzip websocketd-0.3.0-linux_$ARCH.zip websocketd
+    sudo mv websocketd /usr/local/bin/websocketd
     sudo mkdir -p /tmp/tailhist
     sudo tee /root/tailhist.service <<EOF
 [Unit]
@@ -525,16 +746,32 @@ WantedBy=multi-user.target
 
 [Service]
 WorkingDirectory=/tmp/tailhist
-ExecStart=/usr/bin/websocketd --port=1088 --staticdir=. sh -c \"tail -n +1 -f /home/docker/.history || echo 'Could not read history file. Perhaps you need to \\\"chmod +r .history\\\"?'\"
+ExecStart=/usr/local/bin/websocketd --port=1088 --staticdir=. sh -c \"tail -n +1 -f /home/$USER_LOGIN/.history || echo 'Could not read history file. Perhaps you need to \\\"chmod +r .history\\\"?'\"
 User=nobody
 Group=nogroup
 Restart=always
 EOF
-    sudo systemctl enable /root/tailhist.service
-    sudo systemctl start tailhist"
+    sudo systemctl enable /root/tailhist.service --now
+    "
+
     pssh -I sudo tee /tmp/tailhist/index.html <lib/tailhist.html
 }
 
+_cmd tools "Install a bunch of useful tools (editors, git, jq...)"
+_cmd_tools() {
+    TAG=$1
+    need_tag
+
+    pssh "
+    sudo apt-get -q update
+    sudo apt-get -qy install apache2-utils emacs-nox git httping htop jid joe jq mosh python-setuptools tree unzip
+    # This is for VMs with broken PRNG (symptom: running docker-compose randomly hangs)
+    sudo apt-get -qy install haveged
+    # I don't remember why we need to remove this
+    sudo apt-get remove -y --purge dnsmasq-base
+    "
+}
+
 _cmd opensg "Open the default security group to ALL ingress traffic"
 _cmd_opensg() {
     need_infra $1
@@ -600,9 +837,10 @@ _cmd ssh "Open an SSH session to the first node of a tag"
 _cmd_ssh() {
     TAG=$1
     need_tag
+    need_login_password
     IP=$(head -1 tags/$TAG/ips.txt)
-    info "Logging into $IP"
-    ssh $SSHOPTS docker@$IP
+    info "Logging into $IP (default password: $USER_PASSWORD)"
+    ssh $SSHOPTS $USER_LOGIN@$IP
 
 }
 
@@ -651,7 +889,7 @@ _cmd_start() {
     infra_start $COUNT
     sep
     info "Successfully created $COUNT instances with tag $TAG"
-    echo created > tags/$TAG/status
+    echo create_ok > tags/$TAG/status
 
     # If the settings.yaml file has a "steps" field,
     # automatically execute all the actions listed in that field.
@@ -665,8 +903,7 @@ _cmd_start() {
         if [ -z "$step" ]; then
             break
         fi
-        sep
-        info "Automatically executing step '$step'."
+        sep "$TAG -> $step"
         TRY=1
         MAXTRY=10
         while ! $0 $step $TAG ; do
@@ -678,7 +915,7 @@ _cmd_start() {
                 die "Giving up."
             else
                 sep
-                info "Step '$step' failed. Let's wait 10 seconds and try again."
+                info "Step '$step' failed for '$TAG'. Let's wait 10 seconds and try again."
                 info "(Attempt $TRY out of $MAXTRY.)"
                 sleep 10
             fi
@@ -751,7 +988,7 @@ _cmd_helmprom() {
     need_tag
     pssh "
     if i_am_first_node; then
-        sudo -u docker -H helm upgrade --install prometheus prometheus \
+        sudo -u $USER_LOGIN -H helm upgrade --install prometheus prometheus \
             --repo https://prometheus-community.github.io/helm-charts/ \
             --namespace prometheus --create-namespace \
             --set server.service.type=NodePort \
@@ -786,6 +1023,30 @@ _cmd_passwords() {
     info "Done."
 }
 
+_cmd wait "Wait until VMs are ready (reachable and cloud init is done)"
+_cmd_wait() {
+    TAG=$1
+    need_tag
+
+    # Wait until all hosts are reachable.
+    info "Trying to reach $TAG instances..."
+    while ! pssh -t 5 true 2>&1 >/dev/null; do
+        >/dev/stderr echo -n "."
+        sleep 2
+    done
+    >/dev/stderr echo ""
+
+    # If this VM image is using cloud-init,
+    # wait for cloud-init to be done
+    info "Waiting for cloud-init to be done on $TAG instances..."
+    pssh "
+    if [ -d /var/lib/cloud ]; then
+        while [ ! -f /var/lib/cloud/instance/boot-finished ]; do
+            sleep 1
+        done
+    fi"
+}
+
 # Sometimes, weave fails to come up on some nodes.
 # Symptom: the pods on a node are unreachable (they don't even ping).
 # Remedy: wipe out Weave state and delete weave pod on that node.
@@ -808,7 +1069,8 @@ _cmd_webssh() {
     need_tag
     pssh "
     sudo apt-get update &&
-    sudo apt-get install python-tornado python-paramiko -y"
+    sudo apt-get install python-tornado python-paramiko -y ||
+    sudo apt-get install python3-tornado python3-paramiko -y"
     pssh "
     cd /opt
     [ -d webssh ] || sudo git clone https://github.com/jpetazzo/webssh"
@@ -872,16 +1134,12 @@ pull_tag() {
         google/cadvisor \
         dockersamples/visualizer \
         nathanleclaire/redisonrails; do
-        sudo -u docker docker pull $I
+        sudo docker pull $I
     done'
 
     info "Finished pulling images for $TAG."
 }
 
-tag_is_reachable() {
-    pssh -t 5 true 2>&1 >/dev/null
-}
-
 test_tag() {
     ips_file=tags/$TAG/ips.txt
     info "Picking a random IP address in $ips_file to run tests."

prepare-vms/lib/infra/aws.sh

@@ -217,7 +217,7 @@ aws_tag_instances() {
 
 aws_get_ami() {
     ##VERSION##
-    find_ubuntu_ami -r $AWS_DEFAULT_REGION -a ${AWS_ARCHITECTURE-amd64} -v 18.04 -t hvm:ebs -N -q
+    find_ubuntu_ami -r $AWS_DEFAULT_REGION -a ${ARCHITECTURE-amd64} -v 18.04 -t hvm:ebs -N -q
 }
 
 aws_greet() {