From 0a277c856421c4556323cc4353c32ac4ef30c3a2 Mon Sep 17 00:00:00 2001 From: Markus Kahl Date: Tue, 5 Apr 2022 09:38:24 +0100 Subject: [PATCH] kubernetes setup (#24) --- kubernetes/01-backend-networkpolicy.yaml | 13 ++++ kubernetes/01-frontend-networkpolicy.yaml | 13 ++++ .../01-opdata-persistentvolumeclaim.yaml | 14 ++++ .../01-pgdata-persistentvolumeclaim.yaml | 14 ++++ kubernetes/02-cache-deployment.yaml | 24 +++++++ kubernetes/02-db-deployment.yaml | 41 ++++++++++++ kubernetes/03-cache-service.yaml | 15 +++++ kubernetes/03-db-service.yaml | 15 +++++ kubernetes/04-seeder-pod.yaml | 37 +++++++++++ kubernetes/05-cron-deployment.yaml | 48 ++++++++++++++ kubernetes/05-web-deployment.yaml | 62 ++++++++++++++++++ kubernetes/05-worker-deployment.yaml | 48 ++++++++++++++ kubernetes/06-web-service.yaml | 15 +++++ kubernetes/07-proxy-deployment.yaml | 32 +++++++++ kubernetes/08-proxy-service.yaml | 14 ++++ kubernetes/09-proxy-ingress.yaml | 17 +++++ kubernetes/README.md | 65 +++++++++++++++++++ 17 files changed, 487 insertions(+) create mode 100755 kubernetes/01-backend-networkpolicy.yaml create mode 100755 kubernetes/01-frontend-networkpolicy.yaml create mode 100755 kubernetes/01-opdata-persistentvolumeclaim.yaml create mode 100755 kubernetes/01-pgdata-persistentvolumeclaim.yaml create mode 100755 kubernetes/02-cache-deployment.yaml create mode 100755 kubernetes/02-db-deployment.yaml create mode 100755 kubernetes/03-cache-service.yaml create mode 100755 kubernetes/03-db-service.yaml create mode 100755 kubernetes/04-seeder-pod.yaml create mode 100755 kubernetes/05-cron-deployment.yaml create mode 100755 kubernetes/05-web-deployment.yaml create mode 100755 kubernetes/05-worker-deployment.yaml create mode 100755 kubernetes/06-web-service.yaml create mode 100755 kubernetes/07-proxy-deployment.yaml create mode 100755 kubernetes/08-proxy-service.yaml create mode 100755 kubernetes/09-proxy-ingress.yaml create mode 100644 kubernetes/README.md diff --git a/kubernetes/01-backend-networkpolicy.yaml b/kubernetes/01-backend-networkpolicy.yaml new file mode 100755 index 0000000..135d183 --- /dev/null +++ b/kubernetes/01-backend-networkpolicy.yaml @@ -0,0 +1,13 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: backend +spec: + ingress: + - from: + - podSelector: + matchLabels: + openproject.network/backend: "true" + podSelector: + matchLabels: + openproject.network/backend: "true" diff --git a/kubernetes/01-frontend-networkpolicy.yaml b/kubernetes/01-frontend-networkpolicy.yaml new file mode 100755 index 0000000..9663d04 --- /dev/null +++ b/kubernetes/01-frontend-networkpolicy.yaml @@ -0,0 +1,13 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: frontend +spec: + ingress: + - from: + - podSelector: + matchLabels: + openproject.network/frontend: "true" + podSelector: + matchLabels: + openproject.network/frontend: "true" diff --git a/kubernetes/01-opdata-persistentvolumeclaim.yaml b/kubernetes/01-opdata-persistentvolumeclaim.yaml new file mode 100755 index 0000000..333a9d3 --- /dev/null +++ b/kubernetes/01-opdata-persistentvolumeclaim.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + creationTimestamp: null + labels: + openproject.service: opdata + name: opdata +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi +status: {} diff --git a/kubernetes/01-pgdata-persistentvolumeclaim.yaml b/kubernetes/01-pgdata-persistentvolumeclaim.yaml new file mode 100755 index 0000000..60efc2a --- /dev/null +++ b/kubernetes/01-pgdata-persistentvolumeclaim.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + creationTimestamp: null + labels: + openproject.service: pgdata + name: pgdata +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi +status: {} diff --git a/kubernetes/02-cache-deployment.yaml b/kubernetes/02-cache-deployment.yaml new file mode 100755 index 0000000..042f0f0 --- /dev/null +++ b/kubernetes/02-cache-deployment.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + openproject.service: cache + name: cache +spec: + replicas: 1 + selector: + matchLabels: + openproject.service: cache + strategy: {} + template: + metadata: + labels: + openproject.network/backend: "true" + openproject.service: cache + spec: + containers: + - image: memcached + name: cache + resources: {} + restartPolicy: Always +status: {} diff --git a/kubernetes/02-db-deployment.yaml b/kubernetes/02-db-deployment.yaml new file mode 100755 index 0000000..8ff7307 --- /dev/null +++ b/kubernetes/02-db-deployment.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + openproject.service: db + name: db +spec: + replicas: 1 + selector: + matchLabels: + openproject.service: db + strategy: + type: Recreate + template: + metadata: + labels: + openproject.network/backend: "true" + openproject.service: db + spec: + containers: + - env: + - name: POSTGRES_DB + value: openproject + - name: POSTGRES_PASSWORD + value: p4ssw0rd + image: postgres:13 + ports: + - containerPort: 5432 + name: psql + name: db + resources: {} + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: pgdata + restartPolicy: Always + terminationGracePeriodSeconds: 3 + volumes: + - name: pgdata + persistentVolumeClaim: + claimName: pgdata +status: {} diff --git a/kubernetes/03-cache-service.yaml b/kubernetes/03-cache-service.yaml new file mode 100755 index 0000000..752aba6 --- /dev/null +++ b/kubernetes/03-cache-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: cache + labels: + openproject.service: cache +spec: + type: NodePort + selector: + openproject.service: cache + ports: + - name: cache + protocol: TCP + port: 11211 + targetPort: 11211 diff --git a/kubernetes/03-db-service.yaml b/kubernetes/03-db-service.yaml new file mode 100755 index 0000000..b4baaf2 --- /dev/null +++ b/kubernetes/03-db-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: db + labels: + openproject.service: db +spec: + type: NodePort + selector: + openproject.service: db + ports: + - name: db + protocol: TCP + port: 5432 + targetPort: 5432 diff --git a/kubernetes/04-seeder-pod.yaml b/kubernetes/04-seeder-pod.yaml new file mode 100755 index 0000000..28613cd --- /dev/null +++ b/kubernetes/04-seeder-pod.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + openproject.network/backend: "true" + openproject.service: seeder + name: seeder +spec: + containers: + - args: + - ./docker/prod/seeder + env: + - name: DATABASE_URL + value: postgres://postgres:p4ssw0rd@$(DB_SERVICE_HOST):$(DB_SERVICE_PORT)/openproject?pool=20&encoding=unicode&reconnect=true + - name: IMAP_ENABLED + value: "false" + - name: OPENPROJECT_CACHE__MEMCACHE__SERVER + value: $(CACHE_SERVICE_HOST):$(CACHE_SERVICE_PORT) + - name: OPENPROJECT_RAILS__RELATIVE__URL__ROOT + - name: RAILS_CACHE_STORE + value: memcache + - name: RAILS_MAX_THREADS + value: "16" + - name: RAILS_MIN_THREADS + value: "4" + image: openproject/community:12 + name: seeder + resources: {} + volumeMounts: + - mountPath: /var/openproject/assets + name: opdata + restartPolicy: OnFailure + volumes: + - name: opdata + persistentVolumeClaim: + claimName: opdata +status: {} diff --git a/kubernetes/05-cron-deployment.yaml b/kubernetes/05-cron-deployment.yaml new file mode 100755 index 0000000..ecde8f1 --- /dev/null +++ b/kubernetes/05-cron-deployment.yaml @@ -0,0 +1,48 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + openproject.service: cron + name: cron +spec: + replicas: 1 + selector: + matchLabels: + openproject.service: cron + strategy: + type: Recreate + template: + metadata: + labels: + openproject.network/backend: "true" + openproject.service: cron + spec: + containers: + - args: + - ./docker/prod/cron + env: + - name: DATABASE_URL + value: postgres://postgres:p4ssw0rd@db/openproject?pool=20&encoding=unicode&reconnect=true + - name: IMAP_ENABLED + value: "false" + - name: OPENPROJECT_CACHE__MEMCACHE__SERVER + value: cache:11211 + - name: OPENPROJECT_RAILS__RELATIVE__URL__ROOT + - name: RAILS_CACHE_STORE + value: memcache + - name: RAILS_MAX_THREADS + value: "16" + - name: RAILS_MIN_THREADS + value: "4" + image: openproject/community:12 + name: cron + resources: {} + volumeMounts: + - mountPath: /var/openproject/assets + name: opdata + restartPolicy: Always + volumes: + - name: opdata + persistentVolumeClaim: + claimName: opdata +status: {} diff --git a/kubernetes/05-web-deployment.yaml b/kubernetes/05-web-deployment.yaml new file mode 100755 index 0000000..cdd3613 --- /dev/null +++ b/kubernetes/05-web-deployment.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + openproject.service: web + name: web +spec: + replicas: 1 + selector: + matchLabels: + openproject.service: web + strategy: + type: Recreate + template: + metadata: + labels: + openproject.network/backend: "true" + openproject.network/frontend: "true" + openproject.service: web + spec: + containers: + - args: + - ./docker/prod/web + env: + - name: DATABASE_URL + value: postgres://postgres:p4ssw0rd@$(DB_SERVICE_HOST):$(DB_SERVICE_PORT)/openproject?pool=20&encoding=unicode&reconnect=true + - name: IMAP_ENABLED + value: "false" + - name: OPENPROJECT_CACHE__MEMCACHE__SERVER + value: $(CACHE_SERVICE_HOST):$(CACHE_SERVICE_PORT) + - name: OPENPROJECT_RAILS__RELATIVE__URL__ROOT + - name: RAILS_CACHE_STORE + value: memcache + - name: RAILS_MAX_THREADS + value: "16" + - name: RAILS_MIN_THREADS + value: "4" + image: openproject/community:12 + ports: + - containerPort: 8080 + name: http + livenessProbe: + exec: + command: + - curl + - -f + - http://localhost:8080/health_checks/default + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 3 + name: web + resources: {} + volumeMounts: + - mountPath: /var/openproject/assets + name: opdata + restartPolicy: Always + volumes: + - name: opdata + persistentVolumeClaim: + claimName: opdata +status: {} diff --git a/kubernetes/05-worker-deployment.yaml b/kubernetes/05-worker-deployment.yaml new file mode 100755 index 0000000..49aa786 --- /dev/null +++ b/kubernetes/05-worker-deployment.yaml @@ -0,0 +1,48 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + openproject.service: worker + name: worker +spec: + replicas: 1 + selector: + matchLabels: + openproject.service: worker + strategy: + type: Recreate + template: + metadata: + labels: + openproject.network/backend: "true" + openproject.service: worker + spec: + containers: + - args: + - ./docker/prod/worker + env: + - name: DATABASE_URL + value: postgres://postgres:p4ssw0rd@$(DB_SERVICE_HOST):$(DB_SERVICE_PORT)/openproject?pool=20&encoding=unicode&reconnect=true + - name: IMAP_ENABLED + value: "false" + - name: OPENPROJECT_CACHE__MEMCACHE__SERVER + value: $(CACHE_SERVICE_HOST):$(CACHE_SERVICE_PORT) + - name: OPENPROJECT_RAILS__RELATIVE__URL__ROOT + - name: RAILS_CACHE_STORE + value: memcache + - name: RAILS_MAX_THREADS + value: "16" + - name: RAILS_MIN_THREADS + value: "4" + image: openproject/community:12 + name: worker + resources: {} + volumeMounts: + - mountPath: /var/openproject/assets + name: opdata + restartPolicy: Always + volumes: + - name: opdata + persistentVolumeClaim: + claimName: opdata +status: {} diff --git a/kubernetes/06-web-service.yaml b/kubernetes/06-web-service.yaml new file mode 100755 index 0000000..9d264b4 --- /dev/null +++ b/kubernetes/06-web-service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: web + labels: + openproject.service: web +spec: + type: NodePort + selector: + openproject.service: web + ports: + - name: web + protocol: TCP + port: 8080 + targetPort: 8080 diff --git a/kubernetes/07-proxy-deployment.yaml b/kubernetes/07-proxy-deployment.yaml new file mode 100755 index 0000000..07fad58 --- /dev/null +++ b/kubernetes/07-proxy-deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + openproject.service: proxy + name: proxy +spec: + replicas: 1 + selector: + matchLabels: + openproject.service: proxy + strategy: {} + template: + metadata: + labels: + openproject.network/frontend: "true" + openproject.service: proxy + spec: + containers: + - args: + - ./docker/prod/proxy + env: + - name: APP_HOST + value: $(WEB_SERVICE_HOST) + - name: OPENPROJECT_RAILS__RELATIVE__URL__ROOT + image: openproject/community:12 + name: proxy + ports: + - containerPort: 80 + resources: {} + restartPolicy: Always +status: {} diff --git a/kubernetes/08-proxy-service.yaml b/kubernetes/08-proxy-service.yaml new file mode 100755 index 0000000..851925b --- /dev/null +++ b/kubernetes/08-proxy-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + openproject.service: proxy + name: proxy +spec: + type: NodePort + selector: + openproject.service: proxy + ports: + - name: http + port: 80 + targetPort: 80 diff --git a/kubernetes/09-proxy-ingress.yaml b/kubernetes/09-proxy-ingress.yaml new file mode 100755 index 0000000..7b02faa --- /dev/null +++ b/kubernetes/09-proxy-ingress.yaml @@ -0,0 +1,17 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: proxy-ingress +spec: + ingressClassName: nginx + rules: + - host: k8s.openproject-dev.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: proxy + port: + number: 80 diff --git a/kubernetes/README.md b/kubernetes/README.md new file mode 100644 index 0000000..9cf68ac --- /dev/null +++ b/kubernetes/README.md @@ -0,0 +1,65 @@ +# OpenProject installation using Kubernetes + +This is an example setup of OpenProject on Kubernetes. + +## Install + +Clone this repository: + +``` +git clone https://github.com/opf/openproject-deploy --depth=1 --branch=stable/12 openproject +``` + +Go to the compose folder: + +``` +cd openproject/kubernetes +``` + +Adjust the host name for the ingress in [09-proxy-ingress.yaml](./09-proxy-ingress.yaml). +The default value `k8s.openproject-dev.com` simply points to `127.0.0.1`. +You will have to insert the actual host name here and set up the DNS so that it points to the cluster IP. + +Next, apply the definitions: + +``` +kubectl apply -f . +``` + +## Ingress + +For the ingress to work you will need to enable an ingress addon in your cluster. + +If you already have a load balancer you want to use to expose the service, +simply delete [09-proxy-ingress.yaml](./09-proxy-ingress.yaml) and integrate +the [proxy service](./08-proxy-service.yaml) in your existing ingress or load balancer. + +## SSL Termination + +This setup does not include SSL termination. +The ingress simply listens on port 80 and serves HTTP requests. + +You will have to set up HTTPS yourself. +You can find more information on this in the [kubernetes docs](https://kubernetes.github.io/ingress-nginx/user-guide/tls/). + +## Scaling + +You can adjust the `replica` specs in the [web](./05-web-deployment.yaml) and [worker](./05-worker-deployment.yaml) deployments +to scale up the respective processes. + +## TROUBLESHOOTING + +### The **db** deployment fails due to the data directory not being empty + +This can happen if your cluster creates the `opdata` PVC (persistent volume claims) with an ext4 file system +which will automatically have a `lost+found` folder. + +To fix the issue you can add the following to the [db-deployment](./02-db-deployment.yaml)'s env next to +`POSTGRES_USER` and `POSTGRES_PASSWORD`: + +``` +- name: PGDATA + value: /var/lib/postgresql/data/pgdata +``` + +This makes the postgres container use a subfolder of the mount path (`/var/lib/postgresql/data`) as the data directory.