diff --git a/stable/airflow/Chart.yaml b/stable/airflow/Chart.yaml index 419c988bd1..32f25efa62 100644 --- a/stable/airflow/Chart.yaml +++ b/stable/airflow/Chart.yaml @@ -1,6 +1,6 @@ description: Airflow is a platform to programmatically author, schedule and monitor workflows name: airflow -version: 0.13.0 +version: 0.14.0 appVersion: 1.10.0 icon: https://airflow.apache.org/_images/pin_large.png home: https://airflow.apache.org/ diff --git a/stable/airflow/README.md b/stable/airflow/README.md index 8baaa50d93..ea722aa76b 100644 --- a/stable/airflow/README.md +++ b/stable/airflow/README.md @@ -220,6 +220,18 @@ Example of procedure: are well installed (for example consuming a `requirements.txt` in your `Dockerfile`) - Update the value of `airflow.image` in your `values.yaml` and deploy on your Kubernetes cluster +## Logs + +You can store Airflow logs on an external volume and mount this volume inside Airflow pods. + +This is useful when running the Kubernetes executor to centralize logs across the +Airflow UI, scheduler, and kubernetes worker pods, which allows for viewing worker log output +in the airflow UI. + +This is controlled by the `logsPersistence.enabled` setting. + +Refer to the `Mount a Shared Persistent Volume` section above for details on using persistent volumes. + ## Helm chart Configuration The following table lists the configurable parameters of the Airflow chart and their default values. @@ -263,6 +275,11 @@ The following table lists the configurable parameters of the Airflow chart and t | `persistence.storageClass` | Persistent Volume Storage Class | (undefined) | | `persistence.accessMode` | PVC access mode | `ReadWriteOnce` | | `persistence.size` | Persistant storage size request | `1Gi` | +| `logsPersistence.enabled` | enable persistent storage for logs | `false` | +| `logsPersistence.existingClaim` | if using an existing claim, specify the name here | `nil` | +| `logsPersistence.storageClass` | Persistent Volume Storage Class | (undefined) | +| `logsPersistence.accessMode` | PVC access mode | `ReadWriteOnce` | +| `logsPersistence.size` | Persistant storage size request | `1Gi` | | `dags.doNotPickle` | should the scheduler disable DAG pickling | `false` | | `dags.path` | mount path for persistent volume | `/usr/local/airflow/dags` | | `dags.initContainer.enabled` | Fetch the source code when the pods starts | `false` | @@ -271,6 +288,7 @@ The following table lists the configurable parameters of the Airflow chart and t | `dags.initContainer.installRequirements` | auto install requirements.txt deps | `true` | | `dags.git.url` | url to clone the git repository | nil | | `dags.git.ref` | branch name, tag or sha1 to reset to | `master` | +| `logs.path` | mount path for logs persistent volume | `/usr/local/airflow/logs` | | `rbac.create` | create RBAC resources | `true` | | `serviceAccount.create` | create a service account | `true` | | `serviceAccount.name` | the service account name | `` | @@ -281,7 +299,7 @@ The following table lists the configurable parameters of the Airflow chart and t | `postgresql.postgresPassword` | PostgreSQL Password | `airflow` | | `postgresql.postgresDatabase` | PostgreSQL Database name | `airflow` | | `postgresql.persistence.enabled` | Enable Postgres PVC | `true` | -| `postgresql.persistance.storageClass | Persistant class | (undefined) | +| `postgresql.persistance.storageClass` | Persistant class | (undefined) | | `postgresql.persistance.accessMode` | Access mode | `ReadWriteOnce` | | `redis.enabled` | Create a Redis cluster | `true` | | `redis.password` | Redis password | `airflow` | diff --git a/stable/airflow/templates/deployments-scheduler.yaml b/stable/airflow/templates/deployments-scheduler.yaml index 49cef4e111..0db70a0aa2 100755 --- a/stable/airflow/templates/deployments-scheduler.yaml +++ b/stable/airflow/templates/deployments-scheduler.yaml @@ -92,6 +92,10 @@ spec: mountPath: /usr/local/connections {{- end}} {{- end }} + {{- if .Values.logsPersistence.enabled }} + - name: logs-data + mountPath: {{ .Values.logs.path }} + {{- end }} args: - "bash" - "-c" @@ -130,6 +134,11 @@ spec: {{- else }} emptyDir: {} {{- end }} + {{- if .Values.logsPersistence.enabled }} + - name: logs-data + persistentVolumeClaim: + claimName: {{ .Values.logsPersistence.existingClaim | default (printf "%s-logs" (include "airflow.fullname" . | trunc 58 )) }} + {{- end }} {{- if .Values.dags.initContainer.enabled }} - name: scripts configMap: diff --git a/stable/airflow/templates/deployments-web.yaml b/stable/airflow/templates/deployments-web.yaml index 3ded27e42c..fb34e13d81 100644 --- a/stable/airflow/templates/deployments-web.yaml +++ b/stable/airflow/templates/deployments-web.yaml @@ -91,6 +91,10 @@ spec: - name: scripts mountPath: /usr/local/scripts {{- end }} + {{- if .Values.logsPersistence.enabled }} + - name: logs-data + mountPath: {{ .Values.logs.path }} + {{- end }} args: - "bash" - "-c" @@ -131,6 +135,11 @@ spec: {{- else }} emptyDir: {} {{- end }} + {{- if .Values.logsPersistence.enabled }} + - name: logs-data + persistentVolumeClaim: + claimName: {{ .Values.logsPersistence.existingClaim | default (printf "%s-logs" (include "airflow.fullname" . | trunc 58 )) }} + {{- end }} {{- if .Values.dags.initContainer.enabled }} - name: scripts configMap: diff --git a/stable/airflow/templates/pvc-logs.yaml b/stable/airflow/templates/pvc-logs.yaml new file mode 100644 index 0000000000..24248d9cbb --- /dev/null +++ b/stable/airflow/templates/pvc-logs.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.logsPersistence.enabled (not .Values.logsPersistence.existingClaim) }} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ printf "%s-logs" (include "airflow.fullname" . | trunc 58)}} + labels: + app: {{ template "airflow.fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + accessModes: + - {{ .Values.logsPersistence.accessMode | quote }} + resources: + requests: + storage: {{ .Values.logsPersistence.size | quote }} +{{- if .Values.logsPersistence.storageClass }} +{{- if (eq "-" .Values.logsPersistence.storageClass) }} + storageClassName: "" +{{- else }} + storageClassName: "{{ .Values.logsPersistence.storageClass }}" +{{- end }} +{{- end }} +{{- end }} diff --git a/stable/airflow/values.yaml b/stable/airflow/values.yaml index b99094be16..3fdf978a6a 100644 --- a/stable/airflow/values.yaml +++ b/stable/airflow/values.yaml @@ -254,6 +254,30 @@ persistence: ## Persistant storage size request size: 1Gi +## +## Storage configuration for logs +logsPersistence: + ## + ## enable persistance storage + enabled: false + ## + ## Existing claim to use + # existingClaim: nil + ## + ## Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + ## A configuration for shared log storage requires a `storageClass` that + ## supports the `ReadWriteMany` accessMode, such as NFS or AWS EFS. + # storageClass: default + accessMode: ReadWriteOnce + ## + ## Persistant storage size request + size: 1Gi ## ## Configure DAGs deployment and update @@ -289,6 +313,11 @@ dags: ## install requirements.txt dependencies automatically installRequirements: true +## +## Configure logs +logs: + path: /usr/local/airflow/logs + ## ## Enable RBAC rbac: