diff --git a/stable/airflow/Chart.yaml b/stable/airflow/Chart.yaml index 5a6b74e45f..98943040ed 100644 --- a/stable/airflow/Chart.yaml +++ b/stable/airflow/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Airflow is a platform to programmatically author, schedule and monitor workflows name: airflow -version: 2.8.3 +version: 2.8.4 appVersion: 1.10.2 icon: https://airflow.apache.org/_images/pin_large.png home: https://airflow.apache.org/ diff --git a/stable/airflow/README.md b/stable/airflow/README.md index 0c6f7f239b..33d1b42847 100644 --- a/stable/airflow/README.md +++ b/stable/airflow/README.md @@ -271,13 +271,14 @@ Example of procedure: ## Logs -You can store Airflow logs on an external volume and mount this volume inside Airflow pods. +By default (`logsPersistence.enabled: false`) logs from the web server, scheduler, and Celery workers are written to within the Docker container's filesystem. +Therefore, any restart of the pod will wipe the logs. +For production purposes, you will likely want to persist the logs externally (e.g. S3), which you have to set up yourself through configuration. +You can also persist logs using a `PersistentVolume` using `logsPersistence.enabled: true`. +To use an existing volume, pass the name to `logsPersistence.existingClaim`. +Otherwise, a new volume will be created. -This is useful when running the Kubernetes executor to centralize logs across the -Airflow UI, scheduler, and kubernetes worker pods, which allows for viewing worker log output -in the airflow UI. - -This is controlled by the `logsPersistence.enabled` setting. +Note that it is also possible to persist logs by mounting a `PersistentVolume` to the log directory (`/usr/local/airflow/logs` by default) using `airflow.extraVolumes` and `airflow.extraVolumeMounts`. Refer to the `Mount a Shared Persistent Volume` section above for details on using persistent volumes. diff --git a/stable/airflow/templates/statefulsets-workers.yaml b/stable/airflow/templates/statefulsets-workers.yaml index 26167bea32..73bc9284b1 100644 --- a/stable/airflow/templates/statefulsets-workers.yaml +++ b/stable/airflow/templates/statefulsets-workers.yaml @@ -111,6 +111,10 @@ spec: - name: dags-data mountPath: {{ .Values.dags.path }} {{- end }} + {{- if .Values.logsPersistence.enabled }} + - name: logs-data + mountPath: {{ .Values.logs.path }} + {{- end }} {{- range .Values.airflow.extraConfigmapMounts }} - name: {{ .name }} mountPath: {{ .mountPath }} @@ -170,6 +174,11 @@ spec: {{- else }} emptyDir: {} {{- end }} + {{- if .Values.logsPersistence.enabled }} + - name: logs-data + persistentVolumeClaim: + claimName: {{ .Values.logsPersistence.existingClaim | default (printf "%s-logs" (include "airflow.fullname" . | trunc 58 )) }} + {{- end }} {{- if .Values.dags.initContainer.enabled }} - name: git-clone configMap: