mirror of
https://github.com/helm/charts.git
synced 2026-08-23 22:37:45 +00:00
Support persistent volume log storage (#10904)
Signed-off-by: Kevin Pullin <kevin.pullin@gmail.com>
This commit is contained in:
Kevin Pullin
committed by
Kubernetes Prow Robot
Kubernetes Prow Robot
parent
5607847cba
commit
8f0fe69a98
@@ -1,6 +1,6 @@
|
||||
description: Airflow is a platform to programmatically author, schedule and monitor workflows
|
||||
name: airflow
|
||||
version: 0.13.0
|
||||
version: 0.14.0
|
||||
appVersion: 1.10.0
|
||||
icon: https://airflow.apache.org/_images/pin_large.png
|
||||
home: https://airflow.apache.org/
|
||||
|
||||
@@ -220,6 +220,18 @@ Example of procedure:
|
||||
are well installed (for example consuming a `requirements.txt` in your `Dockerfile`)
|
||||
- Update the value of `airflow.image` in your `values.yaml` and deploy on your Kubernetes cluster
|
||||
|
||||
## Logs
|
||||
|
||||
You can store Airflow logs on an external volume and mount this volume inside Airflow pods.
|
||||
|
||||
This is useful when running the Kubernetes executor to centralize logs across the
|
||||
Airflow UI, scheduler, and kubernetes worker pods, which allows for viewing worker log output
|
||||
in the airflow UI.
|
||||
|
||||
This is controlled by the `logsPersistence.enabled` setting.
|
||||
|
||||
Refer to the `Mount a Shared Persistent Volume` section above for details on using persistent volumes.
|
||||
|
||||
## Helm chart Configuration
|
||||
|
||||
The following table lists the configurable parameters of the Airflow chart and their default values.
|
||||
@@ -263,6 +275,11 @@ The following table lists the configurable parameters of the Airflow chart and t
|
||||
| `persistence.storageClass` | Persistent Volume Storage Class | (undefined) |
|
||||
| `persistence.accessMode` | PVC access mode | `ReadWriteOnce` |
|
||||
| `persistence.size` | Persistant storage size request | `1Gi` |
|
||||
| `logsPersistence.enabled` | enable persistent storage for logs | `false` |
|
||||
| `logsPersistence.existingClaim` | if using an existing claim, specify the name here | `nil` |
|
||||
| `logsPersistence.storageClass` | Persistent Volume Storage Class | (undefined) |
|
||||
| `logsPersistence.accessMode` | PVC access mode | `ReadWriteOnce` |
|
||||
| `logsPersistence.size` | Persistant storage size request | `1Gi` |
|
||||
| `dags.doNotPickle` | should the scheduler disable DAG pickling | `false` |
|
||||
| `dags.path` | mount path for persistent volume | `/usr/local/airflow/dags` |
|
||||
| `dags.initContainer.enabled` | Fetch the source code when the pods starts | `false` |
|
||||
@@ -271,6 +288,7 @@ The following table lists the configurable parameters of the Airflow chart and t
|
||||
| `dags.initContainer.installRequirements` | auto install requirements.txt deps | `true` |
|
||||
| `dags.git.url` | url to clone the git repository | nil |
|
||||
| `dags.git.ref` | branch name, tag or sha1 to reset to | `master` |
|
||||
| `logs.path` | mount path for logs persistent volume | `/usr/local/airflow/logs` |
|
||||
| `rbac.create` | create RBAC resources | `true` |
|
||||
| `serviceAccount.create` | create a service account | `true` |
|
||||
| `serviceAccount.name` | the service account name | `` |
|
||||
@@ -281,7 +299,7 @@ The following table lists the configurable parameters of the Airflow chart and t
|
||||
| `postgresql.postgresPassword` | PostgreSQL Password | `airflow` |
|
||||
| `postgresql.postgresDatabase` | PostgreSQL Database name | `airflow` |
|
||||
| `postgresql.persistence.enabled` | Enable Postgres PVC | `true` |
|
||||
| `postgresql.persistance.storageClass | Persistant class | (undefined) |
|
||||
| `postgresql.persistance.storageClass` | Persistant class | (undefined) |
|
||||
| `postgresql.persistance.accessMode` | Access mode | `ReadWriteOnce` |
|
||||
| `redis.enabled` | Create a Redis cluster | `true` |
|
||||
| `redis.password` | Redis password | `airflow` |
|
||||
|
||||
@@ -92,6 +92,10 @@ spec:
|
||||
mountPath: /usr/local/connections
|
||||
{{- end}}
|
||||
{{- end }}
|
||||
{{- if .Values.logsPersistence.enabled }}
|
||||
- name: logs-data
|
||||
mountPath: {{ .Values.logs.path }}
|
||||
{{- end }}
|
||||
args:
|
||||
- "bash"
|
||||
- "-c"
|
||||
@@ -130,6 +134,11 @@ spec:
|
||||
{{- else }}
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- if .Values.logsPersistence.enabled }}
|
||||
- name: logs-data
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ .Values.logsPersistence.existingClaim | default (printf "%s-logs" (include "airflow.fullname" . | trunc 58 )) }}
|
||||
{{- end }}
|
||||
{{- if .Values.dags.initContainer.enabled }}
|
||||
- name: scripts
|
||||
configMap:
|
||||
|
||||
@@ -91,6 +91,10 @@ spec:
|
||||
- name: scripts
|
||||
mountPath: /usr/local/scripts
|
||||
{{- end }}
|
||||
{{- if .Values.logsPersistence.enabled }}
|
||||
- name: logs-data
|
||||
mountPath: {{ .Values.logs.path }}
|
||||
{{- end }}
|
||||
args:
|
||||
- "bash"
|
||||
- "-c"
|
||||
@@ -131,6 +135,11 @@ spec:
|
||||
{{- else }}
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- if .Values.logsPersistence.enabled }}
|
||||
- name: logs-data
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ .Values.logsPersistence.existingClaim | default (printf "%s-logs" (include "airflow.fullname" . | trunc 58 )) }}
|
||||
{{- end }}
|
||||
{{- if .Values.dags.initContainer.enabled }}
|
||||
- name: scripts
|
||||
configMap:
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
{{- if and .Values.logsPersistence.enabled (not .Values.logsPersistence.existingClaim) }}
|
||||
kind: PersistentVolumeClaim
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ printf "%s-logs" (include "airflow.fullname" . | trunc 58)}}
|
||||
labels:
|
||||
app: {{ template "airflow.fullname" . }}
|
||||
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
|
||||
release: "{{ .Release.Name }}"
|
||||
heritage: "{{ .Release.Service }}"
|
||||
spec:
|
||||
accessModes:
|
||||
- {{ .Values.logsPersistence.accessMode | quote }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.logsPersistence.size | quote }}
|
||||
{{- if .Values.logsPersistence.storageClass }}
|
||||
{{- if (eq "-" .Values.logsPersistence.storageClass) }}
|
||||
storageClassName: ""
|
||||
{{- else }}
|
||||
storageClassName: "{{ .Values.logsPersistence.storageClass }}"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -254,6 +254,30 @@ persistence:
|
||||
## Persistant storage size request
|
||||
size: 1Gi
|
||||
|
||||
##
|
||||
## Storage configuration for logs
|
||||
logsPersistence:
|
||||
##
|
||||
## enable persistance storage
|
||||
enabled: false
|
||||
##
|
||||
## Existing claim to use
|
||||
# existingClaim: nil
|
||||
##
|
||||
## Persistent Volume Storage Class
|
||||
## If defined, storageClassName: <storageClass>
|
||||
## If set to "-", storageClassName: "", which disables dynamic provisioning
|
||||
## If undefined (the default) or set to null, no storageClassName spec is
|
||||
## set, choosing the default provisioner. (gp2 on AWS, standard on
|
||||
## GKE, AWS & OpenStack)
|
||||
##
|
||||
## A configuration for shared log storage requires a `storageClass` that
|
||||
## supports the `ReadWriteMany` accessMode, such as NFS or AWS EFS.
|
||||
# storageClass: default
|
||||
accessMode: ReadWriteOnce
|
||||
##
|
||||
## Persistant storage size request
|
||||
size: 1Gi
|
||||
|
||||
##
|
||||
## Configure DAGs deployment and update
|
||||
@@ -289,6 +313,11 @@ dags:
|
||||
## install requirements.txt dependencies automatically
|
||||
installRequirements: true
|
||||
|
||||
##
|
||||
## Configure logs
|
||||
logs:
|
||||
path: /usr/local/airflow/logs
|
||||
|
||||
##
|
||||
## Enable RBAC
|
||||
rbac:
|
||||
|
||||
Reference in New Issue
Block a user