mirror of
https://github.com/helm/charts.git
synced 2026-08-23 22:37:45 +00:00
569 lines
18 KiB
YAML
569 lines
18 KiB
YAML
# Duplicate this file and put your customization here
|
||
|
||
##
|
||
## common settings and setting for the webserver
|
||
airflow:
|
||
extraConfigmapMounts: []
|
||
# - name: extra-metadata
|
||
# mountPath: /opt/metadata
|
||
# configMap: airflow-metadata
|
||
# readOnly: true
|
||
#
|
||
# Example of configmap mount with subPath
|
||
# - name: extra-metadata
|
||
# mountPath: /opt/metadata/file.yaml
|
||
# configMap: airflow-metadata
|
||
# readOnly: true
|
||
# subPath: file.yaml
|
||
|
||
|
||
##
|
||
## Extra environment variables to mount in the web, scheduler, and worker pods:
|
||
extraEnv:
|
||
# - name: AIRFLOW__CORE__FERNET_KEY
|
||
# valueFrom:
|
||
# secretKeyRef:
|
||
# name: airflow
|
||
# key: fernet_key
|
||
# - name: AIRFLOW__LDAP__BIND_PASSWORD
|
||
# valueFrom:
|
||
# secretKeyRef:
|
||
# name: ldap
|
||
# key: password
|
||
|
||
|
||
##
|
||
## You will need to define your fernet key:
|
||
## Generate fernetKey with:
|
||
## python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)"
|
||
## fernetKey: ABCDABCDABCDABCDABCDABCDABCDABCDABCDABCD
|
||
fernetKey: ""
|
||
service:
|
||
annotations: {}
|
||
type: ClusterIP
|
||
externalPort: 8080
|
||
##
|
||
## The executor to use.
|
||
##
|
||
executor: Celery
|
||
##
|
||
## set the max number of retries during container initialization
|
||
initRetryLoop:
|
||
##
|
||
## base image for webserver/scheduler/workers
|
||
## Note: If you want to use airflow HEAD (2.0dev), use the following image:
|
||
# image
|
||
# repository: stibbons31/docker-airflow-dev
|
||
# tag: 2.0dev
|
||
## Airflow 2.0 allows changing the value ingress.web.path and ingress.flower.path (see bellow).
|
||
## In version < 2.0, changing these paths won't have any effect.
|
||
image:
|
||
##
|
||
## docker-airflow image
|
||
repository: puckel/docker-airflow
|
||
##
|
||
## image tag
|
||
tag: 1.10.3
|
||
##
|
||
## Image pull policy
|
||
## values: Always or IfNotPresent
|
||
pullPolicy: IfNotPresent
|
||
##
|
||
## image pull secret for private images
|
||
pullSecret:
|
||
##
|
||
## Set schedulerNumRuns to control how the scheduler behaves:
|
||
## -1 will let him looping indefinitively but it will never update the DAG
|
||
## 1 will have the scheduler quit after each refresh, but kubernetes will restart it.
|
||
##
|
||
## A long running scheduler process, at least with the CeleryExecutor, ends up not scheduling
|
||
## some tasks. We still don’t know the exact cause, unfortunately. Airflow has a built-in
|
||
## workaround in the form of the `num_runs` flag.
|
||
## Airflow runs with num_runs set to 5.
|
||
##
|
||
## If set to a value != -1, you will see your scheduler regularly restart. This is its normal
|
||
## behavior under these conditions.
|
||
schedulerNumRuns: "-1"
|
||
##
|
||
## Set schedulerDoPickle to toggle whether to have the scheduler
|
||
## attempt to pickle the DAG object to send over to the workers,
|
||
## instead of letting workers run their version of the code.
|
||
## See the Airflow documentation for the --do_pickle argument: https://airflow.apache.org/cli.html#scheduler
|
||
schedulerDoPickle: true
|
||
##
|
||
## Number of replicas for web server.
|
||
webReplicas: 1
|
||
##
|
||
## Custom airflow configuration environment variables
|
||
## Use this to override any airflow setting settings defining environment variables in the
|
||
## following form: AIRFLOW__<section>__<key>.
|
||
## See the Airflow documentation: https://airflow.readthedocs.io/en/stable/howto/set-config.html?highlight=setting-configuration
|
||
## Example:
|
||
## config:
|
||
## AIRFLOW__CORE__EXPOSE_CONFIG: "True"
|
||
## HTTP_PROXY: "http://proxy.mycompany.com:123"
|
||
config: {}
|
||
##
|
||
## Configure pod disruption budget for the scheduler
|
||
podDisruptionBudget:
|
||
maxUnavailable: 1
|
||
## Add custom connections
|
||
## Use this to add Airflow connections for operators you use
|
||
## For each connection - the id and type have to be defined.
|
||
## All the other parameters are optional
|
||
## Connections will be created with a script that is stored
|
||
## in a K8s secret and mounted into the scheduler container
|
||
## Example:
|
||
## connections:
|
||
## - id: my_aws
|
||
## type: aws
|
||
## extra: '{"aws_access_key_id": "**********", "aws_secret_access_key": "***", "region_name":"eu-central-1"}'
|
||
connections: {}
|
||
|
||
## Add airflow variables
|
||
## This should be a json string with your variables in it
|
||
## Examples:
|
||
## variables: '{ "environment": "dev" }'
|
||
variables: {}
|
||
|
||
## Add airflow ppols
|
||
## This should be a json string with your pools in it
|
||
## Examples:
|
||
## pools: '{ "example": { "description": "This is an example of a pool", "slots": 2 } }'
|
||
pools: {}
|
||
|
||
##
|
||
## Annotations for the Scheduler, Worker and Web pods
|
||
podAnnotations: {}
|
||
## Example:
|
||
## iam.amazonaws.com/role: airflow-Role
|
||
extraContainers: []
|
||
## Additional containers to run alongside the Scheduler, Worker and Web pods
|
||
## This could, for example, be used to run a sidecar that syncs DAGs from object storage.
|
||
# - name: s3-sync
|
||
# image: my-user/s3sync:latest
|
||
# volumeMounts:
|
||
# - name: synchronised-dags
|
||
# mountPath: /dags
|
||
extraVolumeMounts: []
|
||
## Additional volumeMounts to the main containers in the Scheduler, Worker and Web pods.
|
||
# - name: synchronised-dags
|
||
# mountPath: /usr/local/airflow/dags
|
||
extraVolumes: []
|
||
## Additional volumes for the Scheduler, Worker and Web pods.
|
||
# - name: synchronised-dags
|
||
# emptyDir: {}
|
||
|
||
##
|
||
## Run initdb when the scheduler starts.
|
||
initdb: true
|
||
|
||
|
||
scheduler:
|
||
resources: {}
|
||
# limits:
|
||
# cpu: "1000m"
|
||
# memory: "1Gi"
|
||
# requests:
|
||
# cpu: "500m"
|
||
# memory: "512Mi"
|
||
|
||
|
||
flower:
|
||
resources: {}
|
||
# limits:
|
||
# cpu: "100m"
|
||
# memory: "128Mi"
|
||
# requests:
|
||
# cpu: "100m"
|
||
# memory: "128Mi"
|
||
service:
|
||
annotations: {}
|
||
type: ClusterIP
|
||
externalPort: 5555
|
||
|
||
web:
|
||
resources: {}
|
||
# limits:
|
||
# cpu: "300m"
|
||
# memory: "1Gi"
|
||
# requests:
|
||
# cpu: "100m"
|
||
# memory: "512Mi"
|
||
initialStartupDelay: "60"
|
||
initialDelaySeconds: "360"
|
||
##
|
||
## Workers configuration
|
||
workers:
|
||
enabled: true
|
||
##
|
||
## Number of workers pod to launch
|
||
replicas: 1
|
||
##
|
||
## Custom resource configuration
|
||
resources: {}
|
||
# limits:
|
||
# cpu: "1"
|
||
# memory: "2G"
|
||
# requests:
|
||
# cpu: "0.5"
|
||
# memory: "512Mi"
|
||
##
|
||
## Annotations for the Worker pods
|
||
podAnnotations: {}
|
||
## Example:
|
||
## iam.amazonaws.com/role: airflow-Role
|
||
##
|
||
## Celery worker configuration
|
||
celery:
|
||
##
|
||
## number of parallel celery tasks per worker
|
||
instances: 1
|
||
##
|
||
## Directory in which to mount secrets on worker nodes.
|
||
secretsDir: /var/airflow/secrets
|
||
##
|
||
## Secrets which will be mounted as a file at `secretsDir/<secret name>`.
|
||
secrets: []
|
||
|
||
## Support Node, affinity and tolerations for airflow and workers labels for pod assignment
|
||
## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
|
||
## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
|
||
## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#taints-and-tolerations-beta-feature
|
||
nodeSelector: {}
|
||
affinity: {}
|
||
tolerations: []
|
||
##
|
||
## Ingress configuration
|
||
ingress:
|
||
##
|
||
## enable ingress
|
||
## Note: If you want to change url prefix for web ui or flower even if you do not use ingress,
|
||
## you can still change ingress.web.path and ingress.flower.path
|
||
enabled: false
|
||
##
|
||
## Configure the webserver endpoint
|
||
web:
|
||
## NOTE: This requires an airflow version > 1.9.x
|
||
## For the moment (March 2018) this is **not** available on official package, you will have
|
||
## to use an image where airflow has been updated to its current HEAD.
|
||
## You can use the following one:
|
||
## stibbons31/docker-airflow-dev:2.0dev
|
||
##
|
||
## if path is '/airflow':
|
||
## - UI will be accessible at 'http://mycompany.com/airflow/admin'
|
||
## - Healthcheck is at 'http://mycompany.com/airflow/health'
|
||
## - api is at 'http://mycompany.com/airflow/api'
|
||
## NOTE: do NOT keep trailing slash. For root configuration, set and empty string
|
||
path: ""
|
||
##
|
||
## hostname for the webserver
|
||
host: ""
|
||
##
|
||
## Annotations for the webserver
|
||
## Airflow webserver handles relative path completely, just let your load balancer give the HTTP
|
||
## header like the requested URL (no special configuration neeed)
|
||
annotations:
|
||
##
|
||
## Example for Traefik:
|
||
# traefik.frontend.rule.type: PathPrefix
|
||
# kubernetes.io/ingress.class: traefik
|
||
##
|
||
## Configure the web liveness path.
|
||
## Defaults to the templated value `{{ ingress.web.path }}/health`
|
||
livenessPath:
|
||
tls:
|
||
## Set to "true" to enable TLS termination at the ingress
|
||
enabled: false
|
||
## If enabled, set "secretName" to the secret containing the TLS private key and certificate
|
||
## Example:
|
||
## secretName: example-com-crt
|
||
##
|
||
## Configure the flower endpoind
|
||
flower:
|
||
##
|
||
## If flower is '/airflow/flower':
|
||
## - Flower UI is at 'http://mycompany.com/airflow/flower'
|
||
## NOTE: you need to have a reverse proxy/load balancer able to do URL rewrite in order to have
|
||
## flower mounted on other path than root. Flower only does half the job in url prefixing: it
|
||
## only generates the right URL/relative paths in the **returned HTML files**, but expects the
|
||
## request to have been be at the root.
|
||
## That's why we need a reverse proxy/load balancer that is able to strip the path
|
||
## NOTE: do NOT keep trailing slash. For root configuration, set and empty string
|
||
path: ""
|
||
##
|
||
## Configure the liveness path. Keep to "/" for Flower >= jan 2018.
|
||
## For previous version, enter the same path than in the 'path' key
|
||
## NOTE: keep the trailing slash.
|
||
livenessPath: /
|
||
##
|
||
## hostname for flower
|
||
host: ""
|
||
##
|
||
## Annotation for the Flower endpoint
|
||
##
|
||
## ==== SKIP THE FOLLOWING BLOCK IF YOU HAVE FLOWER > JANUARY 2018 =============================
|
||
## Please note their is a small difference between the way Airflow Web server and Flower handles
|
||
## URL prefixes in HTTP requests:
|
||
## Flower wants HTTP header to behave like there was no URL prefix, and but still generates
|
||
## the right URL in html pages thanks to its `--url-prefix` parameter
|
||
##
|
||
## Extracted from the Flower documentation:
|
||
## (https://github.com/mher/flower/blob/master/docs/config.rst#url_prefix)
|
||
##
|
||
## To access Flower on http://example.com/flower run it with:
|
||
## flower --url-prefix=/flower
|
||
##
|
||
## Use the following nginx configuration:
|
||
## server {
|
||
## listen 80;
|
||
## server_name example.com;
|
||
##
|
||
## location /flower/ {
|
||
## rewrite ^/flower/(.*)$ /$1 break;
|
||
## proxy_pass http://example.com:5555;
|
||
## proxy_set_header Host $host;
|
||
## }
|
||
## }
|
||
## ==== IF YOU HAVE FLOWER > JANUARY 2018, NO MORE NEED TO STRIP THE PREFIX ====================
|
||
annotations:
|
||
##
|
||
## NOTE: it is important here to have your reverse proxy strip the path/rewrite the URL
|
||
## Example for Traefik:
|
||
# traefik.frontend.rule.type: PathPrefix ## Flower >= Jan 2018
|
||
# traefik.frontend.rule.type: PathPrefixStrip ## Flower < Jan 2018
|
||
# kubernetes.io/ingress.class: traefik
|
||
tls:
|
||
## Set to "true" to enable TLS termination at the ingress
|
||
enabled: false
|
||
## If enabled, set "secretName" to the secret containing the TLS private key and certificate
|
||
## Example:
|
||
## secretName: example-com-crt
|
||
|
||
|
||
##
|
||
## Storage configuration for DAGs
|
||
persistence:
|
||
##
|
||
## enable persistance storage
|
||
enabled: false
|
||
##
|
||
## Existing claim to use
|
||
# existingClaim: nil
|
||
## Existing claim's subPath to use, e.g. "dags" (optional)
|
||
# subPath: ""
|
||
##
|
||
## Persistent Volume Storage Class
|
||
## If defined, storageClassName: <storageClass>
|
||
## If set to "-", storageClassName: "", which disables dynamic provisioning
|
||
## If undefined (the default) or set to null, no storageClassName spec is
|
||
## set, choosing the default provisioner. (gp2 on AWS, standard on
|
||
## GKE, AWS & OpenStack)
|
||
# storageClass: default
|
||
accessMode: ReadWriteOnce
|
||
##
|
||
## Persistant storage size request
|
||
size: 1Gi
|
||
|
||
##
|
||
## Storage configuration for logs
|
||
logsPersistence:
|
||
##
|
||
## enable persistance storage
|
||
enabled: false
|
||
##
|
||
## Existing claim to use
|
||
# existingClaim: nil
|
||
## Existing claim's subPath to use, e.g. "logs" (optional)
|
||
# subPath: ""
|
||
##
|
||
## Persistent Volume Storage Class
|
||
## If defined, storageClassName: <storageClass>
|
||
## If set to "-", storageClassName: "", which disables dynamic provisioning
|
||
## If undefined (the default) or set to null, no storageClassName spec is
|
||
## set, choosing the default provisioner. (gp2 on AWS, standard on
|
||
## GKE, AWS & OpenStack)
|
||
##
|
||
## A configuration for shared log storage requires a `storageClass` that
|
||
## supports the `ReadWriteMany` accessMode, such as NFS or AWS EFS.
|
||
# storageClass: default
|
||
accessMode: ReadWriteOnce
|
||
##
|
||
## Persistant storage size request
|
||
size: 1Gi
|
||
|
||
##
|
||
## Configure DAGs deployment and update
|
||
dags:
|
||
##
|
||
## mount path for persistent volume.
|
||
## Note that this location is referred to in airflow.cfg, so if you change it, you must update airflow.cfg accordingly.
|
||
path: /usr/local/airflow/dags
|
||
##
|
||
## Set to True to prevent pickling DAGs from scheduler to workers
|
||
doNotPickle: false
|
||
##
|
||
## Configure Git repository to fetch DAGs
|
||
git:
|
||
##
|
||
## url to clone the git repository
|
||
url:
|
||
##
|
||
## branch name, tag or sha1 to reset to
|
||
ref: master
|
||
## pre-created secret with key, key.pub and known_hosts file for private repos
|
||
secret: {}
|
||
initContainer:
|
||
## Fetch the source code when the pods starts
|
||
enabled: false
|
||
## Image for the init container (any image with git will do)
|
||
image:
|
||
## docker-airflow image
|
||
repository: alpine/git
|
||
## image tag
|
||
tag: 1.0.7
|
||
## Image pull policy
|
||
## values: Always or IfNotPresent
|
||
pullPolicy: IfNotPresent
|
||
## install requirements.txt dependencies automatically
|
||
installRequirements: true
|
||
|
||
##
|
||
## Configure logs
|
||
logs:
|
||
path: /usr/local/airflow/logs
|
||
|
||
##
|
||
## Enable RBAC
|
||
rbac:
|
||
##
|
||
## Specifies whether RBAC resources should be created
|
||
create: true
|
||
|
||
##
|
||
## Create or use ServiceAccount
|
||
serviceAccount:
|
||
##
|
||
## Specifies whether a ServiceAccount should be created
|
||
create: true
|
||
## The name of the ServiceAccount to use.
|
||
## If not set and create is true, a name is generated using the fullname template
|
||
name:
|
||
|
||
|
||
##
|
||
## Configuration values for the postgresql dependency.
|
||
## ref: https://github.com/kubernetes/charts/blob/master/stable/postgresql/README.md
|
||
postgresql:
|
||
##
|
||
## Use the PostgreSQL chart dependency.
|
||
## Set to false if bringing your own PostgreSQL.
|
||
enabled: true
|
||
|
||
##
|
||
## The name of an existing secret that contains the postgres password.
|
||
## Note that the secret must have a key called postgres-password.
|
||
existingSecret:
|
||
|
||
##
|
||
## If you are bringing your own PostgreSQL, you should set postgresHost and
|
||
## also probably service.port, postgresUser, postgresPassword, and postgresDatabase
|
||
## postgresHost:
|
||
##
|
||
## PostgreSQL port
|
||
service:
|
||
port: 5432
|
||
## PostgreSQL User to create.
|
||
postgresUser: postgres
|
||
##
|
||
## PostgreSQL Password for the new user.
|
||
## If not set, a random 10 characters password will be used.
|
||
postgresPassword: airflow
|
||
##
|
||
## PostgreSQL Database to create.
|
||
postgresDatabase: airflow
|
||
##
|
||
## Persistent Volume Storage configuration.
|
||
## ref: https://kubernetes.io/docs/user-guide/persistent-volumes
|
||
persistence:
|
||
##
|
||
## Enable PostgreSQL persistence using Persistent Volume Claims.
|
||
enabled: true
|
||
##
|
||
## Persistant class
|
||
# storageClass: classname
|
||
##
|
||
## Access mode:
|
||
accessMode: ReadWriteOnce
|
||
|
||
|
||
## Configuration values for the Redis dependency.
|
||
## ref: https://github.com/kubernetes/charts/blob/master/stable/redis/README.md
|
||
redis:
|
||
##
|
||
## Use the redis chart dependency.
|
||
## Set to false if bringing your own redis.
|
||
enabled: true
|
||
|
||
##
|
||
## The name of an existing secret that contains the redis password.
|
||
## Note that the secret must have a key called redis-password.
|
||
existingSecret:
|
||
|
||
##
|
||
## If you are bringing your own redis, you can set the host in redisHost.
|
||
## redisHost:
|
||
##
|
||
## Redis password
|
||
##
|
||
password: airflow
|
||
##
|
||
## Master configuration
|
||
master:
|
||
##
|
||
## Image configuration
|
||
# image:
|
||
##
|
||
## docker registry secret names (list)
|
||
# pullSecrets: nil
|
||
##
|
||
## Configure persistance
|
||
persistence:
|
||
##
|
||
## Use a PVC to persist data.
|
||
enabled: false
|
||
##
|
||
## Persistant class
|
||
# storageClass: classname
|
||
##
|
||
## Access mode:
|
||
accessMode: ReadWriteOnce
|
||
##
|
||
## Disable cluster management by default.
|
||
cluster:
|
||
enabled: false
|
||
|
||
# Enable this if you're using https://github.com/coreos/prometheus-operator
|
||
# Don't forget you need to install something like https://github.com/epoch8/airflow-exporter in your airflow docker container
|
||
serviceMonitor:
|
||
enabled: false
|
||
interval: "30s"
|
||
path: /admin/metrics
|
||
## [Kube Prometheus Selector Label](https://github.com/helm/charts/tree/master/stable/prometheus-operator#exporters)
|
||
selector:
|
||
prometheus: kube-prometheus
|
||
|
||
# Enable this if you're using https://github.com/coreos/prometheus-operator
|
||
prometheusRule:
|
||
enabled: false
|
||
## Namespace in which the prometheus rule is created
|
||
# namespace: monitoring
|
||
## Define individual alerting rules as required
|
||
## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#rulegroup
|
||
## https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/
|
||
groups: {}
|
||
|
||
## Used to pass Labels that are used by the Prometheus installed in your cluster to select Prometheus Rules to work with
|
||
## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#prometheusspec
|
||
additionalLabels: {}
|