From 6c52106bc690a3ffc0f439574646379dbb0ca1bb Mon Sep 17 00:00:00 2001 From: Edward Vella Date: Fri, 29 May 2020 22:47:33 +0100 Subject: [PATCH] Adding a secretMap instead of adding multiple secrets (#22587) Signed-off-by: Edward Vella --- stable/airflow/Chart.yaml | 2 +- stable/airflow/README.md | 2 + stable/airflow/templates/deployments-web.yaml | 12 + .../templates/statefulsets-workers.yaml | 12 + stable/airflow/values-test.yaml | 1228 +++++++++++++++++ stable/airflow/values.yaml | 18 + 6 files changed, 1273 insertions(+), 1 deletion(-) create mode 100644 stable/airflow/values-test.yaml diff --git a/stable/airflow/Chart.yaml b/stable/airflow/Chart.yaml index 89f1dfc964..db82cf65f6 100644 --- a/stable/airflow/Chart.yaml +++ b/stable/airflow/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Airflow is a platform to programmatically author, schedule and monitor workflows name: airflow -version: 7.1.1 +version: 7.1.2 appVersion: 1.10.10 icon: https://airflow.apache.org/_images/pin_large.png home: https://airflow.apache.org/ diff --git a/stable/airflow/README.md b/stable/airflow/README.md index 0adfbb8447..f14030ebeb 100644 --- a/stable/airflow/README.md +++ b/stable/airflow/README.md @@ -572,6 +572,7 @@ __Airflow WebUI Values:__ | `web.livenessProbe.*` | configs for the web Service liveness probe | `` | | `web.secretsDir` | the directory in which to mount secrets on web containers | `/var/airflow/secrets` | | `web.secrets` | secret names which will be mounted as a file at `{web.secretsDir}/` | `[]` | +| `web.secretsMap` | you can use secretsMap to specify a map and all the secrets will be stored within it secrets will be mounted as files at `{web.secretsDir}/`. If you use web.secretsMap, then it overrides `web.secrets`.| `""` | __Airflow Worker Values:__ @@ -593,6 +594,7 @@ __Airflow Worker Values:__ | `workers.terminationPeriod` | how many seconds to wait for tasks on a worker to finish before SIGKILL | `60` | | `workers.secretsDir` | directory in which to mount secrets on worker containers | `/var/airflow/secrets` | | `workers.secrets` | secret names which will be mounted as a file at `{workers.secretsDir}/` | `[]` | +| `workers.secretsMap` | you can use secretsMap to specify a map and all the secrets will be stored within it secrets will be mounted as files at `{workers.secretsDir}/`. If you use workers.secretsMap, then it overrides `workers.secrets`.| `""` | __Airflow Flower Values:__ diff --git a/stable/airflow/templates/deployments-web.yaml b/stable/airflow/templates/deployments-web.yaml index efcda37f45..05e8afa616 100644 --- a/stable/airflow/templates/deployments-web.yaml +++ b/stable/airflow/templates/deployments-web.yaml @@ -154,11 +154,17 @@ spec: volumeMounts: - name: scripts mountPath: /home/airflow/scripts + {{- if .Values.web.secretsMap }} + - name: {{ .Values.web.secretsMap }}-volume + readOnly: true + mountPath: {{ $.Values.web.secretsDir }} + {{- else }} {{- range .Values.web.secrets }} - name: {{ . }}-volume readOnly: true mountPath: {{ $.Values.web.secretsDir }}/{{ . }} {{- end }} + {{- end }} {{- if .Values.dags.persistence.enabled }} - name: dags-data mountPath: {{ .Values.dags.path }} @@ -255,11 +261,17 @@ spec: configMap: name: {{ include "airflow.fullname" . }}-scripts defaultMode: 0755 + {{- if .Values.web.secretsMap }} + - name: {{ .Values.web.secretsMap }}-volume + secret: + secretName: {{ .Values.web.secretsMap }} + {{- else }} {{- range .Values.web.secrets }} - name: {{ . }}-volume secret: secretName: {{ . }} {{- end }} + {{- end }} {{- if .Values.dags.persistence.enabled }} - name: dags-data persistentVolumeClaim: diff --git a/stable/airflow/templates/statefulsets-workers.yaml b/stable/airflow/templates/statefulsets-workers.yaml index 48ecddd4d3..8071512e76 100644 --- a/stable/airflow/templates/statefulsets-workers.yaml +++ b/stable/airflow/templates/statefulsets-workers.yaml @@ -159,11 +159,17 @@ spec: - name: scripts mountPath: /home/airflow/scripts {{- $secretsDir := .Values.workers.secretsDir }} + {{- if .Values.workers.secretsMap }} + - name: {{ .Values.workers.secretsMap }}-volume + readOnly: true + mountPath: {{ $secretsDir }} + {{- else }} {{- range .Values.workers.secrets }} - name: {{ . }}-volume readOnly: true mountPath: {{ $secretsDir }}/{{ . }} {{- end }} + {{- end }} {{- if .Values.dags.persistence.enabled }} - name: dags-data mountPath: {{ .Values.dags.path }} @@ -234,11 +240,17 @@ spec: configMap: name: {{ include "airflow.fullname" . }}-scripts defaultMode: 0755 + {{- if .Values.workers.secretsMap }} + - name: {{ .Values.workers.secretsMap }}-volume + secret: + secretName: {{ .Values.workers.secretsMap }} + {{- else }} {{- range .Values.workers.secrets }} - name: {{ . }}-volume secret: secretName: {{ . }} {{- end }} + {{- end }} {{- if .Values.dags.persistence.enabled }} - name: dags-data persistentVolumeClaim: diff --git a/stable/airflow/values-test.yaml b/stable/airflow/values-test.yaml new file mode 100644 index 0000000000..b2653a9758 --- /dev/null +++ b/stable/airflow/values-test.yaml @@ -0,0 +1,1228 @@ +################################### +# Airflow - Common Configs +################################### +airflow: + ## configs for the docker image of the web/scheduler/worker + ## + image: + repository: apache/airflow + tag: 1.10.10-python3.6 + ## values: Always or IfNotPresent + pullPolicy: IfNotPresent + pullSecret: "" + + ## the airflow executor type to use + ## + ## NOTE: + ## - this should be `CeleryExecutor` or `KubernetesExecutor` + ## - if set to `KubernetesExecutor`: + ## - ensure that `workers.enabled` is `false` + ## - ensure that `flower.enabled` is `false` + ## - ensure that `redis.enabled` is `false` + ## - ensure that K8S configs are set in `airflow.config` + ## - we set these configs automatically: + ## - `AIRFLOW__KUBERNETES__NAMESPACE` + ## - `AIRFLOW__KUBERNETES__WORKER_SERVICE_ACCOUNT_NAME` + ## - `AIRFLOW__KUBERNETES__ENV_FROM_CONFIGMAP_REF` + ## + executor: CeleryExecutor + + ## the fernet key used to encrypt the connections/variables in the database + ## + ## WARNING: + ## - you MUST customise this value, otherwise the encryption will be somewhat pointless + ## + ## NOTE: + ## - to prevent this value being stored in your values.yaml (and airflow-env ConfigMap), + ## consider using `airflow.extraEnv` to define it from a pre-created secret + ## + ## GENERATE: + ## python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)" + ## + fernetKey: "7T512UXSSmBOkpWimFHIVb8jK6lfmSAvx4mO6Arehnc=" + + ## environment variables for the web/scheduler/worker Pods (for airflow configs) + ## + ## WARNING: + ## - don't include sensitive variables in here, instead make use of `airflow.extraEnv` with Secrets + ## - don't specify `AIRFLOW__CORE__SQL_ALCHEMY_CONN`, `AIRFLOW__CELERY__RESULT_BACKEND`, + ## or `AIRFLOW__CELERY__BROKER_URL`, they are dynamically created from chart values + ## + ## NOTE: + ## - airflow allows environment configs to be set as environment variables + ## - they take the form: AIRFLOW__
__ + ## - see the Airflow documentation: https://airflow.apache.org/docs/stable/howto/set-config.html + ## + ## EXAMPLE: + ## config: + ## ## Security + ## AIRFLOW__CORE__SECURE_MODE: "True" + ## AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.deny_all" + ## AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "False" + ## AIRFLOW__WEBSERVER__RBAC: "False" + ## + ## ## DAGS + ## AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: "30" + ## AIRFLOW__CORE__LOAD_EXAMPLES: "False" + ## + ## ## Email (SMTP) + ## AIRFLOW__EMAIL__EMAIL_BACKEND: "airflow.utils.email.send_email_smtp" + ## AIRFLOW__SMTP__SMTP_HOST: "smtpmail.example.com" + ## AIRFLOW__SMTP__SMTP_STARTTLS: "False" + ## AIRFLOW__SMTP__SMTP_SSL: "False" + ## AIRFLOW__SMTP__SMTP_PORT: "25" + ## AIRFLOW__SMTP__SMTP_MAIL_FROM: "admin@example.com" + ## + ## ## Disable noisy "Handling signal: ttou" Gunicorn log messages + ## GUNICORN_CMD_ARGS: "--log-level WARNING" + ## + ## ## Proxy Config + ## HTTP_PROXY: "http://proxy.example.com:8080" + ## + config: {} + + ## extra annotations for the web/scheduler/worker Pods + ## + ## EXAMPLE: + ## podAnnotations: + ## iam.amazonaws.com/role: airflow-Role + ## + podAnnotations: {} + + ## extra environment variables for the web/scheduler/worker (AND flower) Pods + ## + ## EXAMPLE: + ## extraEnv: + ## - name: AIRFLOW__CORE__FERNET_KEY + ## valueFrom: + ## secretKeyRef: + ## name: airflow-fernet-key + ## key: value + ## - name: AIRFLOW__LDAP__BIND_PASSWORD + ## valueFrom: + ## secretKeyRef: + ## name: airflow-ldap-password + ## key: value + ## + extraEnv: + - name: AIRFLOW__GOOGLE__CLIENT_ID + valueFrom: + secretKeyRef: + name: airflow + key: google_client_id + - name: AIRFLOW__GOOGLE__CLIENT_SECRET + valueFrom: + secretKeyRef: + name: airflow + key: google_client_secret + - name: AIRFLOW__GOOGLE__OAUTH_CALLBACK_ROUTE + value: "/oauth2callback" + - name: AIRFLOW__GOOGLE__DOMAIN + value: "tractable.ai" + - name: AIRFLOW__WEBSERVER__AUTHENTICATE + value: "True" + - name: AIRFLOW__WEBSERVER__AUTH_BACKEND + value: airflow.contrib.auth.backends.google_auth + + ## extra configMap volumeMounts for the web/scheduler/worker Pods + ## + ## EXAMPLE: + ## extraConfigmapMounts: + ## - name: airflow-webserver-config + ## mountPath: /opt/airflow/webserver_config.py + ## configMap: airflow-webserver-config + ## readOnly: true + ## subPath: webserver_config.py + ## + extraConfigmapMounts: [] + + ## extra containers for the web/scheduler/worker Pods + ## + ## EXAMPLE: (a sidecar that syncs DAGs from object storage) + ## extraContainers: + ## - name: s3-sync + ## image: my-user/s3sync:latest + ## volumeMounts: + ## - name: synchronised-dags + ## mountPath: /dags + ## + extraContainers: [] + + ## extra pip packages to install in the web/scheduler/worker Pods + ## + ## EXAMPLE: + ## extraPipPackages: + ## - "airflow-exporter==1.3.1" + ## + extraPipPackages: + - "airflow-exporter" + - "apache-airflow[aws]" + - "apache-airflow[mysql]" + - "apache-airflow[postgres]" + - "apache-airflow[slack]" + - "apache-airflow[statsd]" + + ## extra volumeMounts for the web/scheduler/worker Pods + ## + ## EXAMPLE: + ## extraVolumeMounts: + ## - name: synchronised-dags + ## mountPath: /opt/airflow/dags + ## + extraVolumeMounts: [] + + ## extra volumes for the web/scheduler/worker Pods + ## + ## EXAMPLE: + ## extraVolumes: + ## - name: synchronised-dags + ## emptyDir: {} + ## + extraVolumes: [] + + +################################### +# Airflow - Scheduler Configs +################################### +scheduler: + ## resource requests/limits for the scheduler Pod + ## + ## EXAMPLE: + ## resources: + ## requests: + ## cpu: "1000m" + ## memory: "1Gi" + ## + resources: {} + + ## the nodeSelector configs for the scheduler Pods + ## + nodeSelector: {} + + ## the affinity configs for the scheduler Pods + ## + affinity: {} + + ## the toleration configs for the scheduler Pods + ## + tolerations: [] + + ## labels for the scheduler Deployment + ## + labels: {} + + ## Pod labels for the scheduler Deployment + ## + podLabels: {} + + ## annotations for the scheduler Deployment + ## + annotations: {} + + ## Pod Annotations for the scheduler Deployment + ## + podAnnotations: {} + + ## configs for the PodDisruptionBudget of the scheduler + ## + podDisruptionBudget: + ## if a PodDisruptionBudget resource is created for the scheduler + ## + enabled: true + + ## the maximum unavailable pods/percentage for the scheduler + ## + ## NOTE: + ## - as there is only ever a single scheduler Pod, + ## this must be 100% for Kubernetes to be able to migrate it + ## + maxUnavailable: "100%" + + ## the minimum available pods/percentage for the scheduler + ## + minAvailable: "" + + ## custom airflow connections for the airflow scheduler + ## + ## NOTE: + ## - connections are created with a script that is stored in a K8s secret and mounted into the scheduler container + ## + ## EXAMPLE: + ## connections: + ## - id: my_aws + ## type: aws + ## extra: | + ## { + ## "aws_access_key_id": "XXXXXXXXXXXXXXXXXXX", + ## "aws_secret_access_key": "XXXXXXXXXXXXXXX", + ## "region_name":"eu-central-1" + ## } + ## + connections: [] + + ## custom airflow variables for the airflow scheduler + ## + ## NOTE: + ## - THIS IS A STRING, containing a JSON object, with your variables in it + ## + ## EXAMPLE: + ## variables: | + ## { "environment": "dev" } + ## + variables: | + {} + + ## custom airflow pools for the airflow scheduler + ## + ## NOTE: + ## - THIS IS A STRING, containing a JSON object, with your pools in it + ## + ## EXAMPLE: + ## pools: | + ## { + ## "example": { + ## "description": "This is an example pool with 2 slots.", + ## "slots": 2 + ## } + ## } + ## + pools: | + {} + + ## the value of the `airflow --num_runs` parameter used to run the airflow scheduler + ## + ## NOTE: + ## - this is the number of 'dag refreshes' before the airflow scheduler process will exit + ## - if not set to `-1`, the scheduler Pod will restart regularly + ## - for most environments, `-1` will be an acceptable value + ## + numRuns: -1 + + ## if we run `airflow initdb` when the scheduler starts + ## + initdb: true + + ## if we run `airflow initdb` inside a special initContainer + ## + ## NOTE: + ## - may be needed if you have custom database hooks configured that will be pulled in by git-sync + ## + preinitdb: false + + ## the number of seconds to wait (in bash) before starting the scheduler container + ## + initialStartupDelay: 0 + + ## extra init containers to run before the scheduler Pod + ## + ## EXAMPLE: + ## extraInitContainers: + ## - name: volume-mount-hack + ## image: busybox + ## command: ["sh", "-c", "chown -R 1000:1000 logs"] + ## volumeMounts: + ## - mountPath: /opt/airflow/logs + ## name: logs-data + ## + extraInitContainers: [] + +################################### +# Airflow - WebUI Configs +################################### +web: + ## resource requests/limits for the airflow web Pods + ## + ## EXAMPLE: + ## resources: + ## requests: + ## cpu: "500m" + ## memory: "1Gi" + ## + resources: {} + + ## the number of web Pods to run + ## + replicas: 1 + + ## the nodeSelector configs for the web Pods + ## + nodeSelector: {} + + ## the affinity configs for the web Pods + ## + affinity: {} + + ## the toleration configs for the web Pods + ## + tolerations: [] + + ## labels for the web Deployment + ## + labels: {} + + ## Pod labels for the web Deployment + ## + podLabels: {} + + ## annotations for the web Deployment + ## + annotations: {} + + ## Pod annotations for the web Deployment + ## + podAnnotations: {} + + ## configs for the Service of the web Pods + ## + service: + annotations: {} + sessionAffinity: "None" + sessionAffinityConfig: {} + type: ClusterIP + externalPort: 8080 + loadBalancerIP: "" + loadBalancerSourceRanges: [] + nodePort: + http: "" + + ## sets `AIRFLOW__WEBSERVER__BASE_URL` + ## + ## NOTE: + ## - should be compatible with `ingress.web.path` config + ## + baseUrl: "https://airflow.dev.k8s.tractable.io" + + ## sets `AIRFLOW__CORE__STORE_SERIALIZED_DAGS` + ## + ## NOTE: + ## - setting true will disable `git-sync` and `git-clone` containers in the web Pod + ## - Docs: https://airflow.apache.org/docs/stable/dag-serialization.html + ## + serializeDAGs: false + + ## extra pip packages to install in the web container + ## + ## EXAMPLE: ( packages used by RBAC UI for OAuth ) + ## extraPipPackages: + ## - "apache-airflow[google_auth]==1.10.10" + ## + extraPipPackages: + - "apache-airflow[google_auth]==1.10.10" + + ## the number of seconds to wait (in bash) before starting the web container + ## + initialStartupDelay: 0 + + ## the number of seconds to wait before declaring a new Pod available + ## + minReadySeconds: 5 + + ## configs for the web Service readiness probe + ## + readinessProbe: + enabled: false + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + + ## configs for the web Service liveness probe + ## + livenessProbe: + enabled: true + scheme: HTTP + ## the number of seconds to wait before checking pod health + ## + ## NOTE: + ## - make larger if you are installing many packages with: + ## `airflow.extraPipPackages`, `web.extraPipPackages`, or `dags.installRequirements` + ## + initialDelaySeconds: 300 + periodSeconds: 30 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 2 + + ## the directory in which to mount secrets on web containers + ## + secretsDir: /var/airflow/secrets + + ## secret names which will be mounted as a file at `{web.secretsDir}/` + ## + ## EXAMPLE: + ## secrets: + ## - airflow-web-secret + ## + secrets: [] + # secretsMap: "airflow" + +################################### +# Airflow - Worker Configs +################################### +workers: + ## if the airflow workers StatefulSet should be deployed + ## + enabled: true + + ## resource requests/limits for the airflow worker Pods + ## + ## EXAMPLE: + ## resources: + ## requests: + ## cpu: "1000m" + ## memory: "2Gi" + ## + resources: {} + + ## the number of workers Pods to run + ## + ## NOTE: + ## - when `workers.autoscaling.enabled` is true, this is the minimum + ## + replicas: 1 + + ## the nodeSelector configs for the worker Pods + ## + nodeSelector: {} + + ## the affinity configs for the worker Pods + ## + affinity: {} + + ## the toleration configs for the worker Pods + ## + tolerations: [] + + ## labels for the worker StatefulSet + ## + labels: {} + + ## Pod labels for the worker StatefulSet + ## + podLabels: {} + + ## annotations for the worker StatefulSet + ## + annotations: {} + + ## Pod annotations for the worker StatefulSet + ## + podAnnotations: {} + + ## configs for the HorizontalPodAutoscaler of the worker Pods + ## + ## EXAMPLE: + ## autoscaling: + ## enabled: true + ## maxReplicas: 16 + ## metrics: + ## - type: Resource + ## resource: + ## name: memory + ## target: + ## type: Utilization + ## averageUtilization: 80 + ## + autoscaling: + enabled: false + maxReplicas: 2 + metrics: [] + + ## the number of seconds to wait (in bash) before starting each worker container + ## + initialStartupDelay: 0 + + ## configs for the celery worker Pods + ## + ## NOTE: + ## - only takes effect if `airflow.executor` is `CeleryExecutor` + ## + celery: + ## the number of tasks each celery worker can run at a time + ## + ## NOTE: + ## - sets AIRFLOW__CELERY__WORKER_CONCURRENCY + ## + instances: 1 + + ## if we should wait for tasks to finish on a celery worker before SIGTERM of Pod + ## + ## NOTE: + ## - `workers.terminationPeriod` is still the overall timeout before worker Pods are killed using SIGKILL + ## + gracefullTermination: false + + ## how many seconds to wait for tasks on a worker to finish before SIGKILL + ## + terminationPeriod: 60 + + ## directory in which to mount secrets on worker containers + ## + secretsDir: /var/airflow/secrets + + ## secret names which will be mounted as a file at `{workers.secretsDir}/` + ## + ## EXAMPLE: + ## secrets: + ## - airflow-worker-secret + ## + secrets: [] + +################################### +# Airflow - Flower Configs +################################### +flower: + ## if the Flower UI should be deployed + ## + ## NOTE: + ## - only takes effect if `airflow.executor` is `CeleryExecutor` + ## + enabled: true + + ## resource requests/limits for the flower Pods + ## + ## EXAMPLE: + ## resources: + ## requests: + ## cpu: "100m" + ## memory: "126Mi" + ## + resources: {} + + ## the nodeSelector configs for the flower Pods + ## + nodeSelector: {} + + ## the affinity configs for the flower Pods + ## + affinity: {} + + ## the toleration configs for the flower Pods + ## + tolerations: [] + + ## labels for the flower Deployment + ## + labels: {} + + ## Pod labels for the flower Deployment + ## + podLabels: {} + + ## annotations for the flower Deployment + ## + annotations: {} + + ## Pod annotations for the flower Deployment + ## + podAnnotations: {} + + ## sets `AIRFLOW__CELERY__FLOWER_URL_PREFIX` + ## + ## NOTE: + ## - should match `ingress.flower.path` config + ## + urlPrefix: "" + + ## configs for the Service of the flower Pods + ## + service: + annotations: {} + type: ClusterIP + externalPort: 5555 + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## the number of seconds to wait (in bash) before starting the flower container + ## + initialStartupDelay: 0 + + ## extra ConfigMaps to mount on the flower Pods + ## + ## EXAMPLE: + ## extraConfigmapMounts: + ## - name: extra-cert + ## mountPath: /etc/ssl/certs/extra-cert.pem + ## configMap: extra-certificates + ## readOnly: true + ## subPath: extra-cert.pem + ## + extraConfigmapMounts: [] + +################################### +# Airflow - Logs Configs +################################### +logs: + ## the airflow logs folder + ## + path: /opt/airflow/logs + + ## configs for the logs PVC + ## + persistence: + ## if a persistent volume is mounted at `logs.path` + ## + enabled: false + + ## the name of an existing PVC to use + ## + existingClaim: "" + + ## sub-path under `logs.persistence.existingClaim` to use + ## + subPath: "" + + ## the name of the StorageClass used by the PVC + ## + ## NOTE: + ## - if set to "", then `PersistentVolumeClaim/spec.storageClassName` is omitted + ## - if set to "-", then `PersistentVolumeClaim/spec.storageClassName` is set to "" + ## + storageClass: "" + + ## the access mode of the PVC + ## + ## WARNING: + ## - must be: `ReadWriteMany` + ## + ## NOTE: + ## - different StorageClass support different access modes: + ## https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes + ## + accessMode: ReadWriteMany + + ## the size of PVC to request + ## + size: 1Gi + +################################### +# Airflow - DAGs Configs +################################### +dags: + ## the airflow dags folder + ## + path: /opt/airflow/dags + + ## whether to disable pickling dags from the scheduler to workers + ## + ## NOTE: + ## - sets AIRFLOW__CORE__DONOT_PICKLE + ## + doNotPickle: false + + ## install any Python `requirements.txt` at the root of `dags.path` automatically + ## + installRequirements: false + + ## configs for the dags PVC + ## + persistence: + ## if a persistent volume is mounted at `dags.path` + ## + enabled: false + + ## the name of an existing PVC to use + ## + existingClaim: "" + + ## sub-path under `dags.persistence.existingClaim` to use + ## + subPath: "" + + ## the name of the StorageClass used by the PVC + ## + ## NOTE: + ## - if set to "", then `PersistentVolumeClaim/spec.storageClassName` is omitted + ## - if set to "-", then `PersistentVolumeClaim/spec.storageClassName` is set to "" + ## + storageClass: "" + + ## the access mode of the PVC + ## + ## WARNING: + ## - must be one of: `ReadOnlyMany` or `ReadWriteMany` + ## + ## NOTE: + ## - different StorageClass support different access modes: + ## https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes + ## + accessMode: ReadOnlyMany + + ## the size of PVC to request + ## + size: 1Gi + + ## configs for the DAG git repository & sync container + ## + git: + url: "ssh://git@bitbucket.org/xmachinas/tractable.airflow.git" + + ## the branch/tag/sha1 which we clone + ## + ref: master + + ## the name of a pre-created secret containing files for ~/.ssh/ + ## + ## NOTE: + ## - this is ONLY RELEVANT for SSH git repos + ## - the secret commonly includes files: id_rsa, id_rsa.pub, known_hosts + ## - known_hosts is NOT NEEDED if `git.sshKeyscan` is true + ## + secret: airflow + + ## if we should implicitly trust [git.repoHost]:git.repoPort, by auto creating a ~/.ssh/known_hosts + ## + ## WARNING: + ## - setting true will increase your vulnerability ot a repo spoofing attack + ## + ## NOTE: + ## - this is ONLY RELEVANT for SSH git repos + ## - this is not needed if known_hosts is provided in `git.secret` + ## - git.repoHost and git.repoPort ARE REQUIRED for this to work + ## + sshKeyscan: true + + ## the name of the private key file in your `git.secret` + ## + ## NOTE: + ## - this is ONLY RELEVANT for PRIVATE SSH git repos + ## + privateKeyName: id_rsa + + ## the host name of the git repo + ## + ## NOTE: + ## - this is ONLY REQUIRED for SSH git repos + ## + ## EXAMPLE: + ## repoHost: "github.com" + ## + repoHost: "bitbucket.org" + + ## the port of the git repo + ## + ## NOTE: + ## - this is ONLY REQUIRED for SSH git repos + ## + repoPort: 22 + + ## configs for the git-sync container + ## + gitSync: + ## enable the git-sync sidecar container + ## + enabled: false + + ## resource requests/limits for the git-sync container + ## + ## NOTE: + ## - when `workers.autoscaling` is true, YOU MUST SPECIFY a resource request + ## + ## EXAMPLE: + ## resources: + ## requests: + ## cpu: "50m" + ## memory: "64Mi" + ## + resources: {} + + ## the docker image for the git-sync container + image: + repository: alpine/git + tag: latest + ## values: Always or IfNotPresent + pullPolicy: Always + + ## the git sync interval in seconds + ## + refreshTime: 60 + + ## configs for the git-clone container + ## + ## NOTE: + ## - use this container if you want to only clone the external git repo + ## at Pod start-time, and not keep it synchronised afterwards + ## + initContainer: + ## enable the git-clone sidecar container + ## + ## NOTE: + ## - this is NOT required for the git-sync sidecar to work + ## - in most environments, you can leave this as false + ## + enabled: true + + ## resource requests/limits for the git-clone container + ## + ## EXAMPLE: + ## resources: + ## requests: + ## cpu: "50m" + ## memory: "64Mi" + ## + resources: {} + + ## the docker image for the git-clone container + image: + repository: alpine/git + tag: latest + ## values: Always or IfNotPresent + pullPolicy: Always + + ## path to mount dags-data volume to + ## + ## WARNING: + ## - this path is also used by the git-sync container + ## + mountPath: "/dags" + + ## sub-path under `dags.initContainer.mountPath` to sync dags to + ## + ## WARNING: + ## - this path is also used by the git-sync container + ## - this MUST INCLUDE the leading / + ## + ## EXAMPLE: + ## syncSubPath: "/subdirWithDags" + ## + syncSubPath: "/src/dags" + +################################### +# Kubernetes - RBAC +################################### +rbac: + ## if Kubernetes RBAC resources are created + ## + ## NOTE: + ## - these allow the service account to create/delete Pods in the airflow namespace, + ## which is required for the KubernetesPodOperator() to function + ## + create: true + +################################### +# Kubernetes - Service Account +################################### +serviceAccount: + ## if a Kubernetes ServiceAccount is created + ## + ## NOTE: + ## - if false, you must create the service account outside of this helm chart, + ## with the name: `serviceAccount.name` + ## + create: true + + ## the name of the ServiceAccount + ## + ## NOTE: + ## - by default the name is generated using the `airflow.serviceAccountName` template in `_helpers.tpl` + ## + name: "" + + ## annotations for the ServiceAccount + ## + ## EXAMPLE: (to use WorkloadIdentity in Google Cloud) + ## annotations: + ## iam.gke.io/gcp-service-account: <>@<>.iam.gserviceaccount.com + ## + annotations: {} + +################################### +# Kubernetes - Extra Manifests +################################### +## additional Kubernetes manifests to include with this chart +## +## EXAMPLE: +## extraManifests: +## - apiVersion: cloud.google.com/v1beta1 +## kind: BackendConfig +## metadata: +## name: "{{ .Release.Name }}-test" +## spec: +## securityPolicy: +## name: "gcp-cloud-armor-policy-test" +## +extraManifests: [] + +################################### +# Database - PostgreSQL Chart +# - https://github.com/helm/charts/tree/master/stable/postgresql +################################### +postgresql: + ## if the `stable/postgresql` chart is used + ## + ## WARNING: + ## - this is NOT SUITABLE for production deployments of Airflow, + ## you should seriously consider using an external database service, + ## which can be configured with values under: `externalDatabase` + ## + ## NOTE: + ## - set to `false` if using an external database + ## + enabled: true + + ## the postgres database to use + ## + postgresqlDatabase: airflow + + ## the postgres user to create + ## + postgresqlUsername: postgres + + ## the postgres user's password + ## + ## WARNING: + ## - you should NOT use this, instead specify `postgresql.existingSecret` + ## + postgresqlPassword: airflow + + ## the name of a pre-created secret containing the postgres password + ## + existingSecret: "" + + ## the key in `postgresql.existingSecret` containing the password string + ## + existingSecretKey: "postgresql-password" + + ## configs for the PVC of postgresql + ## + persistence: + ## if postgres will use Persistent Volume Claims to store data + ## + ## WARNING: + ## - if false, data will be LOST as postgres Pods restart + ## + enabled: true + + ## the name of the StorageClass used by the PVC + ## + storageClass: "" + + ## the access modes of the PVC + ## + accessModes: + - ReadWriteOnce + + ## the size of PVC to request + ## + size: 8Gi + +################################### +# Database - External Database +# - these configs are only used when `postgresql.enabled` is false +################################### +externalDatabase: + ## the type of external database: {mysql,postgres} + ## + type: postgres + + ## the host of the external database + ## + host: localhost + + ## the port of the external database + ## + port: 5432 + + ## the database/scheme to use within the the external database + ## + database: airflow + + ## the user of the external database + ## + user: airflow + + ## the name of a pre-created secret containing the external database password + ## + passwordSecret: "" + + ## the key within `externalDatabase.passwordSecret` containing the password string + ## + passwordSecretKey: "postgresql-password" + +################################### +# Database - Redis Chart +# - https://github.com/helm/charts/tree/master/stable/redis +################################### +redis: + ## if the `stable/redis` chart is used + ## + ## NOTE: + ## - set to `false` if using an external redis database + ## - set to `false` if `airflow.executor` is `KubernetesExecutor` + ## + enabled: true + + ## the redis password + ## + ## WARNING: + ## - you should NOT use this, instead specify `redis.existingSecret` + ## + password: airflow + + ## the name of a pre-created secret containing the redis password + ## + existingSecret: "" + + ## the key in `redis.existingSecret` containing the password string + ## + existingSecretKey: "redis-password" + + ## configs for redis cluster mode + ## + cluster: + ## if redis runs in cluster mode + ## + enabled: false + + ## the number of redis slaves + ## + slaveCount: 1 + + ## configs for the redis master + ## + master: + ## resource requests/limits for the master Pod + ## + ## EXAMPLE: + ## resources: + ## requests: + ## cpu: "100m" + ## memory: "256Mi" + ## + resources: {} + + ## configs for the PVC of the redis master + ## + persistence: + ## use a PVC to persist data + ## + enabled: false + + ## the name of the StorageClass used by the PVC + ## + storageClass: "" + + ## the access mode of the PVC + ## + accessModes: + - ReadWriteOnce + + ## the size of PVC to request + ## + size: 8Gi + + ## configs for the redis slaves + ## + slave: + ## resource requests/limits for the slave Pods + ## + ## EXAMPLE: + ## resources: + ## requests: + ## cpu: "100m" + ## memory: "256Mi" + ## + resources: {} + + ## configs for the PVC of the redis slaves + ## + persistence: + ## use a PVC to persist data + ## + enabled: false + + ## the name of the StorageClass used by the PVC + ## + storageClass: "" + + ## the access mode of the PVC + ## + accessModes: + - ReadWriteOnce + + ## the size of PVC to request + ## + size: 8Gi + +################################### +# Database - External Database +# - these configs are only used when `redis.enabled` is false +################################### +externalRedis: + ## the host of the external redis + ## + host: localhost + + ## the port of the external redis + ## + port: 6379 + + ## the database number to use within the the external redis + ## + databaseNumber: 1 + + ## the name of a pre-created secret containing the external redis password + ## + passwordSecret: "" + + ## the key within `externalRedis.passwordSecret` containing the password string + ## + passwordSecretKey: "redis-password" + +################################### +# Prometheus - ServiceMonitor +################################### +serviceMonitor: + ## if the ServiceMonitor resources should be deployed + ## + ## WARNING: + ## - you will need an exporter in your airflow docker container, for example: + ## https://github.com/epoch8/airflow-exporter + ## + ## NOTE: + ## - you can install pip packages with `airflow.extraPipPackages` + ## - ServiceMonitor is a resource from: https://github.com/coreos/prometheus-operator + ## + enabled: false + + ## labels for ServiceMonitor, so that Prometheus can select it + ## + selector: + prometheus: kube-prometheus + + ## the ServiceMonitor web endpoint path + ## + path: /admin/metrics + + ## the ServiceMonitor web endpoint interval + ## + interval: "30s" + +################################### +# Prometheus - PrometheusRule +################################### +prometheusRule: + ## if the PrometheusRule resources should be deployed + ## + ## WARNING: + ## - you will need an exporter in your airflow docker container, for example: + ## https://github.com/epoch8/airflow-exporter + ## + ## NOTE: + ## - you can install pip packages with `airflow.extraPipPackages` + ## - PrometheusRule a resource from: https://github.com/coreos/prometheus-operator + ## + enabled: false + + ## labels for PrometheusRule, so that Prometheus can select it + ## + additionalLabels: {} + + ## alerting rules for Prometheus + ## + ## NOTE: + ## - documentation: https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/ + ## + groups: [] \ No newline at end of file diff --git a/stable/airflow/values.yaml b/stable/airflow/values.yaml index d84cd2ff5c..55a64dd4a4 100644 --- a/stable/airflow/values.yaml +++ b/stable/airflow/values.yaml @@ -432,6 +432,15 @@ web: ## secrets: [] + ## you can use secretsMap to specify a map and all the secrets will be stored within it + ## secrets will be mounted as files at `{web.secretsDir}/` + ## If you use web.secretsMap, then it overrides web.secrets. + ## + ## EXAMPLE: + ## secretsMap: airflow-secrets + ## + secretsMap: + ################################### # Airflow - Worker Configs ################################### @@ -544,6 +553,15 @@ workers: ## secrets: [] + ## you can use secretsMap to specify a map and all the secrets will be stored within it + ## secrets will be mounted as files at `{workers.secretsDir}/` + ## If you use web.secretsMap, then it overrides workers.secrets. + ## + ## EXAMPLE: + ## secretsMap: airflow-secrets + ## + secretsMap: + ################################### # Airflow - Flower Configs ###################################