From 688c7d36ad89dfdbce479287554979eb8a980f0e Mon Sep 17 00:00:00 2001 From: omonnier Date: Tue, 19 Nov 2019 15:55:42 +0100 Subject: [PATCH] [stable/airflow] Only set Celery & Redis env variables when needed (#18992) The environment variables for Redis and Celery prevent the helm chart to work in LocalExecutori mode. They are now only exported if: - Redis is enabled - and Airflow executor is "Celery" Signed-off-by: Olivier Monnier <5500018+omonnier@users.noreply.github.com> --- stable/airflow/Chart.yaml | 2 +- stable/airflow/templates/configmap-env.yaml | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/stable/airflow/Chart.yaml b/stable/airflow/Chart.yaml index e4b1c139f4..55a854793e 100644 --- a/stable/airflow/Chart.yaml +++ b/stable/airflow/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Airflow is a platform to programmatically author, schedule and monitor workflows name: airflow -version: 4.6.0 +version: 4.7.0 appVersion: 1.10.4 icon: https://airflow.apache.org/_images/pin_large.png home: https://airflow.apache.org/ diff --git a/stable/airflow/templates/configmap-env.yaml b/stable/airflow/templates/configmap-env.yaml index 104c84fc27..fbd132ce4c 100644 --- a/stable/airflow/templates/configmap-env.yaml +++ b/stable/airflow/templates/configmap-env.yaml @@ -14,19 +14,23 @@ data: POSTGRES_HOST: "{{ template "airflow.postgresql.fullname" . }}" POSTGRES_PORT: "{{ .Values.postgresql.service.port }}" POSTGRES_DB: "{{ .Values.postgresql.postgresDatabase }}" + {{- if eq .Values.airflow.executor "Celery" }} ## Redis DB configuration REDIS_HOST: "{{ template "airflow.redis.host" . }}" REDIS_PORT: "{{ .Values.redis.master.port }}" + AIRFLOW__CELERY__FLOWER_URL_PREFIX: "{{ .Values.ingress.flower.path }}" + AIRFLOW__CELERY__WORKER_CONCURRENCY: "{{ .Values.workers.celery.instances }}" ## Flower PORT FLOWER_PORT: "5555" + # For backwards compat with AF < 1.10, CELERY_CONCURRENCY got renamed to WORKER_CONCURRENCY + AIRFLOW__CELERY__CELERY_CONCURRENCY: "{{ .Values.workers.celery.instances }}" + {{- end }} # Configure puckel's docker-airflow entrypoint EXECUTOR: "{{ .Values.airflow.executor }}" FERNET_KEY: "{{ .Values.airflow.fernetKey }}" DO_WAIT_INITDB: "false" ## Custom Airflow settings AIRFLOW__CORE__DONOT_PICKLE: "{{ .Values.dags.doNotPickle }}" - AIRFLOW__CELERY__FLOWER_URL_PREFIX: "{{ .Values.ingress.flower.path }}" - AIRFLOW__CELERY__WORKER_CONCURRENCY: "{{ .Values.workers.celery.instances }}" AIRFLOW__CORE__DAGS_FOLDER: "{{ .Values.dags.path }}" AIRFLOW__CORE__BASE_LOG_FOLDER: "{{ .Values.logs.path }}" AIRFLOW__CORE__DAG_PROCESSOR_MANAGER_LOG_LOCATION: "{{ printf "%s/%s" .Values.logs.path "dag_processor_manager/dag_processor_manager.log" }}" @@ -34,8 +38,6 @@ data: AIRFLOW__WEBSERVER__BASE_URL: "{{ .Values.ingress.web.host }}{{ .Values.ingress.web.path }}" # Disabling XCom pickling for forward compatibility AIRFLOW__CORE__ENABLE_XCOM_PICKLING: "false" - # For backwards compat with AF < 1.10, CELERY_CONCURRENCY got renamed to WORKER_CONCURRENCY - AIRFLOW__CELERY__CELERY_CONCURRENCY: "{{ .Values.workers.celery.instances }}" # Note: changing `Values.airflow.config` won't change the configmap checksum and so won't make # the pods to restart {{- range $setting, $option := .Values.airflow.config }}