diff --git a/stable/airflow/Chart.yaml b/stable/airflow/Chart.yaml index 598b38529d..f4c095fd0a 100644 --- a/stable/airflow/Chart.yaml +++ b/stable/airflow/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Airflow is a platform to programmatically author, schedule and monitor workflows name: airflow -version: 7.9.0 +version: 7.10.0 appVersion: 1.10.12 icon: https://airflow.apache.org/_images/pin_large.png home: https://airflow.apache.org/ diff --git a/stable/airflow/README.md b/stable/airflow/README.md index 3d333ba943..4dc585a90b 100644 --- a/stable/airflow/README.md +++ b/stable/airflow/README.md @@ -4,11 +4,22 @@ ## Installation -To install the Airflow Helm Chart: +(Helm 2) To install the Airflow Helm Chart: ```bash helm install stable/airflow \ - --version "X.X.X" \ --name "airflow" \ + --version "X.X.X" \ + --namespace "airflow" \ + --values ./custom-values.yaml +``` + +(Helm 3) To install the Airflow Helm Chart: +```bash +helm repo add stable https://kubernetes-charts.storage.googleapis.com +helm repo update + +helm install "airflow" stable/airflow \ + --version "X.X.X" \ --namespace "airflow" \ --values ./custom-values.yaml ``` @@ -23,14 +34,15 @@ To uninstall the Airflow Helm Chart: helm delete "airflow" ``` -To run bash commands in the Airflow Scheduler Pod: +To run bash commands in the Airflow Webserver Pod: ```bash -# use this to run commands like: `airflow create_user` +# create an interactive bash session in the Webserver Pod +# use this bash session for commands like: `airflow create_user` kubectl exec \ -it \ --namespace airflow \ - --container airflow-scheduler \ - Deployment/airflow-scheduler \ + --container airflow-web \ + Deployment/airflow-web \ /bin/bash ``` @@ -39,6 +51,7 @@ kubectl exec \ > NOTE: for chart version numbers, see [Chart.yaml](Chart.yaml) or [helm hub](https://hub.helm.sh/charts/stable/airflow). For steps you must take when upgrading this chart, please review: +* [v7.9.X → v7.10.0](UPGRADE.md#v79x--v7100) * [v7.8.X → v7.9.0](UPGRADE.md#v78x--v790) * [v7.7.X → v7.8.0](UPGRADE.md#v77x--v780) * [v7.6.X → v7.7.0](UPGRADE.md#v76x--v770) diff --git a/stable/airflow/UPGRADE.md b/stable/airflow/UPGRADE.md index ec2c77762b..fe36fef4ec 100644 --- a/stable/airflow/UPGRADE.md +++ b/stable/airflow/UPGRADE.md @@ -1,5 +1,14 @@ # Upgrading Steps +## `v7.9.X` → `v7.10.0` + +__The following IMPROVEMENTS have been made:__ +* We now make use of the `_CMD` variables for `AIRFLOW__CORE__SQL_ALCHEMY_CONN_CMD`, `AIRFLOW__CELERY__RESULT_BACKEND_CMD`, and `AIRFLOW__CELERY__BROKER_URL_CMD`: + * This fixes the Scheduler liveness probe implemented in `7.8.0` + * This fixes using `kubectl exec` to run commands like `airflow create_user` +* Configs passed with `airflow.config` are now passed as-defined in your `values.yaml`: + * This fixes an issue where people had to escape `"` characters in JSON strings + ## `v7.8.X` → `v7.9.0` __The following IMPROVEMENTS have been made:__ diff --git a/stable/airflow/templates/_helpers.tpl b/stable/airflow/templates/_helpers.tpl index ff954a39a9..c4eb22c4ca 100644 --- a/stable/airflow/templates/_helpers.tpl +++ b/stable/airflow/templates/_helpers.tpl @@ -67,39 +67,46 @@ Used to discover the master Service and Secret name created by the sub-chart. {{- end -}} {{- end -}} - {{/* -Construct the AIRFLOW__CORE__SQL_ALCHEMY_CONN connection string. +Bash command which echos the DB connection string in SQLAlchemy format. +NOTE: + - used by `AIRFLOW__CORE__SQL_ALCHEMY_CONN_CMD` + - the `DATABASE_PASSWORD_CMD` sub-command is set in `configmap-env` */}} -{{- define "airflow.connection.alchemy" -}} +{{- define "DATABASE_SQLALCHEMY_CMD" -}} {{- if .Values.postgresql.enabled -}} -postgresql+psycopg2://${DATABASE_USER}:$( echo ${DATABASE_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB} +echo -n "postgresql+psycopg2://${DATABASE_USER}:$(eval $DATABASE_PASSWORD_CMD)@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}" {{- else if and (not .Values.postgresql.enabled) (eq "postgres" .Values.externalDatabase.type) -}} -postgresql+psycopg2://${DATABASE_USER}:$( echo ${DATABASE_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}${DATABASE_PROPERTIES} +echo -n "postgresql+psycopg2://${DATABASE_USER}:$(eval $DATABASE_PASSWORD_CMD)@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}${DATABASE_PROPERTIES}" {{- else if and (not .Values.postgresql.enabled) (eq "mysql" .Values.externalDatabase.type) -}} -mysql+mysqldb://${DATABASE_USER}:$( echo ${DATABASE_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}${DATABASE_PROPERTIES} +echo -n "mysql+mysqldb://${DATABASE_USER}:$(eval $DATABASE_PASSWORD_CMD)@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}${DATABASE_PROPERTIES}" {{- end -}} {{- end -}} {{/* -Construct the AIRFLOW__CELERY__RESULT_BACKEND connection string. +Bash command which echos the DB connection string in Celery result_backend format. +NOTE: + - used by `AIRFLOW__CELERY__RESULT_BACKEND_CMD` + - the `DATABASE_PASSWORD_CMD` sub-command is set in `configmap-env` */}} -{{- define "airflow.connection.celery.backend" -}} +{{- define "DATABASE_CELERY_CMD" -}} {{- if .Values.postgresql.enabled -}} -db+postgresql://${DATABASE_USER}:$( echo ${DATABASE_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB} +echo -n "db+postgresql://${DATABASE_USER}:$(eval $DATABASE_PASSWORD_CMD)@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}" {{- else if and (not .Values.postgresql.enabled) (eq "postgres" .Values.externalDatabase.type) -}} -db+postgresql://${DATABASE_USER}:$( echo ${DATABASE_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}${DATABASE_PROPERTIES} +echo -n "db+postgresql://${DATABASE_USER}:$(eval $DATABASE_PASSWORD_CMD)@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}${DATABASE_PROPERTIES}" {{- else if and (not .Values.postgresql.enabled) (eq "mysql" .Values.externalDatabase.type) -}} -db+mysql://${DATABASE_USER}:$( echo ${DATABASE_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}${DATABASE_PROPERTIES} +echo -n "db+mysql://${DATABASE_USER}:$(eval $DATABASE_PASSWORD_CMD)@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB}${DATABASE_PROPERTIES}" {{- end -}} {{- end -}} {{/* -Construct the AIRFLOW__CELERY__BROKER_URL connection string. -Note that a redis URL with no password excludes the "@" prefix. +Bash command which echos the Redis connection string. +NOTE: + - used by `AIRFLOW__CELERY__BROKER_URL_CMD` + - the `REDIS_PASSWORD_CMD` sub-command is set in `configmap-env` */}} -{{- define "airflow.connection.celery.broker" -}} -redis://$( echo ${REDIS_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(f\":{encoded_pass}@\") if len(encoded_pass) > 0 else None" )${REDIS_HOST}:${REDIS_PORT}/${REDIS_DBNUM} +{{- define "REDIS_CONNECTION_CMD" -}} +echo -n "redis://$(eval $REDIS_PASSWORD_CMD)${REDIS_HOST}:${REDIS_PORT}/${REDIS_DBNUM}" {{- end -}} {{/* @@ -139,7 +146,8 @@ When applicable, we use the secrets created by the postgres/redis charts (which {{- /* --------------------------- */ -}} {{- /* ---------- REDIS ---------- */ -}} {{- /* --------------------------- */ -}} -{{- if and (.Values.redis.enabled) (eq .Values.airflow.executor "CeleryExecutor") }} +{{- if eq .Values.airflow.executor "CeleryExecutor" }} +{{- if .Values.redis.enabled }} {{- if .Values.redis.existingSecret }} - name: REDIS_PASSWORD valueFrom: @@ -153,7 +161,7 @@ When applicable, we use the secrets created by the postgres/redis charts (which name: {{ include "airflow.redis.fullname" . }} key: redis-password {{- end }} -{{- else if (eq .Values.airflow.executor "CeleryExecutor") }} +{{- else }} {{- if .Values.externalRedis.passwordSecret }} - name: REDIS_PASSWORD valueFrom: @@ -165,6 +173,7 @@ When applicable, we use the secrets created by the postgres/redis charts (which value: "" {{- end }} {{- end }} +{{- end }} {{- /* ---------------------------- */ -}} {{- /* ---------- FLOWER ---------- */ -}} {{- /* ---------------------------- */ -}} diff --git a/stable/airflow/templates/config/configmap-env.yaml b/stable/airflow/templates/config/configmap-env.yaml index 5623fabd7e..3493748830 100644 --- a/stable/airflow/templates/config/configmap-env.yaml +++ b/stable/airflow/templates/config/configmap-env.yaml @@ -11,7 +11,9 @@ data: ## Force UTC timezone TZ: Etc/UTC + ## ---------------- ## Database + ## ---------------- {{- if .Values.postgresql.enabled }} DATABASE_HOST: "{{ include "airflow.postgresql.fullname" . }}" DATABASE_PORT: "5432" @@ -22,11 +24,24 @@ data: DATABASE_PORT: "{{ .Values.externalDatabase.port }}" DATABASE_USER: "{{ .Values.externalDatabase.user }}" DATABASE_DB: "{{ .Values.externalDatabase.database }}" - DATABASE_PROPERTIES: "{{.Values.externalDatabase.properties}}" + DATABASE_PROPERTIES: | + {{ .Values.externalDatabase.properties }} {{- end }} + # bash command which echos the URL encoded value of $DATABASE_PASSWORD + DATABASE_PASSWORD_CMD: | + echo ${DATABASE_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(encoded_pass)" + # bash command which echos the DB connection string in SQLAlchemy format + DATABASE_SQLALCHEMY_CMD: | + {{ include "DATABASE_SQLALCHEMY_CMD" . }} + # bash command which echos the DB connection string in Celery result_backend format + DATABASE_CELERY_CMD: | + {{ include "DATABASE_CELERY_CMD" . }} - {{- if (eq .Values.airflow.executor "CeleryExecutor") }} + + {{- if eq .Values.airflow.executor "CeleryExecutor" }} + ## ---------------- ## Redis + ## ---------------- {{- if .Values.redis.enabled }} REDIS_HOST: "{{ include "airflow.redis.fullname" . }}-master" REDIS_PORT: "6379" @@ -36,51 +51,70 @@ data: REDIS_PORT: "{{ .Values.externalRedis.port }}" REDIS_DBNUM: "{{ .Values.externalRedis.databaseNumber }}" {{- end }} - {{- if .Values.flower.enabled }} - ## Airflow (Flower) - AIRFLOW__CELERY__FLOWER_URL_PREFIX: "{{ .Values.flower.urlPrefix }}" - AIRFLOW__CELERY__FLOWER_PORT: "5555" + # a bash command which echos the URL encoded value of $REDIS_PASSWORD + # NOTE: if $REDIS_PASSWORD is non-empty, prints `:${REDIS_PASSWORD}@`, else `` + REDIS_PASSWORD_CMD: | + echo ${REDIS_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(f\":{encoded_pass}@\") if len(encoded_pass) > 0 else None" + # a bash command which echos the Redis connection string + REDIS_CONNECTION_CMD: | + {{ include "REDIS_CONNECTION_CMD" . }} {{- end }} - ## Airflow (Worker) + + ## ---------------- + ## Airflow + ## ---------------- + AIRFLOW__CORE__BASE_LOG_FOLDER: "{{ .Values.logs.path }}" + AIRFLOW__CORE__DAGS_FOLDER: "{{ .Values.dags.path }}" + AIRFLOW__CORE__DAG_PROCESSOR_MANAGER_LOG_LOCATION: "{{ .Values.logs.path }}/dag_processor_manager/dag_processor_manager.log" + AIRFLOW__CORE__DONOT_PICKLE: "{{ .Values.dags.doNotPickle }}" + AIRFLOW__CORE__ENABLE_XCOM_PICKLING: "false" # for forward compatibility with 2.0 + AIRFLOW__CORE__EXECUTOR: "{{ .Values.airflow.executor }}" + {{- if .Values.airflow.fernetKey }} + AIRFLOW__CORE__FERNET_KEY: "{{ .Values.airflow.fernetKey }}" + {{- end }} + AIRFLOW__CORE__SQL_ALCHEMY_CONN_CMD: | + bash -c 'eval "$DATABASE_SQLALCHEMY_CMD"' + {{- if .Values.web.serializeDAGs }} + AIRFLOW__CORE__STORE_SERIALIZED_DAGS: "{{ .Values.web.serializeDAGs }}" + {{- end }} + AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY: "{{ .Values.logs.path }}/scheduler" + AIRFLOW__WEBSERVER__BASE_URL: "{{ .Values.web.baseUrl }}" + AIRFLOW__WEBSERVER__WEB_SERVER_PORT: "8080" + + {{- if (eq .Values.airflow.executor "CeleryExecutor") }} + ## ---------------- + ## Airflow - CeleryExecutor + ## ---------------- + AIRFLOW__CELERY__BROKER_URL_CMD: | + bash -c 'eval "$REDIS_CONNECTION_CMD"' + {{- if .Values.flower.enabled }} + AIRFLOW__CELERY__FLOWER_PORT: "5555" + AIRFLOW__CELERY__FLOWER_URL_PREFIX: "{{ .Values.flower.urlPrefix }}" + {{- end }} + AIRFLOW__CELERY__RESULT_BACKEND_CMD: | + bash -c 'eval "$DATABASE_CELERY_CMD"' AIRFLOW__CELERY__WORKER_CONCURRENCY: "{{ .Values.workers.celery.instances }}" AIRFLOW__CELERY__WORKER_LOG_SERVER_PORT: "8793" {{- end }} {{- if (eq .Values.airflow.executor "KubernetesExecutor") }} - ## Airflow (Kubernetes) + ## ---------------- + ## Airflow - KubernetesExecutor + ## ---------------- + {{- if (not .Values.airflow.config.AIRFLOW__KUBERNETES__ENV_FROM_CONFIGMAP_REF) }} + AIRFLOW__KUBERNETES__ENV_FROM_CONFIGMAP_REF: "{{ include "airflow.fullname" . }}-env" + {{- end }} {{- if (not .Values.airflow.config.AIRFLOW__KUBERNETES__NAMESPACE) }} AIRFLOW__KUBERNETES__NAMESPACE: "{{ .Release.Namespace }}" {{- end }} {{- if (not .Values.airflow.config.AIRFLOW__KUBERNETES__WORKER_SERVICE_ACCOUNT_NAME) }} AIRFLOW__KUBERNETES__WORKER_SERVICE_ACCOUNT_NAME: "{{ include "airflow.serviceAccountName" . }}" {{- end }} - {{- if (not .Values.airflow.config.AIRFLOW__KUBERNETES__ENV_FROM_CONFIGMAP_REF) }} - AIRFLOW__KUBERNETES__ENV_FROM_CONFIGMAP_REF: "{{ include "airflow.fullname" . }}-env" - {{- end }} - {{- end }} - - ## Airflow (Logs) - AIRFLOW__CORE__BASE_LOG_FOLDER: "{{ .Values.logs.path }}" - AIRFLOW__CORE__DAG_PROCESSOR_MANAGER_LOG_LOCATION: "{{ .Values.logs.path }}/dag_processor_manager/dag_processor_manager.log" - AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY: "{{ .Values.logs.path }}/scheduler" - - ## Airflow - AIRFLOW__CORE__DAGS_FOLDER: "{{ .Values.dags.path }}" - AIRFLOW__CORE__DONOT_PICKLE: "{{ .Values.dags.doNotPickle }}" - AIRFLOW__CORE__ENABLE_XCOM_PICKLING: "false" # for forward compatibility with 2.0 - AIRFLOW__CORE__EXECUTOR: "{{ .Values.airflow.executor }}" - AIRFLOW__WEBSERVER__BASE_URL: "{{ .Values.web.baseUrl }}" - AIRFLOW__WEBSERVER__WEB_SERVER_PORT: "8080" - {{- if .Values.airflow.fernetKey }} - AIRFLOW__CORE__FERNET_KEY: "{{ .Values.airflow.fernetKey }}" - {{- end }} - {{- if .Values.web.serializeDAGs }} - AIRFLOW__CORE__STORE_SERIALIZED_DAGS: "{{ .Values.web.serializeDAGs }}" {{- end }} {{- if .Values.airflow.config }} - ## User Configs - {{- range $setting, $option := .Values.airflow.config }} - {{ $setting }}: "{{ $option }}" - {{- end }} + ## ---------------- + ## Airflow - User Configs + ## ---------------- + {{- toYaml .Values.airflow.config | nindent 2 }} {{- end }} diff --git a/stable/airflow/templates/config/configmap-scripts.yaml b/stable/airflow/templates/config/configmap-scripts.yaml index 5945d2767b..faccd55f41 100644 --- a/stable/airflow/templates/config/configmap-scripts.yaml +++ b/stable/airflow/templates/config/configmap-scripts.yaml @@ -25,8 +25,8 @@ data: #!/bin/bash -e echo "*** starting graceful worker shutdown" - # source the required environment variables - source /home/airflow/airflow_env.sh + # set the required environment variables + export AIRFLOW__CELERY__BROKER_URL=$(eval $AIRFLOW__CELERY__BROKER_URL_CMD) # prevent the worker accepting new tasks echo "*** preventing worker accepting new tasks" @@ -40,10 +40,7 @@ data: done preinit-db.sh: | #!/bin/bash - # export the required environment variables - export AIRFLOW__CORE__SQL_ALCHEMY_CONN="{{ include "airflow.connection.alchemy" . }}" - - echo "*** Waiting 10s for postgres" + echo "*** Waiting 10s for database" sleep 10 COUNT=0 diff --git a/stable/airflow/templates/config/configmap-variables-pools.yaml b/stable/airflow/templates/config/configmap-variables-pools.yaml index 05aad5520e..1f7c11793b 100644 --- a/stable/airflow/templates/config/configmap-variables-pools.yaml +++ b/stable/airflow/templates/config/configmap-variables-pools.yaml @@ -2,12 +2,12 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "airflow.fullname" . }}-variables-pools - labels: - app: {{ include "airflow.labels.app" . }} - chart: {{ include "airflow.labels.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} + name: {{ include "airflow.fullname" . }}-variables-pools + labels: + app: {{ include "airflow.labels.app" . }} + chart: {{ include "airflow.labels.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} data: {{- if .Values.scheduler.variables }} variables.json: | diff --git a/stable/airflow/templates/flower/flower-deployment.yaml b/stable/airflow/templates/flower/flower-deployment.yaml index 5584be16f4..d17641195b 100644 --- a/stable/airflow/templates/flower/flower-deployment.yaml +++ b/stable/airflow/templates/flower/flower-deployment.yaml @@ -107,13 +107,7 @@ spec: && sleep {{ .Values.flower.initialStartupDelay }} \ {{- end }} && mkdir -p /home/airflow/.local/bin \ - && echo 'export PATH="/home/airflow/.local/bin:$PATH"' >> /home/airflow/airflow_env.sh \ - && echo 'export AIRFLOW__CORE__SQL_ALCHEMY_CONN="{{ include "airflow.connection.alchemy" . }}"' >> /home/airflow/airflow_env.sh \ - && echo 'export AIRFLOW__CELERY__RESULT_BACKEND="{{ include "airflow.connection.celery.backend" . }}"' >> /home/airflow/airflow_env.sh \ - && echo 'export AIRFLOW__CELERY__BROKER_URL="{{ include "airflow.connection.celery.broker" . }}"' >> /home/airflow/airflow_env.sh \ - && echo 'source /home/airflow/airflow_env.sh' >> /home/airflow/.profile \ - && echo 'source /home/airflow/airflow_env.sh' >> /home/airflow/.bashrc \ - && source /home/airflow/airflow_env.sh \ + && export PATH="/home/airflow/.local/bin:$PATH" \ && echo "*** running flower..." \ && exec airflow flower livenessProbe: diff --git a/stable/airflow/templates/flower/flower-ingress.yaml b/stable/airflow/templates/flower/flower-ingress.yaml index 2dd3a37f1c..679a52fe16 100644 --- a/stable/airflow/templates/flower/flower-ingress.yaml +++ b/stable/airflow/templates/flower/flower-ingress.yaml @@ -3,16 +3,16 @@ apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: {{ include "airflow.fullname" . }}-flower + {{- if .Values.ingress.flower.annotations }} + annotations: + {{- toYaml .Values.ingress.flower.annotations | nindent 4 }} + {{- end }} labels: app: {{ include "airflow.labels.app" . }} component: flower - chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - annotations: - {{- range $key, $value := .Values.ingress.flower.annotations }} - {{ $key }}: {{ $value | quote }} - {{- end }} + chart: {{ include "airflow.labels.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} spec: {{- if .Values.ingress.flower.tls.enabled }} tls: diff --git a/stable/airflow/templates/flower/flower-service.yaml b/stable/airflow/templates/flower/flower-service.yaml index 99e5f3d6ac..69f82592e9 100644 --- a/stable/airflow/templates/flower/flower-service.yaml +++ b/stable/airflow/templates/flower/flower-service.yaml @@ -3,16 +3,16 @@ apiVersion: v1 kind: Service metadata: name: {{ include "airflow.fullname" . }}-flower + {{- if .Values.flower.service.annotations }} + annotations: + {{- toYaml .Values.flower.service.annotations | nindent 4 }} + {{- end }} labels: app: {{ include "airflow.labels.app" . }} component: flower chart: {{ include "airflow.labels.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} - annotations: - {{- if .Values.flower.service.annotations }} - {{- toYaml .Values.flower.service.annotations | nindent 4 }} - {{- end }} spec: type: {{ .Values.flower.service.type }} selector: diff --git a/stable/airflow/templates/rbac/airflow-serviceaccount.yaml b/stable/airflow/templates/rbac/airflow-serviceaccount.yaml index 758b6bcddc..9ec7056cdd 100644 --- a/stable/airflow/templates/rbac/airflow-serviceaccount.yaml +++ b/stable/airflow/templates/rbac/airflow-serviceaccount.yaml @@ -3,13 +3,13 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ include "airflow.serviceAccountName" . }} + {{- if .Values.serviceAccount.annotations }} + annotations: + {{- toYaml .Values.serviceAccount.annotations | nindent 4 }} + {{- end }} labels: app: {{ include "airflow.labels.app" . }} chart: {{ include "airflow.labels.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} - {{- if .Values.serviceAccount.annotations }} - annotations: - {{- toYaml .Values.serviceAccount.annotations | nindent 4 }} - {{- end }} {{- end }} diff --git a/stable/airflow/templates/scheduler/scheduler-deployment.yaml b/stable/airflow/templates/scheduler/scheduler-deployment.yaml index ac400c2a54..cd8f084db8 100644 --- a/stable/airflow/templates/scheduler/scheduler-deployment.yaml +++ b/stable/airflow/templates/scheduler/scheduler-deployment.yaml @@ -226,15 +226,7 @@ spec: && sleep {{ .Values.scheduler.initialStartupDelay }} \ {{- end }} && mkdir -p /home/airflow/.local/bin \ - && echo 'export PATH="/home/airflow/.local/bin:$PATH"' >> /home/airflow/airflow_env.sh \ - && echo 'export AIRFLOW__CORE__SQL_ALCHEMY_CONN="{{ include "airflow.connection.alchemy" . }}"' >> /home/airflow/airflow_env.sh \ - {{- if (eq .Values.airflow.executor "CeleryExecutor") }} - && echo 'export AIRFLOW__CELERY__RESULT_BACKEND="{{ include "airflow.connection.celery.backend" . }}"' >> /home/airflow/airflow_env.sh \ - && echo 'export AIRFLOW__CELERY__BROKER_URL="{{ include "airflow.connection.celery.broker" . }}"' >> /home/airflow/airflow_env.sh \ - {{- end }} - && echo 'source /home/airflow/airflow_env.sh' >> /home/airflow/.profile \ - && echo 'source /home/airflow/airflow_env.sh' >> /home/airflow/.bashrc \ - && source /home/airflow/airflow_env.sh \ + && export PATH="/home/airflow/.local/bin:$PATH" \ {{- if .Values.dags.installRequirements }} && echo "*** installing requirements..." \ && /home/airflow/scripts/install-requirements.sh \ diff --git a/stable/airflow/templates/webserver/webserver-deployment.yaml b/stable/airflow/templates/webserver/webserver-deployment.yaml index a9ff40b912..9d457dce64 100644 --- a/stable/airflow/templates/webserver/webserver-deployment.yaml +++ b/stable/airflow/templates/webserver/webserver-deployment.yaml @@ -205,15 +205,7 @@ spec: && sleep {{ .Values.web.initialStartupDelay }} \ {{- end }} && mkdir -p /home/airflow/.local/bin \ - && echo 'export PATH="/home/airflow/.local/bin:$PATH"' >> /home/airflow/airflow_env.sh \ - && echo 'export AIRFLOW__CORE__SQL_ALCHEMY_CONN="{{ include "airflow.connection.alchemy" . }}"' >> /home/airflow/airflow_env.sh \ - {{- if (eq .Values.airflow.executor "CeleryExecutor") }} - && echo 'export AIRFLOW__CELERY__RESULT_BACKEND="{{ include "airflow.connection.celery.backend" . }}"' >> /home/airflow/airflow_env.sh \ - && echo 'export AIRFLOW__CELERY__BROKER_URL="{{ include "airflow.connection.celery.broker" . }}"' >> /home/airflow/airflow_env.sh \ - {{- end }} - && echo 'source /home/airflow/airflow_env.sh' >> /home/airflow/.profile \ - && echo 'source /home/airflow/airflow_env.sh' >> /home/airflow/.bashrc \ - && source /home/airflow/airflow_env.sh \ + && export PATH="/home/airflow/.local/bin:$PATH" \ {{- if .Values.dags.installRequirements }} && echo "*** installing requirements..." \ && /home/airflow/scripts/install-requirements.sh \ diff --git a/stable/airflow/templates/webserver/webserver-ingress.yaml b/stable/airflow/templates/webserver/webserver-ingress.yaml index d570c72371..d1a7c7ef81 100644 --- a/stable/airflow/templates/webserver/webserver-ingress.yaml +++ b/stable/airflow/templates/webserver/webserver-ingress.yaml @@ -3,16 +3,16 @@ apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: {{ include "airflow.fullname" . }}-web + {{- if .Values.ingress.web.annotations }} + annotations: + {{- toYaml .Values.ingress.web.annotations | nindent 4 }} + {{- end }} labels: app: {{ include "airflow.labels.app" . }} component: web - chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - annotations: - {{ range $key, $value := .Values.ingress.web.annotations }} - {{ $key }}: {{ $value | quote }} - {{- end }} + chart: {{ include "airflow.labels.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} spec: {{- if .Values.ingress.web.tls.enabled }} tls: diff --git a/stable/airflow/templates/monitoring/prometheus-rule.yaml b/stable/airflow/templates/webserver/webserver-prometheus-rule.yaml similarity index 96% rename from stable/airflow/templates/monitoring/prometheus-rule.yaml rename to stable/airflow/templates/webserver/webserver-prometheus-rule.yaml index 52dcbd7d65..87167dc6a4 100644 --- a/stable/airflow/templates/monitoring/prometheus-rule.yaml +++ b/stable/airflow/templates/webserver/webserver-prometheus-rule.yaml @@ -5,9 +5,10 @@ metadata: name: {{ include "airflow.fullname" . }} labels: app: {{ include "airflow.labels.app" . }} + component: web chart: {{ include "airflow.labels.chart" . }} - heritage: {{ .Release.Service }} release: {{ .Release.Name }} + heritage: {{ .Release.Service }} {{- if .Values.prometheusRule.additionalLabels }} {{- toYaml .Values.prometheusRule.additionalLabels | nindent 4 }} {{- end }} diff --git a/stable/airflow/templates/monitoring/service-monitor.yaml b/stable/airflow/templates/webserver/webserver-service-monitor.yaml similarity index 82% rename from stable/airflow/templates/monitoring/service-monitor.yaml rename to stable/airflow/templates/webserver/webserver-service-monitor.yaml index c40163212a..d2fc660d5c 100644 --- a/stable/airflow/templates/monitoring/service-monitor.yaml +++ b/stable/airflow/templates/webserver/webserver-service-monitor.yaml @@ -5,12 +5,12 @@ metadata: name: {{ include "airflow.fullname" . }} labels: app: {{ include "airflow.labels.app" . }} - component: worker + component: web chart: {{ include "airflow.labels.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} - {{- range $key, $value := .Values.serviceMonitor.selector }} - {{ $key }}: {{ $value | quote }} + {{- if .Values.serviceMonitor.selector }} + {{- toYaml .Values.serviceMonitor.selector | nindent 4 }} {{- end }} spec: selector: diff --git a/stable/airflow/templates/webserver/webserver-service.yaml b/stable/airflow/templates/webserver/webserver-service.yaml index 8306a1e4e9..f922c30b2c 100644 --- a/stable/airflow/templates/webserver/webserver-service.yaml +++ b/stable/airflow/templates/webserver/webserver-service.yaml @@ -2,16 +2,16 @@ apiVersion: v1 kind: Service metadata: name: {{ include "airflow.fullname" . }}-web + {{- if .Values.web.service.annotations }} + annotations: + {{- toYaml .Values.web.service.annotations | nindent 4 }} + {{- end }} labels: app: {{ include "airflow.labels.app" . }} component: web chart: {{ include "airflow.labels.chart" . }} release: {{ .Release.Name }} heritage: {{ .Release.Service }} - annotations: - {{- if .Values.web.service.annotations }} - {{- toYaml .Values.web.service.annotations | nindent 4 }} - {{- end }} spec: type: {{ .Values.web.service.type }} selector: diff --git a/stable/airflow/templates/worker/worker-statefulset.yaml b/stable/airflow/templates/worker/worker-statefulset.yaml index 45064ecb0f..9c8e2615b5 100644 --- a/stable/airflow/templates/worker/worker-statefulset.yaml +++ b/stable/airflow/templates/worker/worker-statefulset.yaml @@ -215,15 +215,7 @@ spec: && sleep {{ .Values.workers.initialStartupDelay }} \ {{- end }} && mkdir -p /home/airflow/.local/bin \ - && echo 'export PATH="/home/airflow/.local/bin:$PATH"' >> /home/airflow/airflow_env.sh \ - && echo 'export AIRFLOW__CORE__SQL_ALCHEMY_CONN="{{ include "airflow.connection.alchemy" . }}"' >> /home/airflow/airflow_env.sh \ - {{- if (eq .Values.airflow.executor "CeleryExecutor") }} - && echo 'export AIRFLOW__CELERY__RESULT_BACKEND="{{ include "airflow.connection.celery.backend" . }}"' >> /home/airflow/airflow_env.sh \ - && echo 'export AIRFLOW__CELERY__BROKER_URL="{{ include "airflow.connection.celery.broker" . }}"' >> /home/airflow/airflow_env.sh \ - {{- end }} - && echo 'source /home/airflow/airflow_env.sh' >> /home/airflow/.profile \ - && echo 'source /home/airflow/airflow_env.sh' >> /home/airflow/.bashrc \ - && source /home/airflow/airflow_env.sh \ + && export PATH="/home/airflow/.local/bin:$PATH" \ {{- if .Values.dags.installRequirements }} && echo "*** installing requirements..." \ && /home/airflow/scripts/install-requirements.sh \ @@ -232,7 +224,7 @@ spec: && echo "*** installing global extra pip packages..." \ && pip install --user {{ range .Values.airflow.extraPipPackages }} {{ . | quote }} {{ end }} \ {{- end }} - && echo "*** running scheduler..." \ + && echo "*** running worker..." \ && exec airflow worker ports: - name: wlog