From b33e4ce9b9c577b28989595a87f021ccb2ed3b8b Mon Sep 17 00:00:00 2001 From: memory Date: Tue, 10 Mar 2020 17:45:35 -0400 Subject: [PATCH] Airflow updates (#21368) - Install variables before running initdb; custom connections may require variables to be defined. - Install connections before running initdb; models may depend on custom db connections. - Fix URI-style connections: `--conn_type` and `--conn_uri` are not compatible flags. - Fix the default "variables:" value in values.yaml: this should be a string with a JSON object inside, not a yaml map, and this caused confusing warning output from coalesce.go. Signed-off-by: Nathan J. Mehl --- stable/airflow/Chart.yaml | 2 +- .../templates/deployments-scheduler.yaml | 50 +++++++++---------- .../airflow/templates/secret-connections.yaml | 3 +- stable/airflow/values.yaml | 2 +- 4 files changed, 29 insertions(+), 28 deletions(-) diff --git a/stable/airflow/Chart.yaml b/stable/airflow/Chart.yaml index 510e0a225d..966fc48e40 100644 --- a/stable/airflow/Chart.yaml +++ b/stable/airflow/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Airflow is a platform to programmatically author, schedule and monitor workflows name: airflow -version: 6.3.0 +version: 6.3.1 appVersion: 1.10.4 icon: https://airflow.apache.org/_images/pin_large.png home: https://airflow.apache.org/ diff --git a/stable/airflow/templates/deployments-scheduler.yaml b/stable/airflow/templates/deployments-scheduler.yaml index 58ce296ffb..a7b665c8b8 100644 --- a/stable/airflow/templates/deployments-scheduler.yaml +++ b/stable/airflow/templates/deployments-scheduler.yaml @@ -186,53 +186,53 @@ spec: - "-c" {{- if and ( .Values.dags.initContainer.enabled ) ( .Values.dags.initContainer.installRequirements ) }} - > - echo 'waiting 10s...' && + echo "*** waiting 10s..." && sleep 10 && - echo 'installing requirements...' && + echo "*** installing requirements..." && mkdir -p /usr/local/airflow/.local/bin && export PATH=/usr/local/airflow/.local/bin:$PATH && /usr/local/scripts/install-requirements.sh && + {{- if .Values.airflow.variables }} + echo "*** adding variables" && + airflow variables -i /usr/local/variables-pools/variables.json && + {{- end }} + {{- if .Values.airflow.connections }} + echo "*** adding connections" && + /usr/local/connections/add-connections.sh && + {{- end }} {{- if .Values.airflow.initdb }} - echo "executing initdb" && + echo "*** executing initdb" && airflow initdb && {{- end }} - {{- if .Values.airflow.connections }} - echo "adding connections" && - /usr/local/connections/add-connections.sh && - {{- end }} - {{- if .Values.airflow.variables }} - echo "adding variables" && - airflow variables -i /usr/local/variables-pools/variables.json && - {{- end }} {{- if .Values.airflow.pools }} - echo "adding pools" && + echo "*** adding pools" && airflow pool -i /usr/local/variables-pools/pools.json && {{- end }} - echo "executing scheduler" && + echo "*** executing scheduler" && airflow scheduler -n {{ .Values.airflow.schedulerNumRuns }} {{- else }} - > - echo 'waiting 10s...' && + echo "*** waiting 10s..." && sleep 10 && mkdir -p /usr/local/airflow/.local/bin && export PATH=/usr/local/airflow/.local/bin:$PATH && + {{- if .Values.airflow.variables }} + echo "*** adding variables" && + airflow variables -i /usr/local/variables-pools/variables.json && + {{- end }} + {{- if .Values.airflow.connections }} + echo "*** adding connections" && + /usr/local/connections/add-connections.sh && + {{- end }} {{- if .Values.airflow.initdb }} - echo "executing initdb" && + echo "*** executing initdb" && airflow initdb && {{- end }} - {{- if .Values.airflow.connections }} - echo "adding connections" && - /usr/local/connections/add-connections.sh && - {{- end }} - {{- if .Values.airflow.variables }} - echo "adding variables" && - airflow variables -i /usr/local/variables-pools/variables.json && - {{- end }} {{- if .Values.airflow.pools }} - echo "adding pools" && + echo "*** adding pools" && airflow pool -i /usr/local/variables-pools/pools.json && {{- end }} - echo "executing scheduler" && + echo "*** executing scheduler" && airflow scheduler -n {{ .Values.airflow.schedulerNumRuns }} {{- end }} {{- if .Values.airflow.extraContainers }} diff --git a/stable/airflow/templates/secret-connections.yaml b/stable/airflow/templates/secret-connections.yaml index 34c35eccc4..6fd14a455e 100644 --- a/stable/airflow/templates/secret-connections.yaml +++ b/stable/airflow/templates/secret-connections.yaml @@ -1,7 +1,8 @@ {{- define "connections.script" }} #!/bin/sh -e {{- range .Values.airflow.connections }} - airflow connections --add --conn_type {{ .type }} --conn_id {{ .id }} + airflow connections --add --conn_id {{ .id }} + {{- if .type }} --conn_type {{ .type | quote }} {{ end -}} {{- if .uri }} --conn_uri {{ .uri | quote }} {{ end -}} {{- if .host }} --conn_host {{ .host | quote }} {{ end -}} {{- if .login }} --conn_login {{ .login | quote }} {{ end -}} diff --git a/stable/airflow/values.yaml b/stable/airflow/values.yaml index c7d9da9c73..6e55ec0469 100644 --- a/stable/airflow/values.yaml +++ b/stable/airflow/values.yaml @@ -129,7 +129,7 @@ airflow: ## This should be a json string with your variables in it ## Examples: ## variables: '{ "environment": "dev" }' - variables: {} + variables: '{}' ## Add airflow ppols ## This should be a json string with your pools in it