Airflow updates (#21368)

- Install variables before running initdb; custom connections
  may require variables to be defined.

- Install connections before running initdb; models may
  depend on custom db connections.

- Fix URI-style connections: `--conn_type` and `--conn_uri` are
  not compatible flags.

- Fix the default "variables:" value in values.yaml: this should
  be a string with a JSON object inside, not a yaml map, and
  this caused confusing warning output from coalesce.go.

Signed-off-by: Nathan J. Mehl <n@oden.io>

stable/airflow/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 description: Airflow is a platform to programmatically author, schedule and monitor workflows
 name: airflow
-version: 6.3.0
+version: 6.3.1
 appVersion: 1.10.4
 icon: https://airflow.apache.org/_images/pin_large.png
 home: https://airflow.apache.org/

stable/airflow/templates/deployments-scheduler.yaml

@@ -186,53 +186,53 @@ spec:
             - "-c"
           {{- if and ( .Values.dags.initContainer.enabled ) ( .Values.dags.initContainer.installRequirements ) }}
             - >
-              echo 'waiting 10s...' &&
+              echo "*** waiting 10s..." &&
               sleep 10 &&
-              echo 'installing requirements...' &&
+              echo "*** installing requirements..." &&
               mkdir -p /usr/local/airflow/.local/bin &&
               export PATH=/usr/local/airflow/.local/bin:$PATH &&
               /usr/local/scripts/install-requirements.sh &&
+              {{- if .Values.airflow.variables }}
+                echo "*** adding variables" &&
+                airflow variables -i /usr/local/variables-pools/variables.json &&
+              {{- end }}
+              {{- if .Values.airflow.connections }}
+                echo "*** adding connections" &&
+                /usr/local/connections/add-connections.sh &&
+              {{- end }}
               {{- if .Values.airflow.initdb }}
-              echo "executing initdb" &&
+              echo "*** executing initdb" &&
               airflow initdb &&
               {{- end }}
-            {{- if .Values.airflow.connections }}
-              echo "adding connections" &&
-              /usr/local/connections/add-connections.sh &&
-            {{- end }}
-            {{- if .Values.airflow.variables }}
-              echo "adding variables" &&
-              airflow variables -i /usr/local/variables-pools/variables.json &&
-            {{- end }}
             {{- if .Values.airflow.pools }}
-              echo "adding pools" &&
+              echo "*** adding pools" &&
               airflow pool -i /usr/local/variables-pools/pools.json &&
             {{- end }}
-              echo "executing scheduler" &&
+              echo "*** executing scheduler" &&
               airflow scheduler -n {{ .Values.airflow.schedulerNumRuns }}
           {{- else }}
             - >
-              echo 'waiting 10s...' &&
+              echo "*** waiting 10s..." &&
               sleep 10 &&
               mkdir -p /usr/local/airflow/.local/bin &&
               export PATH=/usr/local/airflow/.local/bin:$PATH &&
+              {{- if .Values.airflow.variables }}
+                echo "*** adding variables" &&
+                airflow variables -i /usr/local/variables-pools/variables.json &&
+              {{- end }}
+              {{- if .Values.airflow.connections }}
+                echo "*** adding connections" &&
+                /usr/local/connections/add-connections.sh &&
+              {{- end }}
               {{- if .Values.airflow.initdb }}
-              echo "executing initdb" &&
+              echo "*** executing initdb" &&
               airflow initdb &&
               {{- end }}
-            {{- if .Values.airflow.connections }}
-              echo "adding connections" &&
-              /usr/local/connections/add-connections.sh &&
-            {{- end }}
-            {{- if .Values.airflow.variables }}
-              echo "adding variables" &&
-              airflow variables -i /usr/local/variables-pools/variables.json &&
-            {{- end }}
             {{- if .Values.airflow.pools }}
-              echo "adding pools" &&
+              echo "*** adding pools" &&
               airflow pool -i /usr/local/variables-pools/pools.json &&
             {{- end }}
-              echo "executing scheduler" &&
+              echo "*** executing scheduler" &&
               airflow scheduler -n {{ .Values.airflow.schedulerNumRuns }}
           {{- end }}
 {{- if .Values.airflow.extraContainers }}

stable/airflow/templates/secret-connections.yaml

@@ -1,7 +1,8 @@
 {{- define "connections.script" }}
   #!/bin/sh -e
   {{- range .Values.airflow.connections }}
-  airflow connections --add --conn_type {{ .type }} --conn_id {{ .id }}
+  airflow connections --add --conn_id {{ .id }}
+  {{- if .type }} --conn_type {{ .type | quote }} {{ end -}}
   {{- if .uri }} --conn_uri {{ .uri | quote }} {{ end -}}
   {{- if .host }} --conn_host {{ .host | quote }} {{ end -}}
   {{- if .login }} --conn_login {{ .login | quote }} {{ end -}}

stable/airflow/values.yaml

@@ -129,7 +129,7 @@ airflow:
   ## This should be a json string with your variables in it
   ## Examples:
   ##   variables: '{ "environment": "dev" }'
-  variables: {}
+  variables: '{}'
 
   ## Add airflow ppols
   ## This should be a json string with your pools in it