Added airflow variables support (#11990)

Signed-off-by: Stijn De Haes <stijndehaes@gmail.com>

stable/airflow/Chart.yaml

@@ -1,6 +1,6 @@
 description: Airflow is a platform to programmatically author, schedule and monitor workflows
 name: airflow
-version: 2.1.0
+version: 2.2.0
 appVersion: 1.10.0
 icon: https://airflow.apache.org/_images/pin_large.png
 home: https://airflow.apache.org/

stable/airflow/README.md

@@ -115,6 +115,17 @@ airflow:
 
 Note: As connections may require to include sensitive data - the resulting script is stored encrypted in a kubernetes secret and mounted into the airflow scheduler container. It is probably wise not to put connection data in the default values.yaml and instead create an encrypted my-secret-values.yaml. this way it can be decrypted before the installation and passed to helm with -f <my-secret-values.yaml>
 
+#### Airflow variables
+
+Variables are a generic way to store and retrieve arbitrary content or settings as a simple key value store within Airflow.
+These variables will be automatically imported by the scheduler when it starts up.
+
+Example:
+```yaml
+airflow:
+  variables: '{ "environment": "dev" }'
+```
+
 ### Worker Statefulset
 
 Celery workers uses StatefulSet.

stable/airflow/examples/minikube-values.yaml

@@ -10,6 +10,8 @@ airflow:
     AIRFLOW__CORE__LOGGING_LEVEL: DEBUG
     AIRFLOW__CORE__LOAD_EXAMPLES: True
 
+  variables: '{ "environment": "dev" }'
+
 workers:
   replicas: 1
   celery:

stable/airflow/templates/configmap-variables.yaml

@@ -0,0 +1,14 @@
+{{- if .Values.airflow.variables }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+    name: {{ template "airflow.fullname" . }}-variables
+    labels:
+        app: {{ template "airflow.name" . }}
+        chart: {{ template "airflow.chart" . }}
+        release: {{ .Release.Name }}
+        heritage: {{ .Release.Service }}
+data:
+    variables.json: |
+        {{ .Values.airflow.variables }}
+{{- end }}

stable/airflow/templates/deployments-scheduler.yaml

@@ -89,6 +89,10 @@ spec:
             - name: connections
               mountPath: /usr/local/connections
           {{- end}}
+          {{- if .Values.airflow.variables }}
+            - name: variables
+              mountPath: /usr/local/variables/
+          {{- end}}
           {{- range .Values.airflow.extraConfigmapMounts }}
             - name: {{ .name }}
               mountPath: {{ .mountPath }}
@@ -113,6 +117,10 @@ spec:
             {{- if .Values.airflow.connections }}
               echo "adding connections" &&
               /usr/local/connections/add-connections.sh &&
+            {{- end }}
+            {{- if .Values.airflow.variables }}
+              echo "adding variables" &&
+              airflow variables -i /usr/local/variables/variables.json &&
             {{- end }}
               echo "executing scheduler" &&
               airflow scheduler -n {{ .Values.airflow.schedulerNumRuns }}
@@ -127,6 +135,10 @@ spec:
             {{- if .Values.airflow.connections }}
               echo "adding connections" &&
               /usr/local/connections/add-connections.sh &&
+            {{- end }}
+            {{- if .Values.airflow.variables }}
+              echo "adding variables" &&
+              airflow variables -i /usr/local/variables/variables.json &&
             {{- end }}
               echo "executing scheduler" &&
               airflow scheduler -n {{ .Values.airflow.schedulerNumRuns }}
@@ -169,6 +181,12 @@ spec:
             secretName: {{ template "airflow.fullname" . }}-connections
             defaultMode: 0755
         {{- end }}
+        {{- if .Values.airflow.variables }}
+        - name: variables
+          configMap:
+            name: {{ template "airflow.fullname" . }}-variables
+            defaultMode: 0755
+        {{- end }}
         {{- range .Values.airflow.extraConfigmapMounts }}
         - name: {{ .name }}
           configMap:

stable/airflow/values.yaml

@@ -132,6 +132,12 @@ airflow:
   ##     type: aws
   ##     extra: '{"aws_access_key_id": "**********", "aws_secret_access_key": "***", "region_name":"eu-central-1"}'
   connections: {}
+
+  ## Add airflow variables
+  ## This should be a json string with your variables in it
+  ## Examples:
+  ##   variables: '{ "environment": "dev" }'
+  variables: {}
   ##
   ## Annotations for the Scheduler, Worker and Web pods
   podAnnotations: {}