[stable/airflow] Add container and volume extension points (#11775)

* [stable/airflow] Add extraContainers/VolumeMounts/Volumes

Signed-off-by: Steve Larkin <steve.larkin@sony.com>

* [stable/airflow] Step version

Signed-off-by: Steve Larkin <steve.larkin@sony.com>

stable/airflow/Chart.yaml

@@ -1,6 +1,6 @@
 description: Airflow is a platform to programmatically author, schedule and monitor workflows
 name: airflow
-version: 2.0.2
+version: 2.1.0
 appVersion: 1.10.0
 icon: https://airflow.apache.org/_images/pin_large.png
 home: https://airflow.apache.org/

stable/airflow/README.md

@@ -277,6 +277,9 @@ The following table lists the configurable parameters of the Airflow chart and t
 | `airflow.secretsMapping`                 | override any environment variable with a secret         |                           |
 | `airflow.extraConfigmapMounts`           | Additional configMap volume mounts on the airflow pods. | `[]`                      |
 | `airflow.podAnnotations`                 | annotations for scheduler, worker and web pods          | `{}`                      |
+| `airflow.extraContainers`                | additional containers to run in the scheduler, worker & web pods | `[]`             |
+| `airflow.extraVolumeMounts`              | additional volumeMounts to the main container in scheduler, worker & web pods | `[]`|
+| `airflow.extraVolumes`                   | additional volumes for the scheduler, worker & web pods | `[]`                      |
 | `workers.enabled`                        | enable workers                                          | `true`                    |
 | `workers.replicas`                       | number of workers pods to launch                        | `1`                       |
 | `workers.resources`                      | custom resource configuration for worker pod            | `{}`                      |

stable/airflow/templates/deployments-scheduler.yaml

@@ -94,6 +94,9 @@ spec:
               mountPath: {{ .mountPath }}
               readOnly: {{ .readOnly }}
           {{- end }}
+{{- if .Values.airflow.extraVolumeMounts }}
+{{ toYaml .Values.airflow.extraVolumeMounts | indent 12 }}
+{{- end }}
           args:
             - "bash"
             - "-c"
@@ -128,6 +131,9 @@ spec:
               echo "executing scheduler" &&
               airflow scheduler -n {{ .Values.airflow.schedulerNumRuns }}
           {{- end }}
+{{- if .Values.airflow.extraContainers }}
+{{ toYaml .Values.airflow.extraContainers | indent 8 }}
+{{- end }}
       volumes:
         - name: scripts
           configMap:
@@ -168,3 +174,6 @@ spec:
           configMap:
             name: {{ .configMap }}
         {{- end }}
+{{- if .Values.airflow.extraVolumes }}
+{{ toYaml .Values.airflow.extraVolumes | indent 8 }}
+{{- end }}

stable/airflow/templates/deployments-web.yaml

@@ -93,6 +93,9 @@ spec:
               mountPath: {{ .mountPath }}
               readOnly: {{ .readOnly }}
           {{- end }}
+{{- if .Values.airflow.extraVolumeMounts }}
+{{ toYaml .Values.airflow.extraVolumeMounts | indent 12 }}
+{{- end }}
           args:
             - "bash"
             - "-c"
@@ -125,6 +128,9 @@ spec:
             timeoutSeconds: 1
             successThreshold: 1
             failureThreshold: 5
+{{- if .Values.airflow.extraContainers }}
+{{ toYaml .Values.airflow.extraContainers | indent 8 }}
+{{- end }}
       volumes:
         - name: scripts
           configMap:
@@ -158,4 +164,7 @@ spec:
         - name: {{ .name }}
           configMap:
             name: {{ .configMap }}
-        {{- end }}
+        {{- end }}
+{{- if .Values.airflow.extraVolumes }}
+{{ toYaml .Values.airflow.extraVolumes | indent 8 }}
+{{- end }}

stable/airflow/templates/statefulsets-workers.yaml

@@ -101,6 +101,9 @@ spec:
               mountPath: {{ .mountPath }}
               readOnly: {{ .readOnly }}
           {{- end }}
+{{- if .Values.airflow.extraVolumeMounts }}
+{{ toYaml .Values.airflow.extraVolumeMounts | indent 12 }}
+{{- end }}
           args:
             - "bash"
             - "-c"
@@ -129,6 +132,9 @@ spec:
               protocol: TCP
           resources:
 {{ toYaml .Values.workers.resources | indent 12 }}
+{{- if .Values.airflow.extraContainers }}
+{{ toYaml .Values.airflow.extraContainers | indent 8 }}
+{{- end }}
       volumes:
         - name: scripts
           configMap:
@@ -163,4 +169,7 @@ spec:
           configMap:
             name: {{ .configMap }}
         {{- end }}
+{{- if .Values.airflow.extraVolumes }}
+{{ toYaml .Values.airflow.extraVolumes | indent 8 }}
+{{- end }}
 {{- end }}

stable/airflow/values.yaml

@@ -137,6 +137,22 @@ airflow:
   podAnnotations: {}
     ## Example:
     ## iam.amazonaws.com/role: airflow-Role
+  extraContainers: []
+  ## Additional containers to run alongside the Scheduler, Worker and Web pods
+  ## This could, for example, be used to run a sidecar that syncs DAGs from object storage.
+  # - name: s3-sync
+  #   image: my-user/s3sync:latest
+  #   volumeMounts:
+  #   - name: synchronised-dags
+  #     mountPath: /dags
+  extraVolumeMounts: []
+  ## Additional volumeMounts to the main containers in the Scheduler, Worker and Web pods.
+  # - name: synchronised-dags
+  #   mountPath: /usr/local/airflow/dags
+  extraVolumes: []
+  ## Additional volumes for the Scheduler, Worker and Web pods.
+  # - name: synchronised-dags
+  #   emptyDir: {}
 ##
 ## Workers configuration
 workers: