Adding a git pull secret for dags in Airflow (#10987)

* Adding a git pull secret for dags in Airflow

 * Added `dags.git.secret` key with no default
 * Added `/keys` secret mount to the scheduler, web, and worker defs
 * Added extra copy step to the git clone configmap when key is active
 * Preserved default behavior otherwise
 * Updated documentation to match
 * Updated version number

Signed-off-by: Ken Wronkiewicz <kwronkiewicz@salesforce.com>

* v0.14.1 -> v0.15.0 because new feature.

Signed-off-by: Ken Wronkiewicz <kwronkiewicz@salesforce.com>

stable/airflow/Chart.yaml

@@ -1,6 +1,6 @@
 description: Airflow is a platform to programmatically author, schedule and monitor workflows
 name: airflow
-version: 0.14.0
+version: 0.15.0
 appVersion: 1.10.0
 icon: https://airflow.apache.org/_images/pin_large.png
 home: https://airflow.apache.org/

stable/airflow/README.md

@@ -203,6 +203,10 @@ Python dependencies installed.
 
 This is the easiest way of deploying your DAGs to Airflow.
 
+If you are using a private Git repo, you can set `dags.gitSecret` to the name of a secret you created containing private keys and a `known_hosts` file.
+
+For example, this will create a secret named `my-git-secret` from your ed25519 key and known_hosts file stored in your home directory:  `kubectl create secret generic my-git-secret --from-file=id_ed25519=~/.ssh/id_ed25519 --from-file=known_hosts=~/.ssh/known_hosts --from-file=id_id_ed25519.pub=~/.ssh/id_ed25519.pub`
+
 ### Embedded DAGs
 
 If you want more control on the way you deploy your DAGs, you can use embedded DAGs, where DAGs
@@ -288,6 +292,7 @@ The following table lists the configurable parameters of the Airflow chart and t
 | `dags.initContainer.installRequirements` | auto install requirements.txt deps                      | `true`                    |
 | `dags.git.url`                           | url to clone the git repository                         | nil                       |
 | `dags.git.ref`                           | branch name, tag or sha1 to reset to                    | `master`                  |
+| `dags.git.secret`                        | name of a secret containing an ssh deploy key           | nil                       |
 | `logs.path`                              | mount path for logs persistent volume                   | `/usr/local/airflow/logs` |
 | `rbac.create`                            | create RBAC resources                                   | `true`                    |
 | `serviceAccount.create`                  | create a service account                                | `true`                    |

stable/airflow/templates/configmap-git-clone.yaml

@@ -13,6 +13,11 @@ data:
     REPO=$1
     REF=$2
     DIR=$3
+    {{- if .Values.dags.git.secret }}
+    mkdir -p ~/.ssh/
+    cp -rL /keys/* ~/.ssh/
+    chmod 600 ~/.ssh/*
+    {{- end }}
     # Init Containers will re-run on Pod restart. Remove the directory's contents
     # and reprovision when this happens.
     if [ -d "$DIR" ]; then

stable/airflow/templates/deployments-scheduler.yaml

@@ -54,6 +54,10 @@ spec:
               mountPath: /usr/local/git
             - name: dags-data
               mountPath: /dags
+            {{- if .Values.dags.git.secret }}
+            - name: git-clone-secret
+              mountPath: /keys
+            {{- end }}
       {{- end }}
       containers:
         - name: {{ .Chart.Name }}-scheduler
@@ -154,4 +158,10 @@ spec:
             secretName: {{ template "airflow.fullname" . }}-connections
             defaultMode: 0755
           {{- end }}
-        {{- end }} 
+        {{- if .Values.dags.git.secret }}
+        - name: git-clone-secret
+          secret:
+            secretName: {{ .Values.dags.git.secret }}
+            defaultMode: 0700
+        {{- end }}
+        {{- end }}

stable/airflow/templates/deployments-web.yaml

@@ -53,6 +53,10 @@ spec:
               mountPath: /usr/local/git
             - name: dags-data
               mountPath: /dags
+            {{- if .Values.dags.git.secret }}
+            - name: git-clone-secret
+              mountPath: /keys
+            {{- end }}
       {{- end }}
       containers:
         - name: {{ .Chart.Name }}-web
@@ -149,4 +153,10 @@ spec:
           configMap:
             name: {{ template "airflow.fullname" . }}-git-clone
             defaultMode: 0755
+        {{- if .Values.dags.git.secret }}
+        - name: git-clone-secret
+          secret:
+            secretName: {{ .Values.dags.git.secret }}
+            defaultMode: 0700
+        {{- end }}
         {{- end }}

stable/airflow/templates/statefulsets-workers.yaml

@@ -62,6 +62,10 @@ spec:
             mountPath: /usr/local/git
           - name: dags-data
             mountPath: /dags
+          {{- if .Values.dags.git.secret }}
+          - name: git-clone-secret
+            mountPath: /keys
+          {{- end }}
       {{- end }}
       containers:
         - name: {{ .Chart.Name }}-worker
@@ -153,5 +157,11 @@ spec:
           configMap:
             name: {{ template "airflow.fullname" . }}-git-clone
             defaultMode: 0755
+        {{- if .Values.dags.git.secret }}
+        - name: git-clone-secret
+          secret:
+            secretName: {{ .Values.dags.git.secret }}
+            defaultMode: 0700
+        {{- end }}
         {{- end }}
 {{- end }}