[stable/airflow] init container git clone ssh enhancement (#19171)

* Fixed git clone when using ssh in initContainer

Signed-off-by: Hewitt, Mike <hmike1996@gmail.com>

* Readme vars

Signed-off-by: Hewitt, Mike <hmike1996@gmail.com>

* read me

Signed-off-by: Hewitt, Mike <hmike1996@gmail.com>

stable/airflow/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 description: Airflow is a platform to programmatically author, schedule and monitor workflows
 name: airflow
-version: 5.0.0
+version: 5.1.0
 appVersion: 1.10.4
 icon: https://airflow.apache.org/_images/pin_large.png
 home: https://airflow.apache.org/

stable/airflow/README.md

@@ -285,6 +285,12 @@ If you are using a private Git repo, you can set `dags.gitSecret` to the name of
 
 For example, this will create a secret named `my-git-secret` from your ed25519 key and known_hosts file stored in your home directory:  `kubectl create secret generic my-git-secret --from-file=id_ed25519=~/.ssh/id_ed25519 --from-file=known_hosts=~/.ssh/known_hosts --from-file=id_id_ed25519.pub=~/.ssh/id_ed25519.pub`
 
+#### Init-container git connection ssh
+
+This set of instructions will enable you to clone your repository in the initContainer git-clone.sh script through an ssh connection.
+
+To do this you must have `dags.initContainer.enabled` set to true. Then you need to have the following set. `dags.git.url` This is the repository of your dags. `dags.git.ref` This is the branch with your dags on your repo. `dags.git.secret` This is the name of the secret cotaining your private ssh key. With this you need `dags.git.privateKeyName`, this is the name of the private key in the secret mounted in the keys directory on the initContainer. The last variable you need is this `dags.git.repoHost`. This is the host of your repo, for example if hosted on gitlab set the value as gitlab.com and if github put github.com. This enables us to tell ssh to use our private key when cloning otherwise we would recieve an unable to recognize host bug. 
+
 ### Embedded DAGs
 
 If you want more control on the way you deploy your DAGs, you can use embedded DAGs, where DAGs
@@ -457,6 +463,8 @@ The following table lists the configurable parameters of the Airflow chart and t
 | `dags.git.url`                           | url to clone the git repository                         | nil                       |
 | `dags.git.ref`                           | branch name, tag or sha1 to reset to                    | `master`                  |
 | `dags.git.secret`                        | name of a secret containing an ssh deploy key           | nil                       |
+| `dags.git.privateKeyName`                        | name of private key mounted in secret(only needed if using ssh to connect to git)   | ''                        |
+| `dags.git.repoHost`                        | Host of git repo you are establish an ssh connection to ex. github.com (only needed if using ssh to connect to git)            | ''                        |
 | `logs.path`                              | mount path for logs persistent volume                   | `/usr/local/airflow/logs` |
 | `rbac.create`                            | create RBAC resources                                   | `true`                    |
 | `serviceAccount.create`                  | create a service account                                | `true`                    |

stable/airflow/templates/configmap-git-clone.yaml

@@ -13,10 +13,13 @@ data:
     REPO=$1
     REF=$2
     DIR=$3
+    REPO_HOST=$4
+    PRIVATE_KEY=$5
     {{- if .Values.dags.git.secret }}
     mkdir -p ~/.ssh/
     cp -rL /keys/* ~/.ssh/
     chmod 600 ~/.ssh/*
+    echo -e "HOST $REPO_HOST\n  IdentityFile ~/.ssh/$PRIVATE_KEY" > ~/.ssh/config
     {{- end }}
     # Init Containers will re-run on Pod restart. Remove the directory's contents
     # and reprovision when this happens.

stable/airflow/templates/deployments-scheduler.yaml

@@ -82,6 +82,8 @@ spec:
             - "{{ .Values.dags.git.url }}"
             - "{{ .Values.dags.git.ref }}"
             - "/dags"
+            - "{{ .Values.dags.git.repoHost}}"
+            - "{{ .Values.dags.git.privateKeyName }}"
           volumeMounts:
             - name: git-clone
               mountPath: /usr/local/git

stable/airflow/templates/deployments-web.yaml

@@ -79,6 +79,8 @@ spec:
             - "{{ .Values.dags.git.url }}"
             - "{{ .Values.dags.git.ref }}"
             - "/dags"
+            - "{{ .Values.dags.git.repoHost}}"
+            - "{{ .Values.dags.git.privateKeyName }}"
           volumeMounts:
             - name: git-clone
               mountPath: /usr/local/git

stable/airflow/templates/statefulsets-workers.yaml

@@ -86,6 +86,8 @@ spec:
           - "{{ .Values.dags.git.url }}"
           - "{{ .Values.dags.git.ref }}"
           - "/dags"
+          - "{{ .Values.dags.git.repoHost}}"
+          - "{{ .Values.dags.git.privateKeyName }}"
           volumeMounts:
           - name: git-clone
             mountPath: /usr/local/git

stable/airflow/values.yaml

@@ -525,6 +525,10 @@ dags:
     ref: master
     ## pre-created secret with key, key.pub and known_hosts file for private repos
     secret: ""
+    ## The host of the repo so for example if a github repo put github.com (Only need if using ssh not https git sync)
+    repoHost: ""
+    ## The name of the private key in your git sync secret (Only need if using ssh not https git sync)
+    privateKeyName: ""
   initContainer:
     ## Fetch the source code when the pods starts
     enabled: false