diff --git a/stable/airflow/Chart.yaml b/stable/airflow/Chart.yaml index 01db047212..2ee893c01f 100644 --- a/stable/airflow/Chart.yaml +++ b/stable/airflow/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Airflow is a platform to programmatically author, schedule and monitor workflows name: airflow -version: 5.0.0 +version: 5.1.0 appVersion: 1.10.4 icon: https://airflow.apache.org/_images/pin_large.png home: https://airflow.apache.org/ diff --git a/stable/airflow/README.md b/stable/airflow/README.md index ad14b53c32..5fd2f42b0f 100644 --- a/stable/airflow/README.md +++ b/stable/airflow/README.md @@ -285,6 +285,12 @@ If you are using a private Git repo, you can set `dags.gitSecret` to the name of For example, this will create a secret named `my-git-secret` from your ed25519 key and known_hosts file stored in your home directory: `kubectl create secret generic my-git-secret --from-file=id_ed25519=~/.ssh/id_ed25519 --from-file=known_hosts=~/.ssh/known_hosts --from-file=id_id_ed25519.pub=~/.ssh/id_ed25519.pub` +#### Init-container git connection ssh + +This set of instructions will enable you to clone your repository in the initContainer git-clone.sh script through an ssh connection. + +To do this you must have `dags.initContainer.enabled` set to true. Then you need to have the following set. `dags.git.url` This is the repository of your dags. `dags.git.ref` This is the branch with your dags on your repo. `dags.git.secret` This is the name of the secret cotaining your private ssh key. With this you need `dags.git.privateKeyName`, this is the name of the private key in the secret mounted in the keys directory on the initContainer. The last variable you need is this `dags.git.repoHost`. This is the host of your repo, for example if hosted on gitlab set the value as gitlab.com and if github put github.com. This enables us to tell ssh to use our private key when cloning otherwise we would recieve an unable to recognize host bug. + ### Embedded DAGs If you want more control on the way you deploy your DAGs, you can use embedded DAGs, where DAGs @@ -457,6 +463,8 @@ The following table lists the configurable parameters of the Airflow chart and t | `dags.git.url` | url to clone the git repository | nil | | `dags.git.ref` | branch name, tag or sha1 to reset to | `master` | | `dags.git.secret` | name of a secret containing an ssh deploy key | nil | +| `dags.git.privateKeyName` | name of private key mounted in secret(only needed if using ssh to connect to git) | '' | +| `dags.git.repoHost` | Host of git repo you are establish an ssh connection to ex. github.com (only needed if using ssh to connect to git) | '' | | `logs.path` | mount path for logs persistent volume | `/usr/local/airflow/logs` | | `rbac.create` | create RBAC resources | `true` | | `serviceAccount.create` | create a service account | `true` | diff --git a/stable/airflow/templates/configmap-git-clone.yaml b/stable/airflow/templates/configmap-git-clone.yaml index 241618f680..f1eb671577 100644 --- a/stable/airflow/templates/configmap-git-clone.yaml +++ b/stable/airflow/templates/configmap-git-clone.yaml @@ -13,10 +13,13 @@ data: REPO=$1 REF=$2 DIR=$3 + REPO_HOST=$4 + PRIVATE_KEY=$5 {{- if .Values.dags.git.secret }} mkdir -p ~/.ssh/ cp -rL /keys/* ~/.ssh/ chmod 600 ~/.ssh/* + echo -e "HOST $REPO_HOST\n IdentityFile ~/.ssh/$PRIVATE_KEY" > ~/.ssh/config {{- end }} # Init Containers will re-run on Pod restart. Remove the directory's contents # and reprovision when this happens. diff --git a/stable/airflow/templates/deployments-scheduler.yaml b/stable/airflow/templates/deployments-scheduler.yaml index 35cbca6faa..d5a0dd1f86 100644 --- a/stable/airflow/templates/deployments-scheduler.yaml +++ b/stable/airflow/templates/deployments-scheduler.yaml @@ -82,6 +82,8 @@ spec: - "{{ .Values.dags.git.url }}" - "{{ .Values.dags.git.ref }}" - "/dags" + - "{{ .Values.dags.git.repoHost}}" + - "{{ .Values.dags.git.privateKeyName }}" volumeMounts: - name: git-clone mountPath: /usr/local/git diff --git a/stable/airflow/templates/deployments-web.yaml b/stable/airflow/templates/deployments-web.yaml index e21b0dbd35..19cb86a4d1 100644 --- a/stable/airflow/templates/deployments-web.yaml +++ b/stable/airflow/templates/deployments-web.yaml @@ -79,6 +79,8 @@ spec: - "{{ .Values.dags.git.url }}" - "{{ .Values.dags.git.ref }}" - "/dags" + - "{{ .Values.dags.git.repoHost}}" + - "{{ .Values.dags.git.privateKeyName }}" volumeMounts: - name: git-clone mountPath: /usr/local/git diff --git a/stable/airflow/templates/statefulsets-workers.yaml b/stable/airflow/templates/statefulsets-workers.yaml index 4941b008f1..788655ad18 100644 --- a/stable/airflow/templates/statefulsets-workers.yaml +++ b/stable/airflow/templates/statefulsets-workers.yaml @@ -86,6 +86,8 @@ spec: - "{{ .Values.dags.git.url }}" - "{{ .Values.dags.git.ref }}" - "/dags" + - "{{ .Values.dags.git.repoHost}}" + - "{{ .Values.dags.git.privateKeyName }}" volumeMounts: - name: git-clone mountPath: /usr/local/git diff --git a/stable/airflow/values.yaml b/stable/airflow/values.yaml index 110056ec91..51d373d196 100644 --- a/stable/airflow/values.yaml +++ b/stable/airflow/values.yaml @@ -525,6 +525,10 @@ dags: ref: master ## pre-created secret with key, key.pub and known_hosts file for private repos secret: "" + ## The host of the repo so for example if a github repo put github.com (Only need if using ssh not https git sync) + repoHost: "" + ## The name of the private key in your git sync secret (Only need if using ssh not https git sync) + privateKeyName: "" initContainer: ## Fetch the source code when the pods starts enabled: false