From 783788e36974f8da02fb144367bc995daa8774d7 Mon Sep 17 00:00:00 2001 From: Ken Wronkiewicz Date: Wed, 30 Jan 2019 08:39:49 -0800 Subject: [PATCH] Adding a git pull secret for dags in Airflow (#10987) * Adding a git pull secret for dags in Airflow * Added `dags.git.secret` key with no default * Added `/keys` secret mount to the scheduler, web, and worker defs * Added extra copy step to the git clone configmap when key is active * Preserved default behavior otherwise * Updated documentation to match * Updated version number Signed-off-by: Ken Wronkiewicz * v0.14.1 -> v0.15.0 because new feature. Signed-off-by: Ken Wronkiewicz --- stable/airflow/Chart.yaml | 2 +- stable/airflow/README.md | 5 +++++ stable/airflow/templates/configmap-git-clone.yaml | 5 +++++ stable/airflow/templates/deployments-scheduler.yaml | 12 +++++++++++- stable/airflow/templates/deployments-web.yaml | 10 ++++++++++ stable/airflow/templates/statefulsets-workers.yaml | 10 ++++++++++ 6 files changed, 42 insertions(+), 2 deletions(-) diff --git a/stable/airflow/Chart.yaml b/stable/airflow/Chart.yaml index 32f25efa62..64acd2fdec 100644 --- a/stable/airflow/Chart.yaml +++ b/stable/airflow/Chart.yaml @@ -1,6 +1,6 @@ description: Airflow is a platform to programmatically author, schedule and monitor workflows name: airflow -version: 0.14.0 +version: 0.15.0 appVersion: 1.10.0 icon: https://airflow.apache.org/_images/pin_large.png home: https://airflow.apache.org/ diff --git a/stable/airflow/README.md b/stable/airflow/README.md index ea722aa76b..3d8fbc7144 100644 --- a/stable/airflow/README.md +++ b/stable/airflow/README.md @@ -203,6 +203,10 @@ Python dependencies installed. This is the easiest way of deploying your DAGs to Airflow. +If you are using a private Git repo, you can set `dags.gitSecret` to the name of a secret you created containing private keys and a `known_hosts` file. + +For example, this will create a secret named `my-git-secret` from your ed25519 key and known_hosts file stored in your home directory: `kubectl create secret generic my-git-secret --from-file=id_ed25519=~/.ssh/id_ed25519 --from-file=known_hosts=~/.ssh/known_hosts --from-file=id_id_ed25519.pub=~/.ssh/id_ed25519.pub` + ### Embedded DAGs If you want more control on the way you deploy your DAGs, you can use embedded DAGs, where DAGs @@ -288,6 +292,7 @@ The following table lists the configurable parameters of the Airflow chart and t | `dags.initContainer.installRequirements` | auto install requirements.txt deps | `true` | | `dags.git.url` | url to clone the git repository | nil | | `dags.git.ref` | branch name, tag or sha1 to reset to | `master` | +| `dags.git.secret` | name of a secret containing an ssh deploy key | nil | | `logs.path` | mount path for logs persistent volume | `/usr/local/airflow/logs` | | `rbac.create` | create RBAC resources | `true` | | `serviceAccount.create` | create a service account | `true` | diff --git a/stable/airflow/templates/configmap-git-clone.yaml b/stable/airflow/templates/configmap-git-clone.yaml index 0d2abbcc3c..241618f680 100644 --- a/stable/airflow/templates/configmap-git-clone.yaml +++ b/stable/airflow/templates/configmap-git-clone.yaml @@ -13,6 +13,11 @@ data: REPO=$1 REF=$2 DIR=$3 + {{- if .Values.dags.git.secret }} + mkdir -p ~/.ssh/ + cp -rL /keys/* ~/.ssh/ + chmod 600 ~/.ssh/* + {{- end }} # Init Containers will re-run on Pod restart. Remove the directory's contents # and reprovision when this happens. if [ -d "$DIR" ]; then diff --git a/stable/airflow/templates/deployments-scheduler.yaml b/stable/airflow/templates/deployments-scheduler.yaml index 0db70a0aa2..ff582eefa2 100755 --- a/stable/airflow/templates/deployments-scheduler.yaml +++ b/stable/airflow/templates/deployments-scheduler.yaml @@ -54,6 +54,10 @@ spec: mountPath: /usr/local/git - name: dags-data mountPath: /dags + {{- if .Values.dags.git.secret }} + - name: git-clone-secret + mountPath: /keys + {{- end }} {{- end }} containers: - name: {{ .Chart.Name }}-scheduler @@ -154,4 +158,10 @@ spec: secretName: {{ template "airflow.fullname" . }}-connections defaultMode: 0755 {{- end }} - {{- end }} \ No newline at end of file + {{- if .Values.dags.git.secret }} + - name: git-clone-secret + secret: + secretName: {{ .Values.dags.git.secret }} + defaultMode: 0700 + {{- end }} + {{- end }} diff --git a/stable/airflow/templates/deployments-web.yaml b/stable/airflow/templates/deployments-web.yaml index fb34e13d81..c71431041e 100644 --- a/stable/airflow/templates/deployments-web.yaml +++ b/stable/airflow/templates/deployments-web.yaml @@ -53,6 +53,10 @@ spec: mountPath: /usr/local/git - name: dags-data mountPath: /dags + {{- if .Values.dags.git.secret }} + - name: git-clone-secret + mountPath: /keys + {{- end }} {{- end }} containers: - name: {{ .Chart.Name }}-web @@ -149,4 +153,10 @@ spec: configMap: name: {{ template "airflow.fullname" . }}-git-clone defaultMode: 0755 + {{- if .Values.dags.git.secret }} + - name: git-clone-secret + secret: + secretName: {{ .Values.dags.git.secret }} + defaultMode: 0700 + {{- end }} {{- end }} diff --git a/stable/airflow/templates/statefulsets-workers.yaml b/stable/airflow/templates/statefulsets-workers.yaml index 9a32cd4b4c..3ea0b5c926 100644 --- a/stable/airflow/templates/statefulsets-workers.yaml +++ b/stable/airflow/templates/statefulsets-workers.yaml @@ -62,6 +62,10 @@ spec: mountPath: /usr/local/git - name: dags-data mountPath: /dags + {{- if .Values.dags.git.secret }} + - name: git-clone-secret + mountPath: /keys + {{- end }} {{- end }} containers: - name: {{ .Chart.Name }}-worker @@ -153,5 +157,11 @@ spec: configMap: name: {{ template "airflow.fullname" . }}-git-clone defaultMode: 0755 + {{- if .Values.dags.git.secret }} + - name: git-clone-secret + secret: + secretName: {{ .Values.dags.git.secret }} + defaultMode: 0700 + {{- end }} {{- end }} {{- end }}