From 3a8e1c6ead520db21df4b5ef80b466dc64f1bc16 Mon Sep 17 00:00:00 2001 From: Stijn De Haes Date: Thu, 14 Mar 2019 02:00:41 +0100 Subject: [PATCH] Added airflow variables support (#11990) Signed-off-by: Stijn De Haes --- stable/airflow/Chart.yaml | 2 +- stable/airflow/README.md | 11 +++++++++++ stable/airflow/examples/minikube-values.yaml | 2 ++ .../airflow/templates/configmap-variables.yaml | 14 ++++++++++++++ .../templates/deployments-scheduler.yaml | 18 ++++++++++++++++++ stable/airflow/values.yaml | 6 ++++++ 6 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 stable/airflow/templates/configmap-variables.yaml diff --git a/stable/airflow/Chart.yaml b/stable/airflow/Chart.yaml index b230eb4360..58c5e784c2 100644 --- a/stable/airflow/Chart.yaml +++ b/stable/airflow/Chart.yaml @@ -1,6 +1,6 @@ description: Airflow is a platform to programmatically author, schedule and monitor workflows name: airflow -version: 2.1.0 +version: 2.2.0 appVersion: 1.10.0 icon: https://airflow.apache.org/_images/pin_large.png home: https://airflow.apache.org/ diff --git a/stable/airflow/README.md b/stable/airflow/README.md index e910a5f6f4..fbaef2405b 100644 --- a/stable/airflow/README.md +++ b/stable/airflow/README.md @@ -115,6 +115,17 @@ airflow: Note: As connections may require to include sensitive data - the resulting script is stored encrypted in a kubernetes secret and mounted into the airflow scheduler container. It is probably wise not to put connection data in the default values.yaml and instead create an encrypted my-secret-values.yaml. this way it can be decrypted before the installation and passed to helm with -f +#### Airflow variables + +Variables are a generic way to store and retrieve arbitrary content or settings as a simple key value store within Airflow. +These variables will be automatically imported by the scheduler when it starts up. + +Example: +```yaml +airflow: + variables: '{ "environment": "dev" }' +``` + ### Worker Statefulset Celery workers uses StatefulSet. diff --git a/stable/airflow/examples/minikube-values.yaml b/stable/airflow/examples/minikube-values.yaml index d06fd85b0e..4ffb5b4388 100644 --- a/stable/airflow/examples/minikube-values.yaml +++ b/stable/airflow/examples/minikube-values.yaml @@ -10,6 +10,8 @@ airflow: AIRFLOW__CORE__LOGGING_LEVEL: DEBUG AIRFLOW__CORE__LOAD_EXAMPLES: True + variables: '{ "environment": "dev" }' + workers: replicas: 1 celery: diff --git a/stable/airflow/templates/configmap-variables.yaml b/stable/airflow/templates/configmap-variables.yaml new file mode 100644 index 0000000000..3305441204 --- /dev/null +++ b/stable/airflow/templates/configmap-variables.yaml @@ -0,0 +1,14 @@ +{{- if .Values.airflow.variables }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "airflow.fullname" . }}-variables + labels: + app: {{ template "airflow.name" . }} + chart: {{ template "airflow.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + variables.json: | + {{ .Values.airflow.variables }} +{{- end }} \ No newline at end of file diff --git a/stable/airflow/templates/deployments-scheduler.yaml b/stable/airflow/templates/deployments-scheduler.yaml index 0103b7fbe8..65b8df5e45 100755 --- a/stable/airflow/templates/deployments-scheduler.yaml +++ b/stable/airflow/templates/deployments-scheduler.yaml @@ -89,6 +89,10 @@ spec: - name: connections mountPath: /usr/local/connections {{- end}} + {{- if .Values.airflow.variables }} + - name: variables + mountPath: /usr/local/variables/ + {{- end}} {{- range .Values.airflow.extraConfigmapMounts }} - name: {{ .name }} mountPath: {{ .mountPath }} @@ -113,6 +117,10 @@ spec: {{- if .Values.airflow.connections }} echo "adding connections" && /usr/local/connections/add-connections.sh && + {{- end }} + {{- if .Values.airflow.variables }} + echo "adding variables" && + airflow variables -i /usr/local/variables/variables.json && {{- end }} echo "executing scheduler" && airflow scheduler -n {{ .Values.airflow.schedulerNumRuns }} @@ -127,6 +135,10 @@ spec: {{- if .Values.airflow.connections }} echo "adding connections" && /usr/local/connections/add-connections.sh && + {{- end }} + {{- if .Values.airflow.variables }} + echo "adding variables" && + airflow variables -i /usr/local/variables/variables.json && {{- end }} echo "executing scheduler" && airflow scheduler -n {{ .Values.airflow.schedulerNumRuns }} @@ -169,6 +181,12 @@ spec: secretName: {{ template "airflow.fullname" . }}-connections defaultMode: 0755 {{- end }} + {{- if .Values.airflow.variables }} + - name: variables + configMap: + name: {{ template "airflow.fullname" . }}-variables + defaultMode: 0755 + {{- end }} {{- range .Values.airflow.extraConfigmapMounts }} - name: {{ .name }} configMap: diff --git a/stable/airflow/values.yaml b/stable/airflow/values.yaml index 88532e078d..b5276d6d6f 100644 --- a/stable/airflow/values.yaml +++ b/stable/airflow/values.yaml @@ -132,6 +132,12 @@ airflow: ## type: aws ## extra: '{"aws_access_key_id": "**********", "aws_secret_access_key": "***", "region_name":"eu-central-1"}' connections: {} + + ## Add airflow variables + ## This should be a json string with your variables in it + ## Examples: + ## variables: '{ "environment": "dev" }' + variables: {} ## ## Annotations for the Scheduler, Worker and Web pods podAnnotations: {}