From 9293bc56f18b57a0f16c4f4e9dd196d131ed8eef Mon Sep 17 00:00:00 2001 From: Mathew Wicks Date: Mon, 5 Oct 2020 17:19:06 +1100 Subject: [PATCH] [stable/airflow] release 7.11.1 (#23843) * [stable/airflow] yaml cleanup Signed-off-by: Mathew Wicks * [stable/airflow] release 7.11.1 Signed-off-by: Mathew Wicks --- stable/airflow/Chart.yaml | 2 +- stable/airflow/README.md | 71 ++++++------------- stable/airflow/UPGRADE.md | 6 +- .../templates/config/secret-connections.yaml | 48 ++++++------- stable/airflow/templates/pvc-logs.yaml | 10 +-- stable/airflow/templates/pvc.yaml | 8 +-- .../scheduler/scheduler-deployment.yaml | 13 ++-- stable/airflow/values.yaml | 38 +++++++--- 8 files changed, 98 insertions(+), 98 deletions(-) diff --git a/stable/airflow/Chart.yaml b/stable/airflow/Chart.yaml index d9147ef89f..a66d35f5bf 100644 --- a/stable/airflow/Chart.yaml +++ b/stable/airflow/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Airflow is a platform to programmatically author, schedule and monitor workflows name: airflow -version: 7.11.0 +version: 7.11.1 appVersion: 1.10.12 icon: https://airflow.apache.org/_images/pin_large.png home: https://airflow.apache.org/ diff --git a/stable/airflow/README.md b/stable/airflow/README.md index 3e9bea5565..c45e2b663c 100644 --- a/stable/airflow/README.md +++ b/stable/airflow/README.md @@ -109,6 +109,9 @@ airflow: We expose the `scheduler.connections` value to allow specifying [Airflow Connections](https://airflow.apache.org/docs/stable/concepts.html#connections) at deployment time, these connections will be automatically imported by the Airflow scheduler when it starts up. +By default, we will delete and re-create any connections specified in `scheduler.connections` each time the Airflow scheduler restarts. +(If you want to manually modify a connection in the WebUI, you should disable this behaviour by setting `scheduler.refreshConnections` to `false`) + For example, to add a connection called `my_aws`: ```yaml scheduler: @@ -117,69 +120,38 @@ scheduler: type: aws extra: | { - "aws_access_key_id": "XXXXXXXXXXXXXXXXXXX", - "aws_secret_access_key": "XXXXXXXXXXXXXXX", + "aws_access_key_id": "XXXXXXXX", + "aws_secret_access_key": "XXXXXXXX", "region_name":"eu-central-1" } ``` -We expose the `scheduler.refreshConnections` value to refresh connections by -removing them before adding them when they are automatically being imported. The -default value is true, so if a user wants to add a password after the initial -deployment, they should set `scheduler.refreshConnections` to false. +If you do not want to store connections in your `values.yaml`, use `scheduler.existingSecretConnections` to specify the name of an existing Secret containing an `add-connections.sh` script. +Note: your script will be run EACH TIME the airflow-scheduler Pod restarts, and `scheduler.connections` will not longer work. -We expose the `scheduler.existingSecretConnections` value to allow connections from an existing secrets resource. This may be desireable when you have shared resources between different deployments or you want to use a custom resource, e.g. ExternalSecrets, to avoid exposing credentials. The resulting existing secret should have an `add-connections.sh` data containing `airflow connections --add` statements. There's a few ways to achieve them, including: - -- Secret - -This is a straight-forward way to keep credentials away from the values in the chart, which can be useful to avoid commiting them to a git repository, for example. Here's an example: - +Here is an example Secret you might create: ```yaml apiVersion: v1 kind: Secret metadata: - name: my-custom-airflow-connections + name: my-airflow-connections type: Opaque stringData: - add-connections.sh: |- + add-connections.sh: | #!/usr/bin/env bash - airflow connections --add --conn_id my_aws_custom_conn --conn_type "aws" --conn_extra "{\n \"aws_access_key_id\": \"XXXXXXXXXXXXXXXXXXX\",\n \"aws_secret_access_key\": \"XXXXXXXXXXXXXXX\",\n \"region_name\":\"eu-central-1\"\n}\n" + + # remove any existing connection + airflow connections --delete \ + --conn_id "my_aws" + + # re-add your custom connection + airflow connections --add \ + --conn_id "my_aws" \ + --conn_type "aws" \ + --conn_extra "{\\"aws_access_key_id\\": \\"XXXXXXXX\\", \\"aws_secret_access_key\\": \\"XXXXXXXX\", \\"region_name\\":\\"eu-central-1\\"}" ``` -- ExternalSecret - -You can create an external secret to keep credentials safe in a Secret Manager, adding governance layers and avoiding exposing its values in your Kubernetes cluster. [More information on how to implement this in your cluster here](https://github.com/godaddy/kubernetes-external-secrets#install-with-helm). -Here's an example using AWS Secret Manager ([read more about it here](https://github.com/godaddy/kubernetes-external-secrets#install-with-helm)): - -```yaml -apiVersion: "kubernetes-client.io/v1" -kind: ExternalSecret -metadata: - name: my-safe-custom-airflow-connections -spec: - roleArn: arn:aws:iam::123456789012:role/my-secret-manager-role - template: - type: Opaque - backendType: secretsManager - data: - - key: secrets/manager/key - name: username - property: username - - key: secrets/manager/key - name: password - property: password -``` -(This resource is only appliable after the custom resource is correcly installed) - - -If `scheduler.existingSecretConnections` is defined and set to any valid value, the `scheduler.connections` and `scheduler.refreshConnections` values will be ignored. - - -__NOTE:__ As connections may include sensitive data, we store the bash script which generates the connections in a Kubernetes Secret, and mount this to the pods. - -__WARNING:__ Because some values are sensitive, you should take care to store your custom `values.yaml` securely before passing it to helm with: `helm -f ` - ### Airflow-Configs/Variables We expose the `scheduler.variables` value to allow specifying [Airflow Variables](https://airflow.apache.org/docs/stable/concepts.html#variables) at deployment time, variables will be automatically imported by the Airflow scheduler when it starts up. @@ -623,7 +595,8 @@ __Airflow Scheduler values:__ | `scheduler.safeToEvict` | if we should tell Kubernetes Autoscaler that its safe to evict these Pods | `true` | | `scheduler.podDisruptionBudget.*` | configs for the PodDisruptionBudget of the scheduler | `` | | `scheduler.connections` | custom airflow connections for the airflow scheduler | `[]` | -| `scheduler.refreshConnections` | if we remove before adding a connection resulting in a refresh | `true` | +| `scheduler.refreshConnections` | if `scheduler.connections` are deleted and re-added after each scheduler restart | `true` | +| `scheduler.existingSecretConnections` | the name of an existing Secret containing an `add-connections.sh` script to run on scheduler start | `""` | | `scheduler.variables` | custom airflow variables for the airflow scheduler | `"{}"` | | `scheduler.pools` | custom airflow pools for the airflow scheduler | `"{}"` | | `scheduler.numRuns` | the value of the `airflow --num_runs` parameter used to run the airflow scheduler | `-1` | diff --git a/stable/airflow/UPGRADE.md b/stable/airflow/UPGRADE.md index 9bb1dbb0e0..f8e0776788 100644 --- a/stable/airflow/UPGRADE.md +++ b/stable/airflow/UPGRADE.md @@ -3,7 +3,11 @@ ## `v7.10.X` → `v7.11.0` __The following IMPROVEMENTS have been made:__ -* We can now pass existing secrets containing Scheduler Connections in `scheduler.existingSecretConnections`, allowing for other ways to create those connections without declaring them on Chart Values. +* You can now use `scheduler.existingSecretConnections` with an externally created Secret to store airflow connections. + (Rather than storing them in plain-text with `scheduler.connections`) + +__The following values have been ADDED:__ +* `scheduler.existingSecretConnections` ## `v7.9.X` → `v7.10.0` diff --git a/stable/airflow/templates/config/secret-connections.yaml b/stable/airflow/templates/config/secret-connections.yaml index a01058a373..97ce913fe9 100644 --- a/stable/airflow/templates/config/secret-connections.yaml +++ b/stable/airflow/templates/config/secret-connections.yaml @@ -1,31 +1,31 @@ -{{- define "connections.script" }} - #!/usr/bin/env bash - {{- range .Values.scheduler.connections }} - {{- if $.Values.scheduler.refreshConnections }} - airflow connections --delete --conn_id {{ .id }} - {{- end }} - airflow connections --add --conn_id {{ .id }} - {{- if .type }} --conn_type {{ .type | quote }} {{ end -}} - {{- if .uri }} --conn_uri {{ .uri | quote }} {{ end -}} - {{- if .host }} --conn_host {{ .host | quote }} {{ end -}} - {{- if .login }} --conn_login {{ .login | quote }} {{ end -}} - {{- if .password }} --conn_password {{ .password | quote }} {{ end -}} - {{- if .schema }} --conn_schema {{ .schema | quote }} {{ end -}} - {{- if .port }} --conn_port {{ .port }} {{ end -}} - {{- if .extra }} --conn_extra {{ .extra | quote }} {{ end -}} - {{- end }} -{{- end }} -{{- if and (.Values.scheduler.connections) (not .Values.scheduler.existingSecretConnections) }} +{{- if not .Values.scheduler.existingSecretConnections }} +{{- if .Values.scheduler.connections }} apiVersion: v1 kind: Secret metadata: name: {{ include "airflow.fullname" . }}-connections labels: - app: {{ include "airflow.fullname" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" + app: {{ include "airflow.labels.app" . }} + chart: {{ include "airflow.labels.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} type: Opaque -data: - add-connections.sh: {{ include "connections.script" . | b64enc }} +stringData: + add-connections.sh: | + #!/usr/bin/env bash + {{- range .Values.scheduler.connections }} + {{- if $.Values.scheduler.refreshConnections }} + airflow connections --delete --conn_id {{ .id }} + {{- end }} + airflow connections --add --conn_id {{ .id }} + {{- if .type }} --conn_type {{ .type | quote }} {{ end -}} + {{- if .uri }} --conn_uri {{ .uri | quote }} {{ end -}} + {{- if .host }} --conn_host {{ .host | quote }} {{ end -}} + {{- if .login }} --conn_login {{ .login | quote }} {{ end -}} + {{- if .password }} --conn_password {{ .password | quote }} {{ end -}} + {{- if .schema }} --conn_schema {{ .schema | quote }} {{ end -}} + {{- if .port }} --conn_port {{ .port }} {{ end -}} + {{- if .extra }} --conn_extra {{ .extra | quote }} {{ end -}} + {{- end }} +{{- end }} {{- end }} diff --git a/stable/airflow/templates/pvc-logs.yaml b/stable/airflow/templates/pvc-logs.yaml index c5bb7178ba..b0938322ed 100644 --- a/stable/airflow/templates/pvc-logs.yaml +++ b/stable/airflow/templates/pvc-logs.yaml @@ -2,12 +2,12 @@ kind: PersistentVolumeClaim apiVersion: v1 metadata: - name: {{ printf "%s-logs" (include "airflow.fullname" . | trunc 58)}} + name: {{ printf "%s-logs" (include "airflow.fullname" . | trunc 58) }} labels: - app: {{ include "airflow.fullname" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" + app: {{ include "airflow.labels.app" . }} + chart: {{ include "airflow.labels.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} spec: accessModes: - {{ .Values.logs.persistence.accessMode | quote }} diff --git a/stable/airflow/templates/pvc.yaml b/stable/airflow/templates/pvc.yaml index 7f9f463c61..fae50cf96f 100644 --- a/stable/airflow/templates/pvc.yaml +++ b/stable/airflow/templates/pvc.yaml @@ -4,10 +4,10 @@ apiVersion: v1 metadata: name: {{ include "airflow.fullname" . }} labels: - app: {{ include "airflow.fullname" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" + app: {{ include "airflow.labels.app" . }} + chart: {{ include "airflow.labels.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} spec: accessModes: - {{ .Values.dags.persistence.accessMode | quote }} diff --git a/stable/airflow/templates/scheduler/scheduler-deployment.yaml b/stable/airflow/templates/scheduler/scheduler-deployment.yaml index b687cbdcc1..983c4bcf5b 100644 --- a/stable/airflow/templates/scheduler/scheduler-deployment.yaml +++ b/stable/airflow/templates/scheduler/scheduler-deployment.yaml @@ -194,7 +194,7 @@ spec: mountPath: {{ .Values.logs.path }} subPath: {{ .Values.logs.persistence.subPath }} {{- end }} - {{- if or (.Values.scheduler.connections) (.Values.scheduler.existingSecretConnections) }} + {{- if or (.Values.scheduler.connections) (.Values.scheduler.existingSecretConnections) }} - name: connections mountPath: /home/airflow/connections {{- end}} @@ -243,7 +243,7 @@ spec: && echo "*** adding Airflow variables..." \ && airflow variables -i /home/airflow/variables-pools/variables.json \ {{- end }} - {{- if or (.Values.scheduler.connections) (.Values.scheduler.existingSecretConnections) }} + {{- if or (.Values.scheduler.connections) (.Values.scheduler.existingSecretConnections) }} && echo "*** adding Airflow connections..." \ && /home/airflow/connections/add-connections.sh \ {{- end }} @@ -309,9 +309,12 @@ spec: {{- if or (.Values.scheduler.connections) (.Values.scheduler.existingSecretConnections) }} - name: connections secret: - secretName: {{ if .Values.scheduler.existingSecretConnections }}{{ .Values.scheduler.existingSecretConnections }} - {{- else }}{{ include "airflow.fullname" . }}-connections - {{- end }} + secretName: | + {{- if .Values.scheduler.existingSecretConnections }} + {{ .Values.scheduler.existingSecretConnections }} + {{- else }} + {{ include "airflow.fullname" . }}-connections + {{- end }} defaultMode: 0755 {{- end }} {{- if or (.Values.scheduler.variables) (.Values.scheduler.pools) }} diff --git a/stable/airflow/values.yaml b/stable/airflow/values.yaml index 2d97af8878..6ad45c3f8b 100644 --- a/stable/airflow/values.yaml +++ b/stable/airflow/values.yaml @@ -224,30 +224,50 @@ scheduler: ## custom airflow connections for the airflow scheduler ## - ## NOTE: - ## - connections are created with a script that is stored in a K8s secret and mounted into the scheduler container - ## ## EXAMPLE: ## connections: ## - id: my_aws ## type: aws ## extra: | ## { - ## "aws_access_key_id": "XXXXXXXXXXXXXXXXXXX", - ## "aws_secret_access_key": "XXXXXXXXXXXXXXX", + ## "aws_access_key_id": "XXXXXXXX", + ## "aws_secret_access_key": "XXXXXXXX", ## "region_name":"eu-central-1" ## } ## connections: [] - ## if we remove before adding a connection resulting in a refresh + ## if `scheduler.connections` are deleted and re-added after each scheduler restart ## refreshConnections: true - ## existingSecretConnections can receive an existing secret name to add connections using a custom secrets resource - ## if this variable is set to any valid value, scheduler.connections will be ignored + ## the name of an existing Secret containing an `add-connections.sh` script to run on scheduler start ## - ## existingSecretConnections: "custom-connections" + ## NOTE: + ## - if this is non-empty, `scheduler.connections` will be ignored + ## - use this if you don't want to store connections in your values.yaml + ## + ## EXAMPLE SECRET: + ## apiVersion: v1 + ## kind: Secret + ## metadata: + ## name: my-airflow-connections + ## type: Opaque + ## stringData: + ## add-connections.sh: | + ## #!/usr/bin/env bash + ## + ## # remove any existing connection + ## airflow connections --delete \ + ## --conn_id "my_aws" + ## + ## # re-add your custom connection + ## airflow connections --add \ + ## --conn_id "my_aws" \ + ## --conn_type "aws" \ + ## --conn_extra "{\"region_name\":\"eu-central-1\"}" + ## + existingSecretConnections: "" ## custom airflow variables for the airflow scheduler ##