From 3e7dde89215ac89958398e4e7cf464f8a1d04b5c Mon Sep 17 00:00:00 2001 From: Mathew Wicks Date: Tue, 26 May 2020 17:27:11 +1000 Subject: [PATCH] [stable/airflow] 7.1.1 hotfix (#22523) Signed-off-by: Mathew Wicks --- stable/airflow/Chart.yaml | 2 +- stable/airflow/README.md | 2 +- stable/airflow/UPGRADE.md | 5 +++-- .../examples/google-gke/custom-values.yaml | 1 + .../examples/minikube/custom-values.yaml | 2 +- .../templates/deployments-scheduler.yaml | 7 ++++--- stable/airflow/templates/deployments-web.yaml | 9 +++++---- stable/airflow/templates/pdb-sheduler.yaml | 2 +- .../templates/statefulsets-workers.yaml | 9 +++++---- stable/airflow/values.yaml | 18 ++++++++++-------- 10 files changed, 32 insertions(+), 25 deletions(-) diff --git a/stable/airflow/Chart.yaml b/stable/airflow/Chart.yaml index 54838e10f8..89f1dfc964 100644 --- a/stable/airflow/Chart.yaml +++ b/stable/airflow/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Airflow is a platform to programmatically author, schedule and monitor workflows name: airflow -version: 7.1.0 +version: 7.1.1 appVersion: 1.10.10 icon: https://airflow.apache.org/_images/pin_large.png home: https://airflow.apache.org/ diff --git a/stable/airflow/README.md b/stable/airflow/README.md index 213a5be688..0adfbb8447 100644 --- a/stable/airflow/README.md +++ b/stable/airflow/README.md @@ -516,7 +516,7 @@ __Global Values:__ | --- | --- | --- | | `airflow.image.*` | configs for the docker image of the web/scheduler/worker | `` | | `airflow.executor` | the airflow executor type to use | `CeleryExecutor` | -| `airflow.fernetKey` | the fernet key used to encrypt the connections in the database | `""` | +| `airflow.fernetKey` | the fernet key used to encrypt the connections/variables in the database | `7T512UXSSmBOkpWimFHIVb8jK6lfmSAvx4mO6Arehnc=` | | `airflow.config` | environment variables for the web/scheduler/worker pods (for airflow configs) | `{}` | | `airflow.podAnnotations` | extra annotations for the web/scheduler/worker/flower Pods | `{}` | | `airflow.extraEnv` | extra environment variables for the web/scheduler/worker/flower Pods | `[]` | diff --git a/stable/airflow/UPGRADE.md b/stable/airflow/UPGRADE.md index 2dd4e3b9e0..269b2cd154 100644 --- a/stable/airflow/UPGRADE.md +++ b/stable/airflow/UPGRADE.md @@ -21,8 +21,9 @@ __The following IMPROVEMENTS have been made:__ __The following values have CHANGED DEFAULTS:__ * `airflow.fernetKey`: - * Is now `""` by default, to enforce that users generate a custom one. - (However, please consider using `airflow.extraEnv` to define it from a pre-created secret) + * ~~Is now `""` by default, to enforce that users generate a custom one.~~ + ~~(However, please consider using `airflow.extraEnv` to define it from a pre-created secret)~~ + __(We have undone this change in `7.1.1`, but we still encourage you to set a custom fernetKey!)__ * `dags.installRequirements`: * Is now `false` by default, as this was an unintended change with the 7.0.0 upgrade. diff --git a/stable/airflow/examples/google-gke/custom-values.yaml b/stable/airflow/examples/google-gke/custom-values.yaml index 1f752009e6..35a981ff03 100644 --- a/stable/airflow/examples/google-gke/custom-values.yaml +++ b/stable/airflow/examples/google-gke/custom-values.yaml @@ -27,6 +27,7 @@ # - Secret: airflow-cluster1-fernet-key # - Secret: airflow-cluster1-mysql-password # - Secret: airflow-cluster1-redis-password +# - Secret: airflow-cluster1-git-keys # - ConfigMap: airflow-cluster1-webserver-config # - cert-manager.io/Certificate: airflow-cluster1-cert # diff --git a/stable/airflow/examples/minikube/custom-values.yaml b/stable/airflow/examples/minikube/custom-values.yaml index 0a70f7ed31..533d0b1a23 100644 --- a/stable/airflow/examples/minikube/custom-values.yaml +++ b/stable/airflow/examples/minikube/custom-values.yaml @@ -3,7 +3,7 @@ # - This is intended to be a `custom-values.yaml` starting point for non-production deployment (like minikube) # External Dependencies: -# - A git repo for DAGs: ssh://git@repo.example.com:my-airflow-dags.git +# - A PUBLIC git repo for DAGs: ssh://git@repo.example.com:my-airflow-dags.git # ################################### diff --git a/stable/airflow/templates/deployments-scheduler.yaml b/stable/airflow/templates/deployments-scheduler.yaml index 2a95d807e4..0c7b354dfa 100644 --- a/stable/airflow/templates/deployments-scheduler.yaml +++ b/stable/airflow/templates/deployments-scheduler.yaml @@ -267,13 +267,14 @@ spec: configMap: name: {{ include "airflow.fullname" . }}-scripts defaultMode: 0755 + {{- if .Values.dags.persistence.enabled }} - name: dags-data - {{- if .Values.dags.persistence.enabled }} persistentVolumeClaim: claimName: {{ .Values.dags.persistence.existingClaim | default (include "airflow.fullname" . ) }} - {{- else if or (.Values.dags.initContainer.enabled) (.Values.dags.git.gitSync.enabled) }} + {{- else if or (.Values.dags.initContainer.enabled) (.Values.dags.git.gitSync.enabled) }} + - name: dags-data emptyDir: {} - {{- end }} + {{- end }} {{- if .Values.logs.persistence.enabled }} - name: logs-data persistentVolumeClaim: diff --git a/stable/airflow/templates/deployments-web.yaml b/stable/airflow/templates/deployments-web.yaml index 239976a787..efcda37f45 100644 --- a/stable/airflow/templates/deployments-web.yaml +++ b/stable/airflow/templates/deployments-web.yaml @@ -260,13 +260,14 @@ spec: secret: secretName: {{ . }} {{- end }} + {{- if .Values.dags.persistence.enabled }} - name: dags-data - {{- if .Values.dags.persistence.enabled }} persistentVolumeClaim: - claimName: {{ .Values.dags.persistence.existingClaim | default (include "airflow.fullname" .) }} - {{- else if or (.Values.dags.initContainer.enabled) (.Values.dags.git.gitSync.enabled) }} + claimName: {{ .Values.dags.persistence.existingClaim | default (include "airflow.fullname" . ) }} + {{- else if or (.Values.dags.initContainer.enabled) (.Values.dags.git.gitSync.enabled) }} + - name: dags-data emptyDir: {} - {{- end }} + {{- end }} {{- if .Values.logs.persistence.enabled }} - name: logs-data persistentVolumeClaim: diff --git a/stable/airflow/templates/pdb-sheduler.yaml b/stable/airflow/templates/pdb-sheduler.yaml index 4cd2aafc07..79bfb6705d 100644 --- a/stable/airflow/templates/pdb-sheduler.yaml +++ b/stable/airflow/templates/pdb-sheduler.yaml @@ -11,7 +11,7 @@ metadata: heritage: {{ .Release.Service }} spec: {{- if .Values.scheduler.podDisruptionBudget.maxUnavailable }} - maxUnavailable: {{ .Values.scheduler.podDisruptionBudget.minAvailable }} + maxUnavailable: {{ .Values.scheduler.podDisruptionBudget.maxUnavailable }} {{- end }} {{- if .Values.scheduler.podDisruptionBudget.minAvailable }} minAvailable: {{ .Values.scheduler.podDisruptionBudget.minAvailable }} diff --git a/stable/airflow/templates/statefulsets-workers.yaml b/stable/airflow/templates/statefulsets-workers.yaml index bb806381f0..48ecddd4d3 100644 --- a/stable/airflow/templates/statefulsets-workers.yaml +++ b/stable/airflow/templates/statefulsets-workers.yaml @@ -239,13 +239,14 @@ spec: secret: secretName: {{ . }} {{- end }} + {{- if .Values.dags.persistence.enabled }} - name: dags-data - {{- if .Values.dags.persistence.enabled }} persistentVolumeClaim: - claimName: {{ .Values.dags.persistence.existingClaim | default (include "airflow.fullname" .) }} - {{- else if or (.Values.dags.initContainer.enabled) (.Values.dags.git.gitSync.enabled) }} + claimName: {{ .Values.dags.persistence.existingClaim | default (include "airflow.fullname" . ) }} + {{- else if or (.Values.dags.initContainer.enabled) (.Values.dags.git.gitSync.enabled) }} + - name: dags-data emptyDir: {} - {{- end }} + {{- end }} {{- if .Values.logs.persistence.enabled }} - name: logs-data persistentVolumeClaim: diff --git a/stable/airflow/values.yaml b/stable/airflow/values.yaml index 60c75c211a..d84cd2ff5c 100644 --- a/stable/airflow/values.yaml +++ b/stable/airflow/values.yaml @@ -27,21 +27,19 @@ airflow: ## executor: CeleryExecutor - ## the fernet key used to encrypt the connections in the database + ## the fernet key used to encrypt the connections/variables in the database ## ## WARNING: - ## - you MUST create a custom fernet key, otherwise your connections will not - ## necessarily be readable from all Pods + ## - you MUST customise this value, otherwise the encryption will be somewhat pointless + ## + ## NOTE: ## - to prevent this value being stored in your values.yaml (and airflow-env ConfigMap), ## consider using `airflow.extraEnv` to define it from a pre-created secret ## ## GENERATE: ## python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)" ## - ## EXAMPLE: - ## fernetKey: "7T512UXSSmBOkpWimFHIVb8jK6lfmSAvx4mO6Arehnc=" - ## - fernetKey: "" + fernetKey: "7T512UXSSmBOkpWimFHIVb8jK6lfmSAvx4mO6Arehnc=" ## environment variables for the web/scheduler/worker Pods (for airflow configs) ## @@ -210,7 +208,11 @@ scheduler: ## the maximum unavailable pods/percentage for the scheduler ## - maxUnavailable: "1" + ## NOTE: + ## - as there is only ever a single scheduler Pod, + ## this must be 100% for Kubernetes to be able to migrate it + ## + maxUnavailable: "100%" ## the minimum available pods/percentage for the scheduler ##