From 10c341f488e668a18beedffcee3afcd60afd0bcb Mon Sep 17 00:00:00 2001 From: Mathew Wicks Date: Mon, 25 May 2020 22:35:11 +1000 Subject: [PATCH] [stable/airflow] 7.1.0 release (#22506) Signed-off-by: Mathew Wicks --- stable/airflow/.helmignore | 22 +- stable/airflow/Chart.yaml | 2 +- stable/airflow/README.md | 244 +++------ stable/airflow/UPGRADE.md | 183 +++++++ .../examples/google-gke/custom-values.yaml | 472 ++++++++++++++++++ .../google-gke/k8s_resources/certificate.yaml | 13 + .../k8s_resources/configmap-webserver.yaml | 28 ++ .../k8s_resources/secret-fernet-key.yaml | 7 + .../k8s_resources/secret-git-keys.yaml | 38 ++ .../k8s_resources/secret-mysql-password.yaml | 7 + .../k8s_resources/secret-redis-password.yaml | 7 + stable/airflow/examples/minikube-values.yaml | 44 -- .../examples/minikube/custom-values.yaml | 154 ++++++ stable/airflow/templates/_helpers.tpl | 41 +- stable/airflow/templates/configmap-env.yaml | 19 +- ...-clone.yaml => configmap-scripts-git.yaml} | 20 +- .../airflow/templates/configmap-scripts.yaml | 23 +- .../airflow/templates/deployments-flower.yaml | 31 +- .../templates/deployments-scheduler.yaml | 51 +- stable/airflow/templates/deployments-web.yaml | 45 +- stable/airflow/templates/pvc-logs.yaml | 2 +- stable/airflow/templates/pvc.yaml | 2 +- .../templates/statefulsets-workers.yaml | 40 +- stable/airflow/values.yaml | 133 +++-- 24 files changed, 1275 insertions(+), 353 deletions(-) create mode 100644 stable/airflow/UPGRADE.md create mode 100644 stable/airflow/examples/google-gke/custom-values.yaml create mode 100644 stable/airflow/examples/google-gke/k8s_resources/certificate.yaml create mode 100644 stable/airflow/examples/google-gke/k8s_resources/configmap-webserver.yaml create mode 100644 stable/airflow/examples/google-gke/k8s_resources/secret-fernet-key.yaml create mode 100644 stable/airflow/examples/google-gke/k8s_resources/secret-git-keys.yaml create mode 100644 stable/airflow/examples/google-gke/k8s_resources/secret-mysql-password.yaml create mode 100644 stable/airflow/examples/google-gke/k8s_resources/secret-redis-password.yaml delete mode 100644 stable/airflow/examples/minikube-values.yaml create mode 100644 stable/airflow/examples/minikube/custom-values.yaml rename stable/airflow/templates/{configmap-git-clone.yaml => configmap-scripts-git.yaml} (83%) diff --git a/stable/airflow/.helmignore b/stable/airflow/.helmignore index 6b8710a711..f0c1319444 100644 --- a/stable/airflow/.helmignore +++ b/stable/airflow/.helmignore @@ -1 +1,21 @@ -.git +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/stable/airflow/Chart.yaml b/stable/airflow/Chart.yaml index aa2fdd468c..54838e10f8 100644 --- a/stable/airflow/Chart.yaml +++ b/stable/airflow/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Airflow is a platform to programmatically author, schedule and monitor workflows name: airflow -version: 7.0.1 +version: 7.1.0 appVersion: 1.10.10 icon: https://airflow.apache.org/_images/pin_large.png home: https://airflow.apache.org/ diff --git a/stable/airflow/README.md b/stable/airflow/README.md index a61f4cf8cf..213a5be688 100644 --- a/stable/airflow/README.md +++ b/stable/airflow/README.md @@ -2,11 +2,15 @@ [Airflow](https://airflow.apache.org/) is a platform to programmatically author, schedule and monitor workflows. -## Install Chart +## Installation To install the Airflow Helm Chart: ```bash -helm install --namespace "airflow" --name "airflow" stable/airflow +helm install stable/airflow \ + --version "X.X.X" \ + --name "airflow" \ + --namespace "airflow" \ + --values ./custom-values.yaml ``` To get the status of the Airflow Helm Chart: @@ -19,6 +23,37 @@ To uninstall the Airflow Helm Chart: helm delete "airflow" ``` +To run bash commands in the Airflow Scheduler Pod: +```bash +# use this to run commands like: `airflow create_user` +kubectl exec \ + -it \ + --namespace airflow \ + --container airflow-scheduler \ + Deployment/airflow-scheduler \ + /bin/bash +``` + +### Upgrade Steps: + +> NOTE: for chart version numbers, see [Chart.yaml](Chart.yaml) or [helm hub](https://hub.helm.sh/charts/stable/airflow). + +For steps you must take when upgrading this chart, please review: +* [v7.0.X → v7.1.0](UPGRADE.md#v70x--v710) +* [v6.X.X → v7.0.0](UPGRADE.md#v6xx--v700) +* [v5.X.X → v6.0.0](UPGRADE.md#v5xx--v600) +* [v4.X.X → v5.0.0](UPGRADE.md#v4xx--v500) +* [v3.X.X → v4.0.0](UPGRADE.md#v3xx--v400) + +### Examples: + +There are many ways to deploy this chart, but here are some starting points for your `custom-values.yaml`: + +| Name | File | Description | +| --- | --- | --- | +| (CeleryExecutor) Minimal | [examples/minikube/custom-values.yaml](examples/minikube/custom-values.yaml) | a __non-production__ starting point | +| (CeleryExecutor) Google Cloud | [examples/google-gke/custom-values.yaml](examples/google-gke/custom-values.yaml) | a __production__ starting point for GKE on Google Cloud | + ## Airflow-Configs ### Airflow-Configs/General @@ -46,19 +81,27 @@ airflow: AIRFLOW__SMTP__SMTP_SSL: "False" AIRFLOW__SMTP__SMTP_PORT: "25" AIRFLOW__SMTP__SMTP_MAIL_FROM: "admin@example.com" + + ## Disable noisy "Handling signal: ttou" Gunicorn log messages + GUNICORN_CMD_ARGS: "--log-level WARNING" ``` ### Airflow-Configs/Connections We expose the `scheduler.connections` value to allow specifying [Airflow Connections](https://airflow.apache.org/docs/stable/concepts.html#connections) at deployment time, these connections will be automatically imported by the Airflow scheduler when it starts up. -For example, to add an AWS connection: +For example, to add a connection called `my_aws`: ```yaml airflow: connections: - - id: my_aws - type: aws - extra: '{"aws_access_key_id": "**********", "aws_secret_access_key": "***", "region_name":"eu-central-1"}' + - id: my_aws + type: aws + extra: | + { + "aws_access_key_id": "XXXXXXXXXXXXXXXXXXX", + "aws_secret_access_key": "XXXXXXXXXXXXXXX", + "region_name":"eu-central-1" + } ``` __NOTE:__ As connections may include sensitive data, we store the bash script which generates the connections in a Kubernetes Secret, and mount this to the pods. @@ -69,7 +112,7 @@ __WARNING:__ Because some values are sensitive, you should take care to store yo We expose the `scheduler.variables` value to allow specifying [Airflow Variables](https://airflow.apache.org/docs/stable/concepts.html#variables) at deployment time, variables will be automatically imported by the Airflow scheduler when it starts up. -For example, to specify an `environment` variable: +For example, to specify a variable called `environment`: ```yaml airflow: variables: | @@ -80,13 +123,13 @@ airflow: We expose the `airflow.pools` value to allow specifying [Airflow Variables](https://airflow.apache.org/docs/stable/concepts.html#pools) at deployment time, these pools will be automatically imported by the Airflow scheduler when it starts up. -For example, to create a pool called `pool_1`: +For example, to create a pool called `example`: ```yaml airflow: pools: | { "example": { - "description": "This is an example of a pool", + "description": "This is an example pool with 2 slots.", "slots": 2 } } @@ -194,7 +237,7 @@ workers: # wait until all tasks are finished before SIGTERM of Pod gracefullTermination: true - # wait AT MOST 10min for a celery task to finish before SIGKILL of Pod + # wait AT MOST 10min for tasks on a worker to finish before SIGKILL terminationPeriod: 600 dags: @@ -246,11 +289,11 @@ kubectl create secret generic redshift-user --from-file=redshift-user=~/secrets/ Sometimes you may need to install extra pip packages for things to work, we provide `airflow.extraPipPackages` and `web.extraPipPackages` for this purpose. -For example, enabling the airflow `crypto` package: +For example, enabling the airflow `airflow-exporter` package: ```yaml airflow: extraPipPackages: - - "apache-airflow[crypto]==1.10.10" + - "airflow-exporter==1.3.1" ``` For example, you may be using `flask_oauthlib` to integrate with Okta/Google/etc for authorizing WebUI users: @@ -350,8 +393,7 @@ __WARNING:__ Airflow requires that `explicit_defaults_for_timestamp=1` in your M ### Other-Configs/Local-Binaries -Please note a folder `~/.local/bin` will be automatically created and added to the PATH so that -Bash operators can use command line tools installed by `pip install --user` for instance. +Please note a folder `~/.local/bin` will be automatically created and added to the PATH so that bash operators can use command line tools installed by `pip install --user`. ### Other-Configs/Logging @@ -474,7 +516,7 @@ __Global Values:__ | --- | --- | --- | | `airflow.image.*` | configs for the docker image of the web/scheduler/worker | `` | | `airflow.executor` | the airflow executor type to use | `CeleryExecutor` | -| `airflow.fernetKey` | the fernet key used to encrypt the database | `7T512UXSSmBOkpWimFHIVb8jK6lfmSAvx4mO6Arehnc=` | +| `airflow.fernetKey` | the fernet key used to encrypt the connections in the database | `""` | | `airflow.config` | environment variables for the web/scheduler/worker pods (for airflow configs) | `{}` | | `airflow.podAnnotations` | extra annotations for the web/scheduler/worker/flower Pods | `{}` | | `airflow.extraEnv` | extra environment variables for the web/scheduler/worker/flower Pods | `[]` | @@ -504,6 +546,7 @@ __Airflow Scheduler values:__ | `scheduler.numRuns` | the value of the `airflow --num_runs` parameter used to run the airflow scheduler | `-1` | | `scheduler.initdb` | if we run `airflow initdb` when the scheduler starts | `true` | | `scheduler.preinitdb` | if we run `airflow initdb` inside a special initContainer | `false` | +| `scheduler.initialStartupDelay` | the number of seconds to wait (in bash) before starting the scheduler container | `0` | | `scheduler.extraInitContainers` | extra init containers to run before the scheduler pod | `[]` | __Airflow WebUI Values:__ @@ -523,8 +566,8 @@ __Airflow WebUI Values:__ | `web.baseUrl` | sets `AIRFLOW__WEBSERVER__BASE_URL` | `http://localhost:8080` | | `web.serializeDAGs` | sets `AIRFLOW__CORE__STORE_SERIALIZED_DAGS` | `false` | | `web.extraPipPackages` | extra pip packages to install in the web container | `[]` | -| `web.initialStartupDelay` | the number of seconds to wait (in bash) before starting the web container | `60` | -| `web.minReadySeconds` | the number of seconds to wait before declaring a new Pod available | `120` | +| `web.initialStartupDelay` | the number of seconds to wait (in bash) before starting the web container | `0` | +| `web.minReadySeconds` | the number of seconds to wait before declaring a new Pod available | `5` | | `web.readinessProbe.*` | configs for the web Service readiness probe | `` | | `web.livenessProbe.*` | configs for the web Service liveness probe | `` | | `web.secretsDir` | the directory in which to mount secrets on web containers | `/var/airflow/secrets` | @@ -545,8 +588,9 @@ __Airflow Worker Values:__ | `workers.annotations` | annotations for the worker StatefulSet | `{}` | | `workers.podAnnotations` | Pod annotations for the worker StatefulSet | `{}` | | `workers.autoscaling.*` | configs for the HorizontalPodAutoscaler of the worker Pods | `` | +| `workers.initialStartupDelay` | the number of seconds to wait (in bash) before starting each worker container | `0` | | `workers.celery.*` | configs for the celery worker Pods | `` | -| `workers.terminationPeriod` | how many seconds before worker Pods are killed using SIGKILL | `60` | +| `workers.terminationPeriod` | how many seconds to wait for tasks on a worker to finish before SIGKILL | `60` | | `workers.secretsDir` | directory in which to mount secrets on worker containers | `/var/airflow/secrets` | | `workers.secrets` | secret names which will be mounted as a file at `{workers.secretsDir}/` | `[]` | @@ -564,6 +608,7 @@ __Airflow Flower Values:__ | `flower.podAnnotations` | Pod annotations for the flower Deployment | `{}` | | `flower.urlPrefix` | sets `AIRFLOW__CELERY__FLOWER_URL_PREFIX` | `""` | | `flower.service.*` | configs for the Service of the flower Pods | `` | +| `flower.initialStartupDelay` | the number of seconds to wait (in bash) before starting the flower container | `0` | | `flower.extraConfigmapMounts` | extra ConfigMaps to mount on the flower Pods | `[]` | __Airflow Logs Values:__ @@ -579,7 +624,7 @@ __Airflow DAGs Values:__ | --- | --- | --- | | `dags.path` | the airflow dags folder | `/opt/airflow/logs` | | `dags.doNotPickle` | whether to disable pickling dags from the scheduler to workers | `false` | -| `dags.installRequirements` | install any Python `requirements.txt` at the root of `dags.path` automatically | `true` | +| `dags.installRequirements` | install any Python `requirements.txt` at the root of `dags.path` automatically | `false` | | `dags.persistence.*` | configs for the dags PVC | `` | | `dags.git.*` | configs for the DAG git repository & sync container | `` | | `dags.initContainer.*` | configs for the git-clone container | `` | @@ -658,163 +703,4 @@ __Airflow Prometheus Values:__ | `serviceMonitor.interval` | the ServiceMonitor web endpoint path | `30s` | | `prometheusRule.enabled` | if the PrometheusRule resources should be deployed | `false` | | `prometheusRule.additionalLabels` | labels for PrometheusRule, so that Prometheus can select it | `{}` | -| `prometheusRule.groups` | alerting rules for Prometheus | `[]` | - -## Upgrading - -### To 7.0.0 -This version updates to Airflow 1.10.10, and moves to the official Airflow Docker images. - -You should no longer use images derived from `puckel/docker-airflow` and instead derive from `apache/airflow`. - -Due to the size of these changes, it may be easier to create a new `values.yaml`, starting from the one in this repo. - -The official image has a new `AIRFLOW_HOME`, you must change any references in your custom `values.yaml`: -* `AIRFLOW_HOME`: - * `/usr/local/airflow/` --> `/opt/airflow/` -* `dags.path`: - * `/usr/local/airflow/dags/` --> `/opt/airflow/dags/` -* `logs.path`: - * `/usr/local/airflow/logs/` --> `/opt/airflow/logs/` - -These internal mount paths have moved, you must update any references: -* `/usr/local/git/` --> `/home/airflow/git/` -* `/usr/local/scripts/` --> `/home/airflow/scripts/` -* `/usr/local/connections` --> `/home/airflow/connections/` -* `/usr/local/variables-pools/` --> `/home/airflow/variables-pools/` -* `/usr/local/airflow/.local/` --> `/home/airflow/.local/` - -The following values have been MOVED: -* `airflow.podDisruptionBudgetEnabled` --> `scheduler.podDisruptionBudget.enabled` -* `airflow.podDisruptionBudget.maxUnavailable` --> `scheduler.podDisruptionBudget.maxUnavailable` -* `airflow.podDisruptionBudget.minAvailable` --> `scheduler.podDisruptionBudget.minAvailable` -* `airflow.webReplicas` --> `web.replicas` -* `airflow.initdb` --> `scheduler.initdb` -* `airflow.preinitdb` --> `scheduler.preinitdb` -* `airflow.extraInitContainers` --> `scheduler.extraInitContainers` -* `airflow.schedulerNumRuns` --> `scheduler.numRuns` -* `airflow.connections` --> `scheduler.connections` -* `airflow.variables` --> `scheduler.variables` -* `airflow.pools` --> `scheduler.pools` -* `airflow.service.*` --> `web.service.*` -* `dags.initContainer.installRequirements` --> `dags.installRequirements` -* `logsPersistence.*` --> `logs.persistence.*` -* `persistence.*` --> `dags.persistence.*` - -The following values have been SPLIT: -* `web.initialDelaySeconds`: - * --> `web.readinessProbe.initialDelaySeconds` - * --> `web.livenessProbe.initialDelaySeconds` - -The following values have CHANGED BEHAVIOUR: -* `airflow.executor`: - * Previously you specified the executor name without the `Executor` suffix, now you must include it. - * For example: `Celery` --> `CeleryExecutor` -* `airflow.fernetKey`: - * Previously if omitted, this would be generated for you, we now have a default value, which we STRONGLY ENCOURAGE you to change. - * Also note, you should consider using `airflow.extraEnv` to prevent this value being stored in your `values.yaml` -* `dags.installRequirements`: - * Previously, `dags.installRequirements` only worked if `dags.initContainer.enabled` was true, now it will work regardless of other settings. - -The following values have NEW DEFAULTS: -* `dags.persistence.accessMode`: - * `ReadWriteOnce` --> `ReadOnlyMany` -* `logs.persistence.accessMode`: - * `ReadWriteOnce` --> `ReadWriteMany` - -The following values have been REMOVED: -* `postgresql.service.port`: - * As there is no reason to change the port of the embedded postgresql, and we have separated the external database configs. -* `redis.master.service.port`: - * As there is no reason to change the port of the embedded redis, and we have separated the external redis configs. - -The following values have been ADDED: -* `airflow.extraPipPackages`: - * Allows extra pip packages to be installed in the airflow-web/scheduler/worker containers. -* `web.extraPipPackages`: - * Allows extra pip packages to be installed in the airflow-web container only. - -Other changes: -* Special characters will now be correctly encoded in passwords for postgres/mysql/redis. - -If you are using an EXTERNAL postgres database, some configs have changed: - -| 6.x.x | 7.x.x | Notes | -| --- | --- | ---| -| `N/A` | `externalDatabase.type` | can choose `mysql` or `postgres` | -| `postgresql.postgresHost` | `externalDatabase.host` | | -| `postgresql.service.port` | `externalDatabase.port` | we no longer support changing the port of the embedded postgresql chart | -| `postgresql.postgresqlDatabase` | `externalDatabase.database` | | -| `postgresql.postgresqlUsername` | `externalDatabase.user` | | -| `postgresql.postgresqlPassword` | `N/A` | we don't support storing external database passwords in plain text | -| `postgresql.existingSecret` | `externalDatabase.passwordSecret` | | -| `postgresql.existingSecretKey` | `externalDatabase.passwordSecretKey` | | - -If you are using an EXTERNAL redis database, some configs have changed: - -| 6.x.x | 7.x.x | Notes | -| --- | --- | ---| -| `redis.redisHost` | `externalRedis.host` | | -| `redis.master.service.port` | `externalRedis.port` | we no longer support changing the port of the embedded redis chart | -| `redis.password` | `N/A` | we don't support storing external redis passwords in plain text | -| `N/A` | `externalRedis.databaseNumber` | changing the database number was not previously supported | -| `redis.existingSecret` | `externalRedis.passwordSecret` | | -| `redis.existingSecretKey` | `externalRedis.passwordSecretKey` | | - -### To 6.0.0 -This version updates `postgresql` and `redis` dependencies. - -There are a few Helm value changes, in order to upgrade from a 5.x chart, modify your `values.yaml` by mapping the keys as follows: - -| 5.x.x | 6.x.x | Notes | -| --- | --- | ---| -|`postgresql.postgresHost` |`postgresql.postgresqlHost` | | -|`postgresql.postgresUser` |`postgresql.postgresqlUsername` | | -|`postgresql.postgresPassword` |`postgresql.postgresqlPassword` | | -|`postgresql.postgresDatabase` |`postgresql.postgresqlDatabase` | | -|`postgresql.persistence.accessMode` |`postgresql.persistence.accessModes` | Instead of a single value, now the config accepts an array | -|`redis.master.persistence.accessMode` |`redis.master.persistence.accessModes` | Instead of a single value, now the config accepts an array | - -### To 5.0.0 -This version splits the configuration for webserver and flower web UI from ingress configurations for separation of concerns. - -Two new parameters: -* `web.baseUrl` -* `flower.urlPrefix` - -__WARNING:__ This upgrade will fail if a custom ingress path is set for web and/or flower and `web.baseUrl` and/or `flower.urlPrefix` - -### To 4.0.0 -This version splits the specs for the NodeSelector, Affinity and Toleration features. -Instead of being global, and injected in every component, they are now defined _by component_ to provide more flexibility for your deployments. -As such, the migration steps are really simple. -Just copy and paste your node/affinity/tolerance definitions in the four airflow components, which are `worker`, `scheduler`, `flower` and `web`. -The default values file should help you with locating those. - -### To 3.0.0 -This version introduces a simplified way of managing secrets, including the database credentials to postgres and redis. -With the default settings in prior versions, database credentials were generated and stored in an Airflow-managed Kubernetes secret. -However, these credentials were also stored in postgres- and redis-managed secrets (created by the respective subcharts), leading to duplication. -Moreover, it was tricky to bring your own passwords and to load additional secrets as environment variables. - -To deal with these issues, we've removed the Airflow-managed Kubernetes secret (`templates/secret-env.yaml`). -If your deployment was called `airflow`, this upgrade will delete the `airflow-env` secret. -Instead, the pods now source the database secrets from the postgres- and redis-managed secrets, i.e. the postgres password is in the `airflow-postgres` secret. -This upgrade _shouldn't_ break the deployment, but you may need to make some adjustments if you were doing something nonstandard. - -For production, it's better create random passwords before installing the Helm chart. -You can use these passwords by specifying the newly added `postgres.existingSecret` and `redis.existingSecret` parameters. - -We've also added `airflow.extraEnv`, which provides a flexible way to inject environment variables into your pods. -This parameter is great for things like the Fernet key and LDAP password. - -The following parameters are no longer necessary and have been removed: `airflow.defaultSecretsMapping`, `airflow.secretsMapping`, `airflow.existingAirflowSecret`. -If you were using them, you'll have to migrate your settings to `postgres.existingSecret`, `redis.existingSecret`, and `airflow.extraEnv`, which are described in greater depth in the documentation above. - -### To 2.8.3 -The parameter `airflow.service.type` no longer applies to the Flower service, but the default of `ClusterIP` has been maintained. -If using a custom values file and have changed the service type, also specify `flower.service.type`. - -### To 2.0.0 -The parameter `workers.pod.annotations` has been renamed to `workers.podAnnotations`. -If using a custom values file, rename this parameter. +| `prometheusRule.groups` | alerting rules for Prometheus | `[]` | \ No newline at end of file diff --git a/stable/airflow/UPGRADE.md b/stable/airflow/UPGRADE.md new file mode 100644 index 0000000000..2dd4e3b9e0 --- /dev/null +++ b/stable/airflow/UPGRADE.md @@ -0,0 +1,183 @@ +# Upgrading Steps + +## `v7.0.X` → `v7.1.0` + +__The following IMPROVEMENTS have been made:__ + +* We have dramatically reduced the start time of airflow pods. + This was mostly achieved by removing arbitrary delays in the start commands for airflow pods. + If you still want these delays, please set the added `*.initialStartupDelay` to non-zero values. +* We have improved support for when `airflow.executor` is set to `KubernetesExecutor`: + * redis configs/components are no longer deployed + * we now set `AIRFLOW__KUBERNETES__NAMESPACE`, `AIRFLOW__KUBERNETES__WORKER_SERVICE_ACCOUNT_NAME`, and `AIRFLOW__KUBERNETES__ENV_FROM_CONFIGMAP_REF` +* We have fixed an error caused by including a `'` in your redis/postgres/mysql password. +* We have reverted a change in 7.0.0 which prevented the use of airflow docker images with embedded DAGs. + (Just ensure that `dags.initContainer.enabled` and `git.gitSync.enabled` are `false`) +* The `AIRFLOW__CORE__SQL_ALCHEMY_CONN`, `AIRFLOW__CELERY__RESULT_BACKEND`, and `AIRFLOW__CELERY__BROKER_URL` environment variables are now available if you `kubectl exec ...` into airflow Pods. +* We have improved the script used when `workers.celery.gracefullTermination` is `true`. +* We have fixed an error with pools in `scheduler.pools` not being added to the scheduler. +* We have fixed an error with the `scheduler.preinitdb` container not knowing the database connection string. + +__The following values have CHANGED DEFAULTS:__ + +* `airflow.fernetKey`: + * Is now `""` by default, to enforce that users generate a custom one. + (However, please consider using `airflow.extraEnv` to define it from a pre-created secret) +* `dags.installRequirements`: + * Is now `false` by default, as this was an unintended change with the 7.0.0 upgrade. + +__The following values have been ADDED:__ + +* `scheduler.initialStartupDelay` +* `workers.initialStartupDelay` +* `flower.initialStartupDelay` +* `web.readinessProbe.enabled` +* `web.livenessProbe.enabled` + +## `v6.X.X` → `v7.0.0` + +> __WARNING:__ +> +> You MUST stop using images derived from `puckel/docker-airflow` and instead derive from `apache/airflow` + +This version updates to Airflow 1.10.10, and moves to the official Airflow Docker images. +Due to the size of these changes, it may be easier to create a new [values.yaml](values.yaml), starting from the one in this repo. + +__The official image has a new `AIRFLOW_HOME`, you must change any references in your custom `values.yaml`:__ + +| Variable | 6.x.x | 7.x.x | +| --- | --- | --- | +| `AIRFLOW_HOME` | `/usr/local/airflow` | `/opt/airflow` | +| `dags.path` | `/usr/local/airflow/dags` | `/opt/airflow/dags` | +| `logs.path` | `/usr/local/airflow/logs` | `/opt/airflow/logs` | + +__These internal mount paths have moved, you must update any references:__ + +| 6.x.x | 7.x.x | +| --- | --- | +| `/usr/local/git` | `/home/airflow/git` | +| `/usr/local/scripts` | `/home/airflow/scripts` | +| `/usr/local/connections` | `/home/airflow/connections` | +| `/usr/local/variables-pools` | `/home/airflow/variables-pools` | +| `/usr/local/airflow/.local` | `/home/airflow/.local` | + +__The following values have been MOVED:__ + +| 6.x.x | 7.x.x | +| --- | --- | +| `airflow.podDisruptionBudgetEnabled` | `scheduler.podDisruptionBudget.enabled` | +| `airflow.podDisruptionBudget.maxUnavailable` | `scheduler.podDisruptionBudget.maxUnavailable` | +| `airflow.podDisruptionBudget.minAvailable` | `scheduler.podDisruptionBudget.minAvailable` | +| `airflow.webReplicas` | `web.replicas` | +| `airflow.initdb` | `scheduler.initdb` | +| `airflow.preinitdb` | `scheduler.preinitdb` | +| `airflow.extraInitContainers` | `scheduler.extraInitContainers` | +| `airflow.schedulerNumRuns` | `scheduler.numRuns` | +| `airflow.connections` | `scheduler.connections` | +| `airflow.variables` | `scheduler.variables` | +| `airflow.pools` | `scheduler.pools` | +| `airflow.service.*` | `web.service.*` | +| `dags.initContainer.installRequirements` | `dags.installRequirements` | +| `logsPersistence.*` | `logs.persistence.*` | +| `persistence.*` | `dags.persistence.*` | + +__If you are using an EXTERNAL postgres database, some configs have changed:__ + +| 6.x.x | 7.x.x | Notes | +| --- | --- | ---| +| `N/A` | `externalDatabase.type` | can choose `mysql` or `postgres` | +| `postgresql.postgresHost` | `externalDatabase.host` | | +| `postgresql.service.port` | `externalDatabase.port` | we no longer support changing the port of the embedded postgresql chart | +| `postgresql.postgresqlDatabase` | `externalDatabase.database` | | +| `postgresql.postgresqlUsername` | `externalDatabase.user` | | +| `postgresql.postgresqlPassword` | `N/A` | we don't support storing external database passwords in plain text | +| `postgresql.existingSecret` | `externalDatabase.passwordSecret` | | +| `postgresql.existingSecretKey` | `externalDatabase.passwordSecretKey` | | + +__If you are using an EXTERNAL redis database, some configs have changed:__ + +| 6.x.x | 7.x.x | Notes | +| --- | --- | ---| +| `redis.redisHost` | `externalRedis.host` | | +| `redis.master.service.port` | `externalRedis.port` | we no longer support changing the port of the embedded redis chart | +| `redis.password` | `N/A` | we don't support storing external redis passwords in plain text | +| `N/A` | `externalRedis.databaseNumber` | changing the database number was not previously supported | +| `redis.existingSecret` | `externalRedis.passwordSecret` | | +| `redis.existingSecretKey` | `externalRedis.passwordSecretKey` | | + + +__The following values have been SPLIT:__ + +* `web.initialDelaySeconds`: + * --> `web.readinessProbe.initialDelaySeconds` + * --> `web.livenessProbe.initialDelaySeconds` + +__The following values have CHANGED BEHAVIOUR:__ + +* `airflow.executor`: + * Previously you specified the executor name without the `Executor` suffix, now you must include it. + * For example: `Celery` --> `CeleryExecutor` +* `airflow.fernetKey`: + * Previously if omitted, this would be generated for you, we now have a default value, which we STRONGLY ENCOURAGE you to change. + * Also note, you should consider using `airflow.extraEnv` to prevent this value being stored in your `values.yaml` +* `dags.installRequirements`: + * Previously, `dags.installRequirements` only worked if `dags.initContainer.enabled` was true, now it will work regardless of other settings. + +__The following values have NEW DEFAULTS:__ +* `dags.persistence.accessMode`: + * `ReadWriteOnce` --> `ReadOnlyMany` +* `logs.persistence.accessMode`: + * `ReadWriteOnce` --> `ReadWriteMany` + +__The following values have been REMOVED:__ + +* `postgresql.service.port`: + * As there is no reason to change the port of the embedded postgresql, and we have separated the external database configs. +* `redis.master.service.port`: + * As there is no reason to change the port of the embedded redis, and we have separated the external redis configs. + +__The following values have been ADDED:__ + +* `airflow.extraPipPackages`: + * Allows extra pip packages to be installed in the airflow-web/scheduler/worker containers. +* `web.extraPipPackages`: + * Allows extra pip packages to be installed in the airflow-web container only. + +__Other changes:__ + +* Special characters will now be correctly encoded in passwords for postgres/mysql/redis. + +## `v5.X.X` → `v6.0.0` + +This version updates `postgresql` and `redis` dependencies. + +__Thee following values have CHANGED:__ + +| 5.x.x | 6.x.x | Notes | +| --- | --- | ---| +|`postgresql.postgresHost` |`postgresql.postgresqlHost` | | +|`postgresql.postgresUser` |`postgresql.postgresqlUsername` | | +|`postgresql.postgresPassword` |`postgresql.postgresqlPassword` | | +|`postgresql.postgresDatabase` |`postgresql.postgresqlDatabase` | | +|`postgresql.persistence.accessMode` |`postgresql.persistence.accessModes` | Instead of a single value, now the config accepts an array | +|`redis.master.persistence.accessMode` |`redis.master.persistence.accessModes` | Instead of a single value, now the config accepts an array | + +## `v4.X.X` → `v5.0.0` + +> __WARNING:__ +> +> This upgrade will fail if a custom ingress path is set for web and/or flower and `web.baseUrl` and/or `flower.urlPrefix` + +This version splits the configuration for webserver and flower web UI from Ingress configurations, for separation of concerns. + +__The following values have been ADDED:__ + +* `web.baseUrl` +* `flower.urlPrefix` + +## `v3.X.X` → `v4.0.0` + +This version splits the specs for the NodeSelector, Affinity and Toleration features. +Instead of being global, and injected in every component, they are now defined _by component_ to provide more flexibility for your deployments. +As such, the migration steps are really simple, just ust copy and paste your node/affinity/tolerance definitions in the four airflow components, which are `worker`, `scheduler`, `flower` and `web`. +The default `values.yaml` file should help you with locating those. \ No newline at end of file diff --git a/stable/airflow/examples/google-gke/custom-values.yaml b/stable/airflow/examples/google-gke/custom-values.yaml new file mode 100644 index 0000000000..1f752009e6 --- /dev/null +++ b/stable/airflow/examples/google-gke/custom-values.yaml @@ -0,0 +1,472 @@ +# +# NOTE: +# - This is intended to be a `custom-values.yaml` starting point for production deployment in a GKE cluster +# - We are using GKE Workload Identity rather than storing Service Account JSON tokens: +# https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity +# - Airflow requires that `explicit_defaults_for_timestamp=1` in your CloudSQL MySQL instance + +# External Dependencies: +# - Git repo for DAGs: ssh://git@repo.example.com/my-airflow-dags.git +# - CloudSQL (MySQL): mysql.example.com:3306 +# - Cloud Storage Bucket: gs://XXXXXXXX--airflow-cluster1/ +# - SMTP server: smtpmail.example.com +# - DNS A Record: airflow-cluster1.example.com --> XXX.XXX.XXX.XXX +# - Google Service Account: airflow-cluster1@MY_PROJECT_ID.iam.gserviceaccount.com +# +# Google IAM: +# - (Storage Bucket) +# - gs://XXXXXXXX--airflow-cluster1 +# - roles/storage.objectAdmin --> serviceAccount:airflow-cluster1@$MY_PROJECT_NAME.iam.gserviceaccount.com +# - roles/storage.legacyBucketReader --> serviceAccount:airflow-cluster1@$MY_PROJECT_NAME.iam.gserviceaccount.com +# - (Service Account) +# - airflow-cluster1@MY_PROJECT_ID.iam.gserviceaccount.com +# - roles/iam.workloadIdentityUser --> MY_PROJECT_NAME.svc.id.goog[airflow-cluster1/airflow] +# +# Kubernetes Resources: (see: ./examples/google-gke/k8s_resources/) +# - Namespace: airflow-cluster1 +# - Secret: airflow-cluster1-fernet-key +# - Secret: airflow-cluster1-mysql-password +# - Secret: airflow-cluster1-redis-password +# - ConfigMap: airflow-cluster1-webserver-config +# - cert-manager.io/Certificate: airflow-cluster1-cert +# +# Helm Install Commands: +# helm install stable/airflow \ +# --version "X.X.X" \ +# --name "airflow-cluster1" \ +# --namespace "airflow-cluster1" \ +# --values ./custom-values.yaml +# +# Run bash commands in the Scheduler Pod: (use to: `airflow create_user`) +# kubectl exec \ +# -it \ +# --namespace airflow-cluster1 \ +# --container airflow-scheduler \ +# Deployment/airflow--airflow-cluster1-scheduler \ +# /bin/bash +# + +################################### +# Airflow - Common Configs +################################### +airflow: + ## the airflow executor type to use + ## + executor: CeleryExecutor + + ## environment variables for the web/scheduler/worker Pods (for airflow configs) + ## + config: + ## Security + AIRFLOW__CORE__SECURE_MODE: "True" + AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.deny_all" + AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "False" + AIRFLOW__WEBSERVER__RBAC: "True" + + ## SSL + ## NOTE: This effectively disables HTTP, so `web.readinessProbe.scheme` and `web.livenessProbe.scheme` + ## need to be set accordingly + AIRFLOW__WEBSERVER__WEB_SERVER_SSL_CERT: "/var/airflow/secrets/airflow-cluster1-cert/tls.crt" + AIRFLOW__WEBSERVER__WEB_SERVER_SSL_KEY: "/var/airflow/secrets/airflow-cluster1-cert/tls.key" + + ## DAGS + AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: "30" + + ## GCP Remote Logging + AIRFLOW__CORE__REMOTE_LOGGING: "True" + AIRFLOW__CORE__REMOTE_BASE_LOG_FOLDER: "gs://XXXXXXXX--airflow-cluster1/airflow/logs" + AIRFLOW__CORE__REMOTE_LOG_CONN_ID: "google_cloud_airflow" + + ## Email (SMTP) + AIRFLOW__EMAIL__EMAIL_BACKEND: "airflow.utils.email.send_email_smtp" + AIRFLOW__SMTP__SMTP_HOST: "smtpmail.example.com" + AIRFLOW__SMTP__SMTP_STARTTLS: "False" + AIRFLOW__SMTP__SMTP_SSL: "False" + AIRFLOW__SMTP__SMTP_PORT: "25" + AIRFLOW__SMTP__SMTP_MAIL_FROM: "admin@airflow-cluster1.example.com" + + ## Disable noisy "Handling signal: ttou" Gunicorn log messages + GUNICORN_CMD_ARGS: "--log-level WARNING" + + ## extra environment variables for the web/scheduler/worker (AND flower) Pods + ## + extraEnv: + - name: AIRFLOW__CORE__FERNET_KEY + valueFrom: + secretKeyRef: + name: airflow-cluster1-fernet-key + key: value + + ## extra configMap volumeMounts for the web/scheduler/worker Pods + ## + extraConfigmapMounts: + - name: airflow-cluster1-webserver-config + mountPath: /opt/airflow/webserver_config.py + configMap: airflow-cluster1-webserver-config + readOnly: true + subPath: webserver_config.py + +################################### +# Airflow - Scheduler Configs +################################### +scheduler: + ## resource requests/limits for the scheduler Pod + ## + resources: + requests: + cpu: "1000m" + memory: "1Gi" + + ## custom airflow connections for the airflow scheduler + ## + connections: + - id: google_cloud_airflow + type: google_cloud_platform + extra: '{"extra__google_cloud_platform__num_retries": "5"}' + + ## custom airflow pools for the airflow scheduler + ## + variables: | + { "environment": "prod" } + + ## custom airflow pools for the airflow scheduler + ## + pools: | + { + "example": { + "description": "This is an example pool with 2 slots.", + "slots": 2 + } + } + +################################### +# Airflow - WebUI Configs +################################### +web: + ## resource requests/limits for the airflow web Pods + ## + resources: + requests: + cpu: "500m" + memory: "1Gi" + + ## the number of web Pods to run + ## + replicas: 1 + + ## configs for the Service of the web Pods + ## + service: + annotations: + cloud.google.com/load-balancer-type: "Internal" + type: LoadBalancer + externalPort: 443 + loadBalancerIP: XXX.XXX.XXX.XXX + loadBalancerSourceRanges: [] + + ## sets `AIRFLOW__WEBSERVER__BASE_URL` + ## + baseUrl: "https://airflow-cluster1.example.com/" + + ## extra pip packages to install in the web container + ## + extraPipPackages: [] + + ## configs for the web Service liveness probe + ## + livenessProbe: + ## the scheme used in the liveness probe: {HTTP,HTTPS} + ## + scheme: HTTPS + + ## the number of seconds to wait before checking pod health + ## + ## NOTE: + ## - make larger if you are installing many packages with: + ## `airflow.extraPipPackages`, `web.extraPipPackages`, or `dags.installRequirements` + ## + initialDelaySeconds: 300 + + ## the directory in which to mount secrets on web containers + ## + secretsDir: /var/airflow/secrets + + ## secret names which will be mounted as a file at `{web.secretsDir}/` + ## + secrets: + - airflow-cluster1-cert + +################################### +# Airflow - Worker Configs +################################### +workers: + ## if the airflow workers StatefulSet should be deployed + ## + enabled: true + + ## resource requests/limits for the airflow worker Pods + ## + resources: + requests: + cpu: "1000m" + memory: "2Gi" + + ## the number of workers Pods to run + ## + replicas: 2 + + ## configs for the HorizontalPodAutoscaler of the worker Pods + ## + autoscaling: + enabled: true + maxReplicas: 8 + metrics: + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 + + ## configs for the celery worker Pods + ## + celery: + ## the number of tasks each celery worker can run at a time + ## + ## NOTE: + ## - sets AIRFLOW__CELERY__WORKER_CONCURRENCY + ## + instances: 10 + + ## if we should wait for tasks to finish on a celery worker before SIGTERM of Pod + ## + gracefullTermination: true + + ## how many seconds to wait for tasks on a worker to finish before SIGKILL + ## + terminationPeriod: 600 + + ## directory in which to mount secrets on worker containers + ## + secretsDir: /var/airflow/secrets + + ## secret names which will be mounted as a file at `{workers.secretsDir}/` + ## + secrets: [] + +################################### +# Airflow - Flower Configs +################################### +flower: + ## if the Flower UI should be deployed + ## + enabled: true + + ## resource requests/limits for the flower Pods + ## + resources: + requests: + cpu: "100m" + memory: "126Mi" + + ## configs for the Service of the flower Pods + ## + service: + annotations: {} + type: ClusterIP + externalPort: 5555 + loadBalancerIP: "" + loadBalancerSourceRanges: [] + +################################### +# Airflow - Logs Configs +################################### +logs: + ## configs for the logs PVC + ## + persistence: + ## if a persistent volume is mounted at `logs.path` + ## + enabled: false + +################################### +# Airflow - DAGs Configs +################################### +dags: + ## configs for the DAG git repository & sync container + ## + git: + ## url of the git repository + ## + url: "ssh://git@repo.example.com/my-airflow-dags.git" + + ## the branch/tag/sha1 which we clone + ## + ref: master + + ## the name of a pre-created secret containing files for ~/.ssh/ + ## + ## NOTE: + ## - this is ONLY RELEVANT for SSH git repos + ## - the secret commonly includes files: id_rsa, id_rsa.pub, known_hosts + ## - known_hosts is NOT NEEDED if `git.sshKeyscan` is true + ## + secret: airflow-cluster1-git-keys + + ## the name of the private key file in your `git.secret` + ## + ## NOTE: + ## - this is ONLY RELEVANT for PRIVATE SSH git repos + ## + privateKeyName: id_rsa + + ## the host name of the git repo + ## + ## NOTE: + ## - this is ONLY REQUIRED for SSH git repos + ## + repoHost: "repo.example.com" + + ## the port of the git repo + ## + ## NOTE: + ## - this is ONLY REQUIRED for SSH git repos + ## + repoPort: 22 + + ## configs for the git-sync container + ## + gitSync: + ## enable the git-sync sidecar container + ## + enabled: true + + ## the git sync interval in seconds + ## + refreshTime: 60 + +################################### +# Kubernetes - RBAC +################################### +rbac: + ## if Kubernetes RBAC resources are created + ## + create: true + +################################### +# Kubernetes - Service Account +################################### +serviceAccount: + ## if a Kubernetes ServiceAccount is created + ## + create: true + + ## the name of the ServiceAccount + ## + name: "airflow" + + ## annotations for the ServiceAccount + ## + annotations: + iam.gke.io/gcp-service-account: airflow-cluster1@MY_PROJECT_ID.iam.gserviceaccount.com + +################################### +# Database - PostgreSQL Chart +################################### +postgresql: + ## if the `stable/postgresql` chart is used + ## + enabled: false + +################################### +# Database - External Database +################################### +externalDatabase: + ## the type of external database: {mysql,postgres} + ## + type: mysql + + ## the host of the external database + ## + host: mysql.airflow-cluster1.example.com + + ## the port of the external database + ## + port: 3306 + + ## the database/scheme to use within the the external database + ## + database: airflow_cluster1 + + ## the user of the external database + ## + user: airflow_cluster1 + + ## the name of a pre-created secret containing the external database password + ## + passwordSecret: airflow-cluster1-mysql-password + + ## the key within `externalDatabase.passwordSecret` containing the password string + ## + passwordSecretKey: mysql-password + +################################### +# Database - Redis Chart +################################### +redis: + ## if the `stable/redis` chart is used + ## + enabled: true + + ## the name of a pre-created secret containing the redis password + ## + existingSecret: "airflow-cluster1-redis-password" + + ## the key in `redis.existingSecret` containing the password string + ## + existingSecretKey: "redis-password" + + ## configs for redis cluster mode + ## + cluster: + ## if redis runs in cluster mode + ## + enabled: false + + ## the number of redis slaves + ## + slaveCount: 1 + + ## configs for the redis master + ## + master: + ## resource requests/limits for the master Pod + ## + resources: + requests: + cpu: "100m" + memory: "256Mi" + + ## configs for the PVC of the redis master + ## + persistence: + ## use a PVC to persist data + ## + enabled: false + + ## configs for the redis slaves + ## + slave: + ## resource requests/limits for the slave Pods + ## + resources: + requests: + cpu: "100m" + memory: "256Mi" + + ## configs for the PVC of the redis slaves + ## + persistence: + ## use a PVC to persist data + ## + enabled: false \ No newline at end of file diff --git a/stable/airflow/examples/google-gke/k8s_resources/certificate.yaml b/stable/airflow/examples/google-gke/k8s_resources/certificate.yaml new file mode 100644 index 0000000000..7c58af1ec8 --- /dev/null +++ b/stable/airflow/examples/google-gke/k8s_resources/certificate.yaml @@ -0,0 +1,13 @@ +apiVersion: cert-manager.io/v1alpha2 +kind: Certificate +metadata: + name: airflow-cluster1-cert + namespace: airflow-cluster1 +spec: + secretName: airflow-cluster1-cert + commonName: "airflow-cluster1.example.com" + dnsNames: + - "airflow-cluster1.example.com" + issuerRef: + kind: ClusterIssuer + name: letsencrypt-issuer \ No newline at end of file diff --git a/stable/airflow/examples/google-gke/k8s_resources/configmap-webserver.yaml b/stable/airflow/examples/google-gke/k8s_resources/configmap-webserver.yaml new file mode 100644 index 0000000000..7fc8dcec63 --- /dev/null +++ b/stable/airflow/examples/google-gke/k8s_resources/configmap-webserver.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: airflow-cluster1-webserver-config + namespace: airflow-cluster1 +data: + webserver_config.py: | + import os + from airflow.configuration import conf + from flask_appbuilder.security.manager import AUTH_DB + + basedir = os.path.abspath(os.path.dirname(__file__)) + + # The SQLAlchemy connection string. + SQLALCHEMY_DATABASE_URI = conf.get("core", "SQL_ALCHEMY_CONN") + + # Flask-WTF flag for CSRF + CSRF_ENABLED = True + + # Force users to re-auth after 15min of inactivity + PERMANENT_SESSION_LIFETIME = 900 + + # Don't allow user self registration + AUTH_USER_REGISTRATION = False + AUTH_USER_REGISTRATION_ROLE = "Viewer" + + # Use Database authentication + AUTH_TYPE = AUTH_DB \ No newline at end of file diff --git a/stable/airflow/examples/google-gke/k8s_resources/secret-fernet-key.yaml b/stable/airflow/examples/google-gke/k8s_resources/secret-fernet-key.yaml new file mode 100644 index 0000000000..bda9f88fee --- /dev/null +++ b/stable/airflow/examples/google-gke/k8s_resources/secret-fernet-key.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: airflow-cluster1-fernet-key + namespace: airflow-cluster1 +stringData: + value: "7T512UXSSmBOkpWimFHIVb8jK6lfmSAvx4mO6Arehnc=" \ No newline at end of file diff --git a/stable/airflow/examples/google-gke/k8s_resources/secret-git-keys.yaml b/stable/airflow/examples/google-gke/k8s_resources/secret-git-keys.yaml new file mode 100644 index 0000000000..5d3c132802 --- /dev/null +++ b/stable/airflow/examples/google-gke/k8s_resources/secret-git-keys.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Secret +metadata: + name: airflow-cluster1-git-keys + namespace: airflow-cluster1 +stringData: + git_key: | + -----BEGIN OPENSSH PRIVATE KEY----- + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + -----END OPENSSH PRIVATE KEY----- + git_key.pub: | + ssh-rsa XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX airflow-cluster1@gke-cluster + known_hosts: | + repo.example.com, ssh-rsa XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX diff --git a/stable/airflow/examples/google-gke/k8s_resources/secret-mysql-password.yaml b/stable/airflow/examples/google-gke/k8s_resources/secret-mysql-password.yaml new file mode 100644 index 0000000000..019f1c449b --- /dev/null +++ b/stable/airflow/examples/google-gke/k8s_resources/secret-mysql-password.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: airflow-cluster1-mysql-password + namespace: airflow-cluster1 +stringData: + mysql-password: "XXXXXXXXXXXXXXXXXXXXXXX" \ No newline at end of file diff --git a/stable/airflow/examples/google-gke/k8s_resources/secret-redis-password.yaml b/stable/airflow/examples/google-gke/k8s_resources/secret-redis-password.yaml new file mode 100644 index 0000000000..1fd20a8732 --- /dev/null +++ b/stable/airflow/examples/google-gke/k8s_resources/secret-redis-password.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: airflow-cluster1-redis-password + namespace: airflow-cluster1 +stringData: + redis-password: "XXXXXXXXXXXXXXXXXXXXXXX" \ No newline at end of file diff --git a/stable/airflow/examples/minikube-values.yaml b/stable/airflow/examples/minikube-values.yaml deleted file mode 100644 index 15905a7c4f..0000000000 --- a/stable/airflow/examples/minikube-values.yaml +++ /dev/null @@ -1,44 +0,0 @@ -airflow: - image: - repository: apache/airflow - tag: 1.10.10-python3.6 - pullPolicy: IfNotPresent - - config: - AIRFLOW__CORE__LOGGING_LEVEL: DEBUG - AIRFLOW__CORE__LOAD_EXAMPLES: True - -web: - service: - type: NodePort - -scheduler: - # note, this is a string - variables: | - { "environment": "dev" } - - # note, this is a string - pools: | - { - "example": { - "description": "This is an example of a pool", - "slots": 2 - } - } - -workers: - replicas: 1 - celery: - instances: 1 - -dags: - persistence: - enabled: true - accessMode: ReadWriteMany - size: 1Gi - -postgresql: - enabled: true - -redis: - enabled: true \ No newline at end of file diff --git a/stable/airflow/examples/minikube/custom-values.yaml b/stable/airflow/examples/minikube/custom-values.yaml new file mode 100644 index 0000000000..0a70f7ed31 --- /dev/null +++ b/stable/airflow/examples/minikube/custom-values.yaml @@ -0,0 +1,154 @@ +# +# NOTE: +# - This is intended to be a `custom-values.yaml` starting point for non-production deployment (like minikube) + +# External Dependencies: +# - A git repo for DAGs: ssh://git@repo.example.com:my-airflow-dags.git +# + +################################### +# Airflow - Common Configs +################################### +airflow: + ## the airflow executor type to use + ## + executor: CeleryExecutor + + ## the fernet key used to encrypt the connections in the database + ## + fernetKey: "7T512UXSSmBOkpWimFHIVb8jK6lfmSAvx4mO6Arehnc=" + + ## environment variables for the web/scheduler/worker Pods (for airflow configs) + ## + config: + # Security + AIRFLOW__CORE__SECURE_MODE: "True" + AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.deny_all" + AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "False" + AIRFLOW__WEBSERVER__RBAC: "False" + + # DAGS + AIRFLOW__CORE__LOAD_EXAMPLES: "True" + + ## Disable noisy "Handling signal: ttou" Gunicorn log messages + GUNICORN_CMD_ARGS: "--log-level WARNING" + +################################### +# Airflow - Scheduler Configs +################################### +scheduler: + + ## custom airflow connections for the airflow scheduler + ## + connections: + - id: my_aws + type: aws + extra: | + { + "aws_access_key_id": "XXXXXXXXXXXXXXXXXXX", + "aws_secret_access_key": "XXXXXXXXXXXXXXX", + "region_name":"eu-central-1" + } + + ## custom airflow pools for the airflow scheduler + ## + variables: | + { "environment": "dev" } + + ## custom airflow pools for the airflow scheduler + ## + pools: | + { + "example": { + "description": "This is an example pool with 2 slots.", + "slots": 2 + } + } + +################################### +# Airflow - WebUI Configs +################################### +web: + ## configs for the Service of the web Pods + ## + service: + type: NodePort + +################################### +# Airflow - Worker Configs +################################### +workers: + ## the number of workers Pods to run + ## + replicas: 1 + +################################### +# Airflow - DAGs Configs +################################### +dags: + ## configs for the DAG git repository & sync container + ## + git: + ## url of the git repository + ## + url: "ssh://git@repo.example.com/my-airflow-dags.git" + + ## the branch/tag/sha1 which we clone + ## + ref: master + + ## the name of a pre-created secret containing files for ~/.ssh/ + ## + ## NOTE: + ## - this is ONLY RELEVANT for SSH git repos + ## - the secret commonly includes files: id_rsa, id_rsa.pub, known_hosts + ## - known_hosts is NOT NEEDED if `git.sshKeyscan` is true + ## + secret: "" + + ## the name of the private key file in your `git.secret` + ## + ## NOTE: + ## - this is ONLY RELEVANT for PRIVATE SSH git repos + ## + privateKeyName: id_rsa + + ## the host name of the git repo + ## + ## NOTE: + ## - this is ONLY REQUIRED for SSH git repos + ## + ## EXAMPLE: + ## repoHost: "github.com" + ## + repoHost: "repo.example.com" + + ## the port of the git repo + ## + ## NOTE: + ## - this is ONLY REQUIRED for SSH git repos + ## + repoPort: 22 + + ## configs for the git-sync container + ## + gitSync: + ## enable the git-sync sidecar container + ## + enabled: true + + ## the git sync interval in seconds + ## + refreshTime: 60 + +################################### +# Database - PostgreSQL Chart +################################### +postgresql: + enabled: true + +################################### +# Database - Redis Chart +################################### +redis: + enabled: true \ No newline at end of file diff --git a/stable/airflow/templates/_helpers.tpl b/stable/airflow/templates/_helpers.tpl index 6f7d830b07..47269db08c 100644 --- a/stable/airflow/templates/_helpers.tpl +++ b/stable/airflow/templates/_helpers.tpl @@ -73,11 +73,11 @@ Construct the AIRFLOW__CORE__SQL_ALCHEMY_CONN connection string. */}} {{- define "airflow.connection.alchemy" -}} {{- if .Values.postgresql.enabled -}} -postgresql+psycopg2://${DATABASE_USER}:$( python -c "import urllib.parse; encoded_pass = urllib.parse.quote('''${DATABASE_PASSWORD}'''); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB} +postgresql+psycopg2://${DATABASE_USER}:$( echo ${DATABASE_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB} {{- else if and (not .Values.postgresql.enabled) (eq "postgres" .Values.externalDatabase.type) -}} -postgresql+psycopg2://${DATABASE_USER}:$( python -c "import urllib.parse; encoded_pass = urllib.parse.quote('''${DATABASE_PASSWORD}'''); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB} +postgresql+psycopg2://${DATABASE_USER}:$( echo ${DATABASE_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB} {{- else if and (not .Values.postgresql.enabled) (eq "mysql" .Values.externalDatabase.type) -}} -mysql+mysqldb://${DATABASE_USER}:$( python -c "import urllib.parse; encoded_pass = urllib.parse.quote('''${DATABASE_PASSWORD}'''); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB} +mysql+mysqldb://${DATABASE_USER}:$( echo ${DATABASE_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB} {{- end -}} {{- end -}} @@ -86,11 +86,11 @@ Construct the AIRFLOW__CELERY__RESULT_BACKEND connection string. */}} {{- define "airflow.connection.celery.backend" -}} {{- if .Values.postgresql.enabled -}} -db+postgresql://${DATABASE_USER}:$( python -c "import urllib.parse; encoded_pass = urllib.parse.quote('''${DATABASE_PASSWORD}'''); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB} +db+postgresql://${DATABASE_USER}:$( echo ${DATABASE_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB} {{- else if and (not .Values.postgresql.enabled) (eq "postgres" .Values.externalDatabase.type) -}} -db+postgresql://${DATABASE_USER}:$( python -c "import urllib.parse; encoded_pass = urllib.parse.quote('''${DATABASE_PASSWORD}'''); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB} +db+postgresql://${DATABASE_USER}:$( echo ${DATABASE_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB} {{- else if and (not .Values.postgresql.enabled) (eq "mysql" .Values.externalDatabase.type) -}} -db+mysql://${DATABASE_USER}:$( python -c "import urllib.parse; encoded_pass = urllib.parse.quote('''${DATABASE_PASSWORD}'''); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB} +db+mysql://${DATABASE_USER}:$( echo ${DATABASE_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(encoded_pass)" )@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_DB} {{- end -}} {{- end -}} @@ -99,7 +99,7 @@ Construct the AIRFLOW__CELERY__BROKER_URL connection string. Note that a redis URL with no password excludes the "@" prefix. */}} {{- define "airflow.connection.celery.broker" -}} -redis://$( python -c "import urllib.parse; encoded_pass = urllib.parse.quote('''${REDIS_PASSWORD}'''); print(':{}@'.format(encoded_pass)) if len(encoded_pass) > 0 else None" )${REDIS_HOST}:${REDIS_PORT}/${REDIS_DBNUM} +redis://$( echo ${REDIS_PASSWORD} | python3 -c "import urllib.parse; encoded_pass = urllib.parse.quote(input()); print(f\":{encoded_pass}@\") if len(encoded_pass) > 0 else None" )${REDIS_HOST}:${REDIS_PORT}/${REDIS_DBNUM} {{- end -}} {{/* @@ -107,6 +107,9 @@ Construct a set of secret environment variables to be mounted in web, scheduler, When applicable, we use the secrets created by the postgres/redis charts (which have fixed names and secret keys). */}} {{- define "airflow.mapenvsecrets" -}} +{{- /* ------------------------------ */ -}} +{{- /* ---------- POSTGRES ---------- */ -}} +{{- /* ------------------------------ */ -}} {{- if .Values.postgresql.enabled }} {{- if .Values.postgresql.existingSecret }} - name: DATABASE_PASSWORD @@ -122,14 +125,21 @@ When applicable, we use the secrets created by the postgres/redis charts (which key: postgresql-password {{- end }} {{- else }} +{{- if .Values.externalDatabase.passwordSecret }} - name: DATABASE_PASSWORD valueFrom: secretKeyRef: name: {{ .Values.externalDatabase.passwordSecret }} key: {{ .Values.externalDatabase.passwordSecretKey }} +{{- else }} +- name: DATABASE_PASSWORD + value: "" {{- end }} - -{{- if .Values.redis.enabled }} +{{- end }} +{{- /* --------------------------- */ -}} +{{- /* ---------- REDIS ---------- */ -}} +{{- /* --------------------------- */ -}} +{{- if and (.Values.redis.enabled) (eq .Values.airflow.executor "CeleryExecutor") }} {{- if .Values.redis.existingSecret }} - name: REDIS_PASSWORD valueFrom: @@ -143,15 +153,22 @@ When applicable, we use the secrets created by the postgres/redis charts (which name: {{ include "airflow.redis.fullname" . }} key: redis-password {{- end }} -{{- else }} +{{- else if (eq .Values.airflow.executor "CeleryExecutor") }} +{{- if .Values.externalRedis.passwordSecret }} - name: REDIS_PASSWORD valueFrom: secretKeyRef: name: {{ .Values.externalRedis.passwordSecret }} key: {{ .Values.externalRedis.passwordSecretKey }} +{{- else }} +- name: REDIS_PASSWORD + value: "" {{- end }} +{{- end }} +{{- /* ---------------------------- */ -}} +{{- /* ---------- EXTRAS ---------- */ -}} +{{- /* ---------------------------- */ -}} {{- if .Values.airflow.extraEnv }} {{ toYaml .Values.airflow.extraEnv }} {{- end }} - -{{- end -}} +{{- end }} \ No newline at end of file diff --git a/stable/airflow/templates/configmap-env.yaml b/stable/airflow/templates/configmap-env.yaml index 186160f37a..a86b1d48d0 100644 --- a/stable/airflow/templates/configmap-env.yaml +++ b/stable/airflow/templates/configmap-env.yaml @@ -11,7 +11,7 @@ data: ## Force UTC timezone TZ: Etc/UTC - ## Airflow (Database) + ## Database {{- if .Values.postgresql.enabled }} DATABASE_HOST: "{{ include "airflow.postgresql.fullname" . }}" DATABASE_PORT: "5432" @@ -24,7 +24,8 @@ data: DATABASE_DB: "{{ .Values.externalDatabase.database }}" {{- end }} - ## Airflow (Redis) + {{- if (eq .Values.airflow.executor "CeleryExecutor") }} + ## Redis {{- if .Values.redis.enabled }} REDIS_HOST: "{{ include "airflow.redis.fullname" . }}-master" REDIS_PORT: "6379" @@ -38,6 +39,20 @@ data: ## Airflow (Flower) AIRFLOW__CELERY__FLOWER_URL_PREFIX: "{{ .Values.flower.urlPrefix }}" AIRFLOW__CELERY__WORKER_CONCURRENCY: "{{ .Values.workers.celery.instances }}" + {{- end }} + + {{- if (eq .Values.airflow.executor "KubernetesExecutor") }} + ## Airflow (Kubernetes) + {{- if (not .Values.airflow.config.AIRFLOW__KUBERNETES__NAMESPACE) }} + AIRFLOW__KUBERNETES__NAMESPACE: "{{ .Release.Namespace }}" + {{- end }} + {{- if (not .Values.airflow.config.AIRFLOW__KUBERNETES__WORKER_SERIVCE_ACCOUNT_NAME) }} + AIRFLOW__KUBERNETES__WORKER_SERVICE_ACCOUNT_NAME: "{{ include "airflow.serviceAccountName" . }}" + {{- end }} + {{- if (not .Values.airflow.config.AIRFLOW__KUBERNETES__ENV_FROM_CONFIGMAP_REF) }} + AIRFLOW__KUBERNETES__ENV_FROM_CONFIGMAP_REF: "{{ include "airflow.fullname" . }}-env" + {{- end }} + {{- end }} ## Airflow (Logs) AIRFLOW__CORE__BASE_LOG_FOLDER: "{{ .Values.logs.path }}" diff --git a/stable/airflow/templates/configmap-git-clone.yaml b/stable/airflow/templates/configmap-scripts-git.yaml similarity index 83% rename from stable/airflow/templates/configmap-git-clone.yaml rename to stable/airflow/templates/configmap-scripts-git.yaml index a42c225605..ebb341e1cf 100644 --- a/stable/airflow/templates/configmap-git-clone.yaml +++ b/stable/airflow/templates/configmap-scripts-git.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: {{ include "airflow.fullname" . }}-git-clone + name: {{ include "airflow.fullname" . }}-scripts-git labels: app: {{ include "airflow.labels.app" . }} chart: {{ include "airflow.labels.chart" . }} @@ -16,20 +16,24 @@ data: REPO_HOST=$4 REPO_PORT=$5 PRIVATE_KEY=$6 + mkdir -p ~/.ssh/ + {{- if .Values.dags.git.sshKeyscan }} ssh-keyscan -p $REPO_PORT $REPO_HOST >> ~/.ssh/known_hosts {{- end }} + {{- if .Values.dags.git.secret }} cp -rL /keys/* ~/.ssh/ chmod 600 ~/.ssh/* echo -e "Host $REPO_HOST\n Port $REPO_PORT\n IdentityFile ~/.ssh/$PRIVATE_KEY" > ~/.ssh/config {{- end }} - # Init Containers will re-run on Pod restart. - # Remove the directory's contents and reprovision when this happens. + + # ensure the git directory is empty, so we can safely clone if [ -d "$DIR" ]; then rm -rf $( find $DIR -mindepth 1 ) fi + git clone $REPO -b $REF $DIR git-sync.sh: | #!/bin/sh -e @@ -40,21 +44,29 @@ data: REPO_PORT=$5 PRIVATE_KEY=$6 SYNC_TIME=$7 + mkdir -p ~/.ssh/ + {{- if .Values.dags.git.sshKeyscan }} ssh-keyscan -p $REPO_PORT $REPO_HOST >> ~/.ssh/known_hosts {{- end }} + {{- if .Values.dags.git.secret }} cp -rL /keys/* ~/.ssh/ chmod 600 ~/.ssh/* echo -e "Host $REPO_HOST\n Port $REPO_PORT\n IdentityFile ~/.ssh/$PRIVATE_KEY" > ~/.ssh/config {{- end }} - {{- if and ( .Values.dags.git.gitSync.enabled ) (not .Values.dags.initContainer.enabled ) }} + + {{- if and (.Values.dags.git.gitSync.enabled) (not .Values.dags.initContainer.enabled) }} if [ -d "$DIR" ]; then rm -rf $( find $DIR -mindepth 1 ) fi git clone $REPO -b $REF $DIR {{- end }} + + # to break the infinite loop when we receive SIGTERM + trap "exit 0" SIGTERM + cd $DIR while true; do git fetch origin $REF; diff --git a/stable/airflow/templates/configmap-scripts.yaml b/stable/airflow/templates/configmap-scripts.yaml index 592d5dbf56..579c428579 100644 --- a/stable/airflow/templates/configmap-scripts.yaml +++ b/stable/airflow/templates/configmap-scripts.yaml @@ -10,7 +10,6 @@ metadata: data: install-requirements.sh: | #!/bin/bash -e - if [ ! -d {{ .Values.dags.path | quote }} ]; then echo 'No folder {{ .Values.dags.path | quote }}' exit 0 @@ -22,19 +21,27 @@ data: else exit 0 fi - stop-worker.sh: | + graceful-stop-celery-worker.sh: | #!/bin/bash -e + echo "*** starting graceful worker shutdown" - celery -b $AIRFLOW__CELERY__BROKER_URL -d celery@$HOSTNAME control cancel_consumer default + # source the required environment variables + source /home/airflow/airflow_env.sh - # wait 10 second before checking the status of the worker - sleep 10 + # prevent the worker accepting new tasks + echo "*** preventing worker accepting new tasks" + celery control --broker $AIRFLOW__CELERY__BROKER_URL --destination celery@$HOSTNAME cancel_consumer default + sleep 5 - while (( $(celery -b $AIRFLOW__CELERY__BROKER_URL inspect active --json | python -c "import sys, json; print(len(json.load(sys.stdin)['celery@$HOSTNAME']))") > 0 )); do - sleep 60 + # loop until all active task are finished + echo "*** waiting for active tasks to finish" + while (( celery inspect --broker $AIRFLOW__CELERY__BROKER_URL --destination celery@$HOSTNAME --json active | python3 -c "import json; active_tasks = json.loads(input())['celery@$HOSTNAME']; print(len(active_tasks))" > 0 )); do + sleep 30 done preinit-db.sh: | - #!/bin/bash -e + #!/bin/bash + # export the required environment variables + export AIRFLOW__CORE__SQL_ALCHEMY_CONN="{{ include "airflow.connection.alchemy" . }}" echo "*** Waiting 10s for postgres" sleep 10 diff --git a/stable/airflow/templates/deployments-flower.yaml b/stable/airflow/templates/deployments-flower.yaml index 02ff4f1db0..8f4885ec33 100644 --- a/stable/airflow/templates/deployments-flower.yaml +++ b/stable/airflow/templates/deployments-flower.yaml @@ -1,4 +1,4 @@ -{{- if .Values.flower.enabled }} +{{- if and (.Values.flower.enabled) (eq .Values.airflow.executor "CeleryExecutor") }} apiVersion: apps/v1 kind: Deployment metadata: @@ -91,24 +91,33 @@ spec: - "-c" - > true \ - && echo "*** waiting 60s..." \ - && sleep 60 \ + {{- if gt .Values.flower.initialStartupDelay 0.0 }} + && echo "*** waiting {{ .Values.flower.initialStartupDelay }}s..." \ + && sleep {{ .Values.flower.initialStartupDelay }} \ + {{- end }} && mkdir -p /home/airflow/.local/bin \ - && export PATH=/home/airflow/.local/bin:$PATH \ - && export AIRFLOW__CORE__SQL_ALCHEMY_CONN="{{ include "airflow.connection.alchemy" . }}" \ - && export AIRFLOW__CELERY__RESULT_BACKEND="{{ include "airflow.connection.celery.backend" . }}" \ - && export AIRFLOW__CELERY__BROKER_URL="{{ include "airflow.connection.celery.broker" . }}" \ + && echo 'export PATH="/home/airflow/.local/bin:$PATH"' >> /home/airflow/airflow_env.sh \ + && echo 'export AIRFLOW__CORE__SQL_ALCHEMY_CONN="{{ include "airflow.connection.alchemy" . }}"' >> /home/airflow/airflow_env.sh \ + && echo 'export AIRFLOW__CELERY__RESULT_BACKEND="{{ include "airflow.connection.celery.backend" . }}"' >> /home/airflow/airflow_env.sh \ + && echo 'export AIRFLOW__CELERY__BROKER_URL="{{ include "airflow.connection.celery.broker" . }}"' >> /home/airflow/airflow_env.sh \ + && echo 'source /home/airflow/airflow_env.sh' >> /home/airflow/.profile \ + && echo 'source /home/airflow/airflow_env.sh' >> /home/airflow/.bashrc \ + && source /home/airflow/airflow_env.sh \ && echo "*** running flower..." \ - && airflow flower + && exec airflow flower livenessProbe: httpGet: - path: "{{ .Values.ingress.flower.livenessPath }}/" + {{- if .Values.ingress.flower.livenessPath }} + path: "{{ .Values.ingress.flower.livenessPath }}" + {{- else }} + path: "{{ .Values.ingress.flower.path }}/" + {{- end }} port: flower initialDelaySeconds: 60 - periodSeconds: 60 + periodSeconds: 30 timeoutSeconds: 1 successThreshold: 1 - failureThreshold: 5 + failureThreshold: 3 resources: {{- toYaml .Values.flower.resources | nindent 12 }} {{- end }} diff --git a/stable/airflow/templates/deployments-scheduler.yaml b/stable/airflow/templates/deployments-scheduler.yaml index 2ed567d553..2a95d807e4 100644 --- a/stable/airflow/templates/deployments-scheduler.yaml +++ b/stable/airflow/templates/deployments-scheduler.yaml @@ -33,12 +33,12 @@ spec: metadata: annotations: checksum/config-env: {{ include (print $.Template.BasePath "/configmap-env.yaml") . | sha256sum }} - checksum/config-git-clone: {{ include (print $.Template.BasePath "/configmap-git-clone.yaml") . | sha256sum }} + checksum/config-git-clone: {{ include (print $.Template.BasePath "/configmap-scripts-git.yaml") . | sha256sum }} checksum/config-scripts: {{ include (print $.Template.BasePath "/configmap-scripts.yaml") . | sha256sum }} checksum/config-variables-pools: {{ include (print $.Template.BasePath "/configmap-variables-pools.yaml") . | sha256sum }} checksum/secret-connections: {{ include (print $.Template.BasePath "/secret-connections.yaml") . | sha256sum }} - {{- if and ( .Values.dags.git.url ) ( .Values.dags.git.ref ) }} - checksum/dags-git-ref: {{ .Values.dags.git.ref }} + {{- if and (.Values.dags.git.url) (.Values.dags.git.ref) }} + checksum/dags-git-ref: {{ .Values.dags.git.ref | sha256sum }} {{- end }} {{- if .Values.airflow.podAnnotations }} {{- toYaml .Values.airflow.podAnnotations | nindent 8 }} @@ -90,8 +90,8 @@ spec: - "-c" - "/home/airflow/scripts/preinit-db.sh" envFrom: - - configMapRef: - name: "{{ include "airflow.fullname" . }}-env" + - configMapRef: + name: "{{ include "airflow.fullname" . }}-env" env: {{- include "airflow.mapenvsecrets" . | indent 12 }} resources: @@ -140,8 +140,8 @@ spec: image: {{ .Values.dags.git.gitSync.image.repository }}:{{ .Values.dags.git.gitSync.image.tag }} imagePullPolicy: {{ .Values.dags.git.gitSync.image.pullPolicy }} envFrom: - - configMapRef: - name: "{{ include "airflow.fullname" . }}-env" + - configMapRef: + name: "{{ include "airflow.fullname" . }}-env" env: {{- include "airflow.mapenvsecrets" . | indent 12 }} command: @@ -170,8 +170,8 @@ spec: image: {{ .Values.airflow.image.repository }}:{{ .Values.airflow.image.tag }} imagePullPolicy: {{ .Values.airflow.image.pullPolicy}} envFrom: - - configMapRef: - name: "{{ include "airflow.fullname" . }}-env" + - configMapRef: + name: "{{ include "airflow.fullname" . }}-env" env: {{- include "airflow.mapenvsecrets" . | indent 12 }} resources: @@ -183,7 +183,7 @@ spec: - name: dags-data mountPath: {{ .Values.dags.path }} subPath: {{ .Values.dags.persistence.subPath }} - {{- else }} + {{- else if or (.Values.dags.initContainer.enabled) (.Values.dags.git.gitSync.enabled) }} - name: dags-data mountPath: {{ .Values.dags.path }} {{- end }} @@ -196,7 +196,7 @@ spec: - name: connections mountPath: /home/airflow/connections {{- end}} - {{- if or .Values.scheduler.variables .Values.scheduler.pools }} + {{- if or (.Values.scheduler.variables) (.Values.scheduler.pools) }} - name: variables-pools mountPath: /home/airflow/variables-pools/ {{- end}} @@ -219,13 +219,20 @@ spec: - "-c" - > true \ - && echo "*** waiting 10s..." \ - && sleep 10 \ + {{- if gt .Values.scheduler.initialStartupDelay 0.0 }} + && echo "*** waiting {{ .Values.scheduler.initialStartupDelay }}s..." \ + && sleep {{ .Values.scheduler.initialStartupDelay }} \ + {{- end }} && mkdir -p /home/airflow/.local/bin \ - && export PATH=/home/airflow/.local/bin:$PATH \ - && export AIRFLOW__CORE__SQL_ALCHEMY_CONN="{{ include "airflow.connection.alchemy" . }}" \ - && export AIRFLOW__CELERY__RESULT_BACKEND="{{ include "airflow.connection.celery.backend" . }}" \ - && export AIRFLOW__CELERY__BROKER_URL="{{ include "airflow.connection.celery.broker" . }}" \ + && echo 'export PATH="/home/airflow/.local/bin:$PATH"' >> /home/airflow/airflow_env.sh \ + && echo 'export AIRFLOW__CORE__SQL_ALCHEMY_CONN="{{ include "airflow.connection.alchemy" . }}"' >> /home/airflow/airflow_env.sh \ + {{- if (eq .Values.airflow.executor "CeleryExecutor") }} + && echo 'export AIRFLOW__CELERY__RESULT_BACKEND="{{ include "airflow.connection.celery.backend" . }}"' >> /home/airflow/airflow_env.sh \ + && echo 'export AIRFLOW__CELERY__BROKER_URL="{{ include "airflow.connection.celery.broker" . }}"' >> /home/airflow/airflow_env.sh \ + {{- end }} + && echo 'source /home/airflow/airflow_env.sh' >> /home/airflow/.profile \ + && echo 'source /home/airflow/airflow_env.sh' >> /home/airflow/.bashrc \ + && source /home/airflow/airflow_env.sh \ {{- if .Values.dags.installRequirements }} && echo "*** installing requirements..." \ && /home/airflow/scripts/install-requirements.sh \ @@ -246,12 +253,12 @@ spec: && echo "*** adding Airflow connections..." \ && /home/airflow/connections/add-connections.sh \ {{- end }} - {{- if .Values.airflow.scheduler }} + {{- if .Values.scheduler.pools }} && echo "*** adding Airflow pools..." \ && airflow pool -i /home/airflow/variables-pools/pools.json \ {{- end }} && echo "*** running scheduler..." \ - && airflow scheduler -n {{ .Values.scheduler.numRuns }} + && exec airflow scheduler -n {{ .Values.scheduler.numRuns }} {{- if .Values.airflow.extraContainers }} {{- toYaml .Values.airflow.extraContainers | nindent 8 }} {{- end }} @@ -264,7 +271,7 @@ spec: {{- if .Values.dags.persistence.enabled }} persistentVolumeClaim: claimName: {{ .Values.dags.persistence.existingClaim | default (include "airflow.fullname" . ) }} - {{- else }} + {{- else if or (.Values.dags.initContainer.enabled) (.Values.dags.git.gitSync.enabled) }} emptyDir: {} {{- end }} {{- if .Values.logs.persistence.enabled }} @@ -275,7 +282,7 @@ spec: {{- if or (.Values.dags.initContainer.enabled) (.Values.dags.git.gitSync.enabled) }} - name: git-clone configMap: - name: {{ include "airflow.fullname" . }}-git-clone + name: {{ include "airflow.fullname" . }}-scripts-git defaultMode: 0755 {{- if .Values.dags.git.secret }} - name: git-clone-secret @@ -290,7 +297,7 @@ spec: secretName: {{ include "airflow.fullname" . }}-connections defaultMode: 0755 {{- end }} - {{- if or .Values.scheduler.variables .Values.scheduler.pools }} + {{- if or (.Values.scheduler.variables) (.Values.scheduler.pools) }} - name: variables-pools configMap: name: {{ include "airflow.fullname" . }}-variables-pools diff --git a/stable/airflow/templates/deployments-web.yaml b/stable/airflow/templates/deployments-web.yaml index a878ed361a..239976a787 100644 --- a/stable/airflow/templates/deployments-web.yaml +++ b/stable/airflow/templates/deployments-web.yaml @@ -33,10 +33,10 @@ spec: metadata: annotations: checksum/config-env: {{ include (print $.Template.BasePath "/configmap-env.yaml") . | sha256sum }} - checksum/config-git-clone: {{ include (print $.Template.BasePath "/configmap-git-clone.yaml") . | sha256sum }} + checksum/config-git-clone: {{ include (print $.Template.BasePath "/configmap-scripts-git.yaml") . | sha256sum }} checksum/config-scripts: {{ include (print $.Template.BasePath "/configmap-scripts.yaml") . | sha256sum }} - {{- if and ( .Values.dags.git.url ) ( .Values.dags.git.ref ) }} - checksum/dags-git-ref: {{ .Values.dags.git.ref }} + {{- if and (.Values.dags.git.url) (.Values.dags.git.ref) }} + checksum/dags-git-ref: {{ .Values.dags.git.ref | sha256sum }} {{- end }} {{- if .Values.airflow.podAnnotations }} {{- toYaml .Values.airflow.podAnnotations | nindent 8 }} @@ -80,8 +80,8 @@ spec: image: {{ .Values.dags.initContainer.image.repository }}:{{ .Values.dags.initContainer.image.tag }} imagePullPolicy: {{ .Values.dags.initContainer.image.pullPolicy }} envFrom: - - configMapRef: - name: "{{ include "airflow.fullname" . }}-env" + - configMapRef: + name: "{{ include "airflow.fullname" . }}-env" env: {{- include "airflow.mapenvsecrets" . | indent 12 }} resources: @@ -111,8 +111,8 @@ spec: image: {{ .Values.dags.git.gitSync.image.repository }}:{{ .Values.dags.git.gitSync.image.tag }} imagePullPolicy: {{ .Values.dags.git.gitSync.image.pullPolicy }} envFrom: - - configMapRef: - name: "{{ include "airflow.fullname" . }}-env" + - configMapRef: + name: "{{ include "airflow.fullname" . }}-env" env: {{- include "airflow.mapenvsecrets" . | indent 12 }} command: @@ -163,7 +163,7 @@ spec: - name: dags-data mountPath: {{ .Values.dags.path }} subPath: {{ .Values.dags.persistence.subPath }} - {{- else }} + {{- else if or (.Values.dags.initContainer.enabled) (.Values.dags.git.gitSync.enabled) }} - name: dags-data mountPath: {{ .Values.dags.path }} {{- end }} @@ -191,13 +191,20 @@ spec: - "-c" - > true \ + {{- if gt .Values.web.initialStartupDelay 0.0 }} && echo "*** waiting {{ .Values.web.initialStartupDelay }}s..." \ && sleep {{ .Values.web.initialStartupDelay }} \ + {{- end }} && mkdir -p /home/airflow/.local/bin \ - && export PATH=/home/airflow/.local/bin:$PATH \ - && export AIRFLOW__CORE__SQL_ALCHEMY_CONN="{{ include "airflow.connection.alchemy" . }}" \ - && export AIRFLOW__CELERY__RESULT_BACKEND="{{ include "airflow.connection.celery.backend" . }}" \ - && export AIRFLOW__CELERY__BROKER_URL="{{ include "airflow.connection.celery.broker" . }}" \ + && echo 'export PATH="/home/airflow/.local/bin:$PATH"' >> /home/airflow/airflow_env.sh \ + && echo 'export AIRFLOW__CORE__SQL_ALCHEMY_CONN="{{ include "airflow.connection.alchemy" . }}"' >> /home/airflow/airflow_env.sh \ + {{- if (eq .Values.airflow.executor "CeleryExecutor") }} + && echo 'export AIRFLOW__CELERY__RESULT_BACKEND="{{ include "airflow.connection.celery.backend" . }}"' >> /home/airflow/airflow_env.sh \ + && echo 'export AIRFLOW__CELERY__BROKER_URL="{{ include "airflow.connection.celery.broker" . }}"' >> /home/airflow/airflow_env.sh \ + {{- end }} + && echo 'source /home/airflow/airflow_env.sh' >> /home/airflow/.profile \ + && echo 'source /home/airflow/airflow_env.sh' >> /home/airflow/.bashrc \ + && source /home/airflow/airflow_env.sh \ {{- if .Values.dags.installRequirements }} && echo "*** installing requirements..." \ && /home/airflow/scripts/install-requirements.sh \ @@ -211,7 +218,8 @@ spec: && pip install --user {{ range .Values.web.extraPipPackages }} {{ . | quote }} {{ end }} \ {{- end }} && echo "*** running webserver..." \ - && airflow webserver + && exec airflow webserver + {{- if .Values.web.livenessProbe.enabled }} livenessProbe: httpGet: scheme: {{ .Values.web.livenessProbe.scheme }} @@ -226,6 +234,8 @@ spec: timeoutSeconds: {{ .Values.web.livenessProbe.timeoutSeconds }} successThreshold: {{ .Values.web.livenessProbe.successThreshold }} failureThreshold: {{ .Values.web.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.web.readinessProbe.enabled }} readinessProbe: httpGet: scheme: {{ .Values.web.readinessProbe.scheme }} @@ -236,6 +246,7 @@ spec: timeoutSeconds: {{ .Values.web.readinessProbe.timeoutSeconds }} successThreshold: {{ .Values.web.readinessProbe.successThreshold }} failureThreshold: {{ .Values.web.readinessProbe.failureThreshold }} + {{- end }} {{- if .Values.airflow.extraContainers }} {{- toYaml .Values.airflow.extraContainers | nindent 8 }} {{- end }} @@ -250,12 +261,12 @@ spec: secretName: {{ . }} {{- end }} - name: dags-data - {{- if .Values.dags.persistence.enabled }} + {{- if .Values.dags.persistence.enabled }} persistentVolumeClaim: claimName: {{ .Values.dags.persistence.existingClaim | default (include "airflow.fullname" .) }} - {{- else }} + {{- else if or (.Values.dags.initContainer.enabled) (.Values.dags.git.gitSync.enabled) }} emptyDir: {} - {{- end }} + {{- end }} {{- if .Values.logs.persistence.enabled }} - name: logs-data persistentVolumeClaim: @@ -264,7 +275,7 @@ spec: {{- if or (.Values.dags.initContainer.enabled) (.Values.dags.git.gitSync.enabled) }} - name: git-clone configMap: - name: {{ include "airflow.fullname" . }}-git-clone + name: {{ include "airflow.fullname" . }}-scripts-git defaultMode: 0755 {{- if .Values.dags.git.secret }} - name: git-clone-secret diff --git a/stable/airflow/templates/pvc-logs.yaml b/stable/airflow/templates/pvc-logs.yaml index 32e8448144..c5bb7178ba 100644 --- a/stable/airflow/templates/pvc-logs.yaml +++ b/stable/airflow/templates/pvc-logs.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.logs.persistence.enabled (not .Values.logs.persistence.existingClaim) }} +{{- if and (.Values.logs.persistence.enabled) (not .Values.logs.persistence.existingClaim) }} kind: PersistentVolumeClaim apiVersion: v1 metadata: diff --git a/stable/airflow/templates/pvc.yaml b/stable/airflow/templates/pvc.yaml index c239ddd0ab..7f9f463c61 100644 --- a/stable/airflow/templates/pvc.yaml +++ b/stable/airflow/templates/pvc.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.dags.persistence.enabled (not .Values.dags.persistence.existingClaim) }} +{{- if and (.Values.dags.persistence.enabled) (not .Values.dags.persistence.existingClaim) }} kind: PersistentVolumeClaim apiVersion: v1 metadata: diff --git a/stable/airflow/templates/statefulsets-workers.yaml b/stable/airflow/templates/statefulsets-workers.yaml index 1ff772f80c..bb806381f0 100644 --- a/stable/airflow/templates/statefulsets-workers.yaml +++ b/stable/airflow/templates/statefulsets-workers.yaml @@ -36,8 +36,11 @@ spec: metadata: annotations: checksum/config-env: {{ include (print $.Template.BasePath "/configmap-env.yaml") . | sha256sum }} - checksum/config-git-clone: {{ include (print $.Template.BasePath "/configmap-git-clone.yaml") . | sha256sum }} + checksum/config-git-clone: {{ include (print $.Template.BasePath "/configmap-scripts-git.yaml") . | sha256sum }} checksum/config-scripts: {{ include (print $.Template.BasePath "/configmap-scripts.yaml") . | sha256sum }} + {{- if and (.Values.dags.git.url) (.Values.dags.git.ref) }} + checksum/dags-git-ref: {{ .Values.dags.git.ref | sha256sum }} + {{- end }} {{- if .Values.airflow.podAnnotations }} {{- toYaml .Values.airflow.podAnnotations | nindent 8 }} {{- end }} @@ -141,11 +144,11 @@ spec: - name: {{ .Chart.Name }}-worker imagePullPolicy: {{ .Values.airflow.image.pullPolicy }} image: "{{ .Values.airflow.image.repository }}:{{ .Values.airflow.image.tag }}" - {{- if and (eq .Values.airflow.executor "Celery") (.Values.workers.celery.gracefullTermination)}} + {{- if and (eq .Values.airflow.executor "CeleryExecutor") (.Values.workers.celery.gracefullTermination) }} lifecycle: preStop: exec: - command: ["/home/airflow/scripts/stop-worker.sh"] + command: ["/home/airflow/scripts/graceful-stop-celery-worker.sh"] {{- end}} envFrom: - configMapRef: @@ -165,7 +168,7 @@ spec: - name: dags-data mountPath: {{ .Values.dags.path }} subPath: {{ .Values.dags.persistence.subPath }} - {{- else }} + {{- else if or (.Values.dags.initContainer.enabled) (.Values.dags.git.gitSync.enabled) }} - name: dags-data mountPath: {{ .Values.dags.path }} {{- end }} @@ -193,13 +196,20 @@ spec: - "-c" - > true \ - && echo "*** waiting 60s..." \ - && sleep 60 \ + {{- if gt .Values.workers.initialStartupDelay 0.0 }} + && echo "*** waiting {{ .Values.workers.initialStartupDelay }}s..." \ + && sleep {{ .Values.workers.initialStartupDelay }} \ + {{- end }} && mkdir -p /home/airflow/.local/bin \ - && export PATH=/home/airflow/.local/bin:$PATH \ - && export AIRFLOW__CORE__SQL_ALCHEMY_CONN="{{ include "airflow.connection.alchemy" . }}" \ - && export AIRFLOW__CELERY__RESULT_BACKEND="{{ include "airflow.connection.celery.backend" . }}" \ - && export AIRFLOW__CELERY__BROKER_URL="{{ include "airflow.connection.celery.broker" . }}" \ + && echo 'export PATH="/home/airflow/.local/bin:$PATH"' >> /home/airflow/airflow_env.sh \ + && echo 'export AIRFLOW__CORE__SQL_ALCHEMY_CONN="{{ include "airflow.connection.alchemy" . }}"' >> /home/airflow/airflow_env.sh \ + {{- if (eq .Values.airflow.executor "CeleryExecutor") }} + && echo 'export AIRFLOW__CELERY__RESULT_BACKEND="{{ include "airflow.connection.celery.backend" . }}"' >> /home/airflow/airflow_env.sh \ + && echo 'export AIRFLOW__CELERY__BROKER_URL="{{ include "airflow.connection.celery.broker" . }}"' >> /home/airflow/airflow_env.sh \ + {{- end }} + && echo 'source /home/airflow/airflow_env.sh' >> /home/airflow/.profile \ + && echo 'source /home/airflow/airflow_env.sh' >> /home/airflow/.bashrc \ + && source /home/airflow/airflow_env.sh \ {{- if .Values.dags.installRequirements }} && echo "*** installing requirements..." \ && /home/airflow/scripts/install-requirements.sh \ @@ -209,7 +219,7 @@ spec: && pip install --user {{ range .Values.airflow.extraPipPackages }} {{ . | quote }} {{ end }} \ {{- end }} && echo "*** running scheduler..." \ - && airflow worker + && exec airflow worker ports: - name: wlog containerPort: 8793 @@ -230,12 +240,12 @@ spec: secretName: {{ . }} {{- end }} - name: dags-data - {{- if .Values.dags.persistence.enabled }} + {{- if .Values.dags.persistence.enabled }} persistentVolumeClaim: claimName: {{ .Values.dags.persistence.existingClaim | default (include "airflow.fullname" .) }} - {{- else }} + {{- else if or (.Values.dags.initContainer.enabled) (.Values.dags.git.gitSync.enabled) }} emptyDir: {} - {{- end }} + {{- end }} {{- if .Values.logs.persistence.enabled }} - name: logs-data persistentVolumeClaim: @@ -244,7 +254,7 @@ spec: {{- if or (.Values.dags.initContainer.enabled) (.Values.dags.git.gitSync.enabled) }} - name: git-clone configMap: - name: {{ include "airflow.fullname" . }}-git-clone + name: {{ include "airflow.fullname" . }}-scripts-git defaultMode: 0755 {{- if .Values.dags.git.secret }} - name: git-clone-secret diff --git a/stable/airflow/values.yaml b/stable/airflow/values.yaml index 83e9753fdd..60c75c211a 100644 --- a/stable/airflow/values.yaml +++ b/stable/airflow/values.yaml @@ -14,23 +14,42 @@ airflow: ## the airflow executor type to use ## ## NOTE: - ## - this should almost always be `CeleryExecutor` + ## - this should be `CeleryExecutor` or `KubernetesExecutor` + ## - if set to `KubernetesExecutor`: + ## - ensure that `workers.enabled` is `false` + ## - ensure that `flower.enabled` is `false` + ## - ensure that `redis.enabled` is `false` + ## - ensure that K8S configs are set in `airflow.config` + ## - we set these configs automatically: + ## - `AIRFLOW__KUBERNETES__NAMESPACE` + ## - `AIRFLOW__KUBERNETES__WORKER_SERVICE_ACCOUNT_NAME` + ## - `AIRFLOW__KUBERNETES__ENV_FROM_CONFIGMAP_REF` ## executor: CeleryExecutor - ## the fernet key used to encrypt the database + ## the fernet key used to encrypt the connections in the database ## ## WARNING: - ## - to prevent this value being stored in your values.yaml and airflow-env ConfigMap, - ## consider using an `airflow.extraEnv` to define it from a pre-created secret + ## - you MUST create a custom fernet key, otherwise your connections will not + ## necessarily be readable from all Pods + ## - to prevent this value being stored in your values.yaml (and airflow-env ConfigMap), + ## consider using `airflow.extraEnv` to define it from a pre-created secret ## ## GENERATE: ## python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)" ## - fernetKey: "7T512UXSSmBOkpWimFHIVb8jK6lfmSAvx4mO6Arehnc=" + ## EXAMPLE: + ## fernetKey: "7T512UXSSmBOkpWimFHIVb8jK6lfmSAvx4mO6Arehnc=" + ## + fernetKey: "" ## environment variables for the web/scheduler/worker Pods (for airflow configs) ## + ## WARNING: + ## - don't include sensitive variables in here, instead make use of `airflow.extraEnv` with Secrets + ## - don't specify `AIRFLOW__CORE__SQL_ALCHEMY_CONN`, `AIRFLOW__CELERY__RESULT_BACKEND`, + ## or `AIRFLOW__CELERY__BROKER_URL`, they are dynamically created from chart values + ## ## NOTE: ## - airflow allows environment configs to be set as environment variables ## - they take the form: AIRFLOW__
__ @@ -38,17 +57,17 @@ airflow: ## ## EXAMPLE: ## config: - ## # Security + ## ## Security ## AIRFLOW__CORE__SECURE_MODE: "True" ## AIRFLOW__API__AUTH_BACKEND: "airflow.api.auth.backend.deny_all" ## AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "False" ## AIRFLOW__WEBSERVER__RBAC: "False" ## - ## # DAGS + ## ## DAGS ## AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: "30" ## AIRFLOW__CORE__LOAD_EXAMPLES: "False" ## - ## # Email (SMTP) + ## ## Email (SMTP) ## AIRFLOW__EMAIL__EMAIL_BACKEND: "airflow.utils.email.send_email_smtp" ## AIRFLOW__SMTP__SMTP_HOST: "smtpmail.example.com" ## AIRFLOW__SMTP__SMTP_STARTTLS: "False" @@ -56,7 +75,10 @@ airflow: ## AIRFLOW__SMTP__SMTP_PORT: "25" ## AIRFLOW__SMTP__SMTP_MAIL_FROM: "admin@example.com" ## - ## # Proxy Config + ## ## Disable noisy "Handling signal: ttou" Gunicorn log messages + ## GUNICORN_CMD_ARGS: "--log-level WARNING" + ## + ## ## Proxy Config ## HTTP_PROXY: "http://proxy.example.com:8080" ## config: {} @@ -112,9 +134,9 @@ airflow: ## extra pip packages to install in the web/scheduler/worker Pods ## - ## EXAMPLE: ( enable airflow crypto package ) + ## EXAMPLE: ## extraPipPackages: - ## - "apache-airflow[crypto]==1.10.10" + ## - "airflow-exporter==1.3.1" ## extraPipPackages: [] @@ -202,8 +224,13 @@ scheduler: ## EXAMPLE: ## connections: ## - id: my_aws - ## type: aws - ## extra: '{"aws_access_key_id": "**********", "aws_secret_access_key": "***", "region_name":"eu-central-1"}' + ## type: aws + ## extra: | + ## { + ## "aws_access_key_id": "XXXXXXXXXXXXXXXXXXX", + ## "aws_secret_access_key": "XXXXXXXXXXXXXXX", + ## "region_name":"eu-central-1" + ## } ## connections: [] @@ -228,7 +255,7 @@ scheduler: ## pools: | ## { ## "example": { - ## "description": "This is an example of a pool", + ## "description": "This is an example pool with 2 slots.", ## "slots": 2 ## } ## } @@ -240,8 +267,8 @@ scheduler: ## ## NOTE: ## - this is the number of 'dag refreshes' before the airflow scheduler process will exit - ## - if not set to -1, you will see your scheduler regularly restart - ## - for most environments, -1 will be a suitable value + ## - if not set to `-1`, the scheduler Pod will restart regularly + ## - for most environments, `-1` will be an acceptable value ## numRuns: -1 @@ -256,6 +283,10 @@ scheduler: ## preinitdb: false + ## the number of seconds to wait (in bash) before starting the scheduler container + ## + initialStartupDelay: 0 + ## extra init containers to run before the scheduler Pod ## ## EXAMPLE: @@ -269,9 +300,9 @@ scheduler: ## extraInitContainers: [] -##################### +################################### # Airflow - WebUI Configs -##################### +################################### web: ## resource requests/limits for the airflow web Pods ## @@ -353,31 +384,39 @@ web: ## the number of seconds to wait (in bash) before starting the web container ## - initialStartupDelay: 20 + initialStartupDelay: 0 ## the number of seconds to wait before declaring a new Pod available ## - minReadySeconds: 20 + minReadySeconds: 5 ## configs for the web Service readiness probe ## readinessProbe: + enabled: false scheme: HTTP - initialDelaySeconds: 60 + initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 1 successThreshold: 1 - failureThreshold: 5 + failureThreshold: 3 ## configs for the web Service liveness probe ## livenessProbe: + enabled: true scheme: HTTP - initialDelaySeconds: 360 - periodSeconds: 10 - timeoutSeconds: 1 + ## the number of seconds to wait before checking pod health + ## + ## NOTE: + ## - make larger if you are installing many packages with: + ## `airflow.extraPipPackages`, `web.extraPipPackages`, or `dags.installRequirements` + ## + initialDelaySeconds: 300 + periodSeconds: 30 + timeoutSeconds: 3 successThreshold: 1 - failureThreshold: 5 + failureThreshold: 2 ## the directory in which to mount secrets on web containers ## @@ -463,8 +502,15 @@ workers: maxReplicas: 2 metrics: [] + ## the number of seconds to wait (in bash) before starting each worker container + ## + initialStartupDelay: 0 + ## configs for the celery worker Pods ## + ## NOTE: + ## - only takes effect if `airflow.executor` is `CeleryExecutor` + ## celery: ## the number of tasks each celery worker can run at a time ## @@ -480,7 +526,7 @@ workers: ## gracefullTermination: false - ## how many seconds before worker Pods are killed using SIGKILL + ## how many seconds to wait for tasks on a worker to finish before SIGKILL ## terminationPeriod: 60 @@ -502,6 +548,9 @@ workers: flower: ## if the Flower UI should be deployed ## + ## NOTE: + ## - only takes effect if `airflow.executor` is `CeleryExecutor` + ## enabled: true ## resource requests/limits for the flower Pods @@ -558,6 +607,10 @@ flower: loadBalancerIP: "" loadBalancerSourceRanges: [] + ## the number of seconds to wait (in bash) before starting the flower container + ## + initialStartupDelay: 0 + ## extra ConfigMaps to mount on the flower Pods ## ## EXAMPLE: @@ -633,7 +686,7 @@ dags: ## install any Python `requirements.txt` at the root of `dags.path` automatically ## - installRequirements: true + installRequirements: false ## configs for the dags PVC ## @@ -768,6 +821,10 @@ dags: ## configs for the git-clone container ## + ## NOTE: + ## - use this container if you want to only clone the external git repo + ## at Pod start-time, and not keep it synchronised afterwards + ## initContainer: ## enable the git-clone sidecar container ## @@ -917,10 +974,13 @@ ingress: ## the livenessPath for the flower Ingress ## - ## NOTE: + ## WARNING: ## - keep the trailing slash ## - livenessPath: "/" + ## NOTE: + ## - if set to "", defaults to: `{ingress.flower.path}/` + ## + livenessPath: "" ## configs for flower Ingress TLS ## @@ -1005,7 +1065,7 @@ postgresql: ## which can be configured with values under: `externalDatabase` ## ## NOTE: - ## - set to false if using an external database + ## - set to `false` if using an external database ## enabled: true @@ -1096,7 +1156,8 @@ redis: ## if the `stable/redis` chart is used ## ## NOTE: - ## - set to false if using an external redis database + ## - set to `false` if using an external redis database + ## - set to `false` if `airflow.executor` is `KubernetesExecutor` ## enabled: true @@ -1124,7 +1185,7 @@ redis: ## the number of redis slaves ## - slaveCount: 3 + slaveCount: 1 ## configs for the redis master ## @@ -1228,7 +1289,8 @@ serviceMonitor: ## https://github.com/epoch8/airflow-exporter ## ## NOTE: - ## - ServiceMonitor is from: https://github.com/coreos/prometheus-operator + ## - you can install pip packages with `airflow.extraPipPackages` + ## - ServiceMonitor is a resource from: https://github.com/coreos/prometheus-operator ## enabled: false @@ -1256,7 +1318,8 @@ prometheusRule: ## https://github.com/epoch8/airflow-exporter ## ## NOTE: - ## - PrometheusRule is from: https://github.com/coreos/prometheus-operator + ## - you can install pip packages with `airflow.extraPipPackages` + ## - PrometheusRule a resource from: https://github.com/coreos/prometheus-operator ## enabled: false