diff --git a/stable/dask/Chart.yaml b/stable/dask/Chart.yaml new file mode 100755 index 0000000000..0d2d54dc7f --- /dev/null +++ b/stable/dask/Chart.yaml @@ -0,0 +1,13 @@ +name: dask +version: 1.0.0 +appVersion: 0.17.1 +description: Distributed computation in Python with task scheduling +home: https://dask.pydata.org +icon: https://avatars3.githubusercontent.com/u/17131925?v=3&s=200 +sources: +- https://github.com/dask/dask +maintainers: +- name: danielfrg + email: df.rodriguez143@gmail.com +- name: mrocklin + email: mrocklin@gmail.com diff --git a/stable/dask/README.md b/stable/dask/README.md new file mode 100644 index 0000000000..5b99395b75 --- /dev/null +++ b/stable/dask/README.md @@ -0,0 +1,103 @@ +# Dask Helm Chart + +Dask allows distributed computation in Python. + +- https://dask.pydata.org +- http://jupyter.org/ + + +## Chart Details + +This chart will deploy the following: + +- 1 x Dask scheduler with port 8786 (scheduler) and 80 (Web UI) exposed on an external LoadBalancer +- 3 x Dask workers that connect to the scheduler +- 1 x Jupyter notebook (optional) with port 80 exposed on an external LoadBalancer +- All using Kubernetes Deployments + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```bash +$ helm install --name my-release stable/dask +``` + +## Configuration + +The following tables lists the configurable parameters of the Dask chart and their default values. + +### Dask scheduler + +| Parameter | Description | Default | +| -------------------------- | -------------------------| -----------------| +| `scheduler.name` | Dask scheduler name | `scheduler` | +| `scheduler.image` | Container image name | `daskdev/dask` | +| `scheduler.imageTag` | Container image tag | `latest` | +| `scheduler.replicas` | k8s deployment replicas | `1` | +| `scheduler.resources` | Container resources | `{}` | + +### Dask webUI + +| Parameter | Description | Default | +|-----------------------|-------------------|-----------| +| `webUI.name` | Dask webui name | `webui` | +| `webUI.servicePort` | k8s service port | `80` | + +### Dask worker + +| Parameter | Description | Default | +| ----------------------- | ---------------------------------| ---------------| +| `worker.name` | Dask worker name | `worker` | +| `worker.image` | Container image name | `daskdev/dask` | +| `worker.imageTag` | Container image tag | `0.17.1` | +| `worker.replicas` | k8s hpa and deployment replicas | `3` | +| `worker.resources` | Container resources | `{}` | +| + +### jupyter + +| Parameter | Description | Default | +|-------------------------|----------------------------------|--------------------------| +| `jupyter.name` | jupyter name | `jupyter` | +| `jupyter.enabled` | Include optional Jupyter server | `true` | +| `jupyter.image` | Container image name | `daskdev/dask-notebook` | +| `jupyter.imageTag` | Container image tag | `0.17.1` | +| `jupyter.replicas` | k8s deployment replicas | `1` | +| `jupyter.servicePort` | k8s service port | `80` | +| `jupyter.resources` | Container resources | `{}` | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. + +Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, + +```bash +$ helm install --name my-release -f values.yaml stable/dask +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) + + +### Customizing Python Environment + +The default `daskdev/dask` images have a standard Miniconda installation along +with some common packages like NumPy and Pandas. You can install custom packages +with either Conda or Pip using optional environment variables. This happens +when your container starts up. Consider the following config.yaml file as an +example: + +```yaml +jupyter: + env: + - EXTRA_PIP_PACKAGES: s3fs git+https://github.com/user/repo.git --upgrade + - EXTRA_CONDA_PACKAGES: scipy matplotlib -c conda-forge + +worker: + env: + - EXTRA_PIP_PACKAGES: s3fs git+https://github.com/user/repo.git --upgrade + - EXTRA_CONDA_PACKAGES: scipy -c conda-forge +``` + +Note that the Jupyter and Dask worker environments should have matching +software environments, at least where a user is likely to distribute that +functionality. diff --git a/stable/dask/templates/NOTES.txt b/stable/dask/templates/NOTES.txt new file mode 100644 index 0000000000..12e78dc14c --- /dev/null +++ b/stable/dask/templates/NOTES.txt @@ -0,0 +1,24 @@ +Thank you for installing {{ .Chart.Name | upper }}, released at name: {{ .Release.Name }}. + +To learn more about the release, try: + + $ helm status {{ .Release.Name }} # information about running pods and this message + $ helm get {{ .Release.Name }} # get full Kubernetes specification + +This release includes a Dask scheduler, {{ .Values.worker.replicas }} Dask workers, and {{ .Values.jupyter.replicas }} Jupyter servers. + +The Jupyter notebook server and Dask scheduler expose external services to +which you can connect to manage notebooks, or connect directly to the Dask +cluster. You can get these addresses by running the following: + + export DASK_SCHEDULER=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "dask.fullname" . }}-scheduler -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + export DASK_SCHEDULER_UI_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "dask.fullname" . }}-scheduler -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + export JUPYTER_NOTEBOOK_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "dask.fullname" . }}-jupyter -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$JUPYTER_NOTEBOOK_IP:{{ .Values.jupyter.servicePort }} -- Jupyter notebook + echo http://$DASK_SCHEDULER_UI_IP:{{ .Values.webUI.servicePort }} -- Dask dashboard + echo http://$DASK_SCHEDULER:{{ .Values.scheduler.servicePort }} -- Dask Client connection + + NOTE: The default password to login to the notebook server is `dask`. + + NOTE: It may take a few minutes for the LoadBalancer IP to be available, until that the commands below will not work. + You can watch the status by running 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ template "dask.fullname" . }}-jupyter' diff --git a/stable/dask/templates/_helpers.tpl b/stable/dask/templates/_helpers.tpl new file mode 100644 index 0000000000..527d9f7c38 --- /dev/null +++ b/stable/dask/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "dask.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "dask.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "dask.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/stable/dask/templates/dask-jupyter-config.yaml b/stable/dask/templates/dask-jupyter-config.yaml new file mode 100644 index 0000000000..20dd6d73e1 --- /dev/null +++ b/stable/dask/templates/dask-jupyter-config.yaml @@ -0,0 +1,18 @@ +{{ if .Values.jupyter.enabled -}} + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "dask.fullname" . }}-jupyter-config + labels: + app: {{ template "dask.name" . }} + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: {{ template "dask.chart" . }} + component: jupyter +data: + jupyter_notebook_config.py: | + c = get_config() + c.NotebookApp.password = '{{ .Values.jupyter.password }}' + +{{ end }} diff --git a/stable/dask/templates/dask-jupyter-deployment.yaml b/stable/dask/templates/dask-jupyter-deployment.yaml new file mode 100644 index 0000000000..f704ec29ee --- /dev/null +++ b/stable/dask/templates/dask-jupyter-deployment.yaml @@ -0,0 +1,51 @@ +{{ if .Values.jupyter.enabled -}} + +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: {{ template "dask.fullname" . }}-jupyter + labels: + app: {{ template "dask.name" . }} + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: {{ template "dask.chart" . }} + component: jupyter +spec: + replicas: {{ .Values.jupyter.replicas }} + selector: + matchLabels: + app: {{ template "dask.name" . }} + release: {{ .Release.Name | quote }} + component: jupyter + strategy: + type: RollingUpdate + template: + metadata: + labels: + app: {{ template "dask.name" . }} + release: {{ .Release.Name | quote }} + component: jupyter + spec: + containers: + - name: {{ template "dask.fullname" . }}-jupyter + image: "{{ .Values.jupyter.image.repository }}:{{ .Values.jupyter.image.tag }}" + imagePullPolicy: {{ .Values.jupyter.image.pullPolicy }} + ports: + - containerPort: 8888 + resources: +{{ toYaml .Values.jupyter.resources | indent 12 }} + volumeMounts: + - name: config-volume + mountPath: /home/jovyan/.jupyter + env: + - name: DASK_SCHEDULER_ADDRESS + value: {{ template "dask.fullname" . }}-scheduler:{{ .Values.scheduler.servicePort }} +{{- if .Values.jupyter.env }} +{{ toYaml .Values.jupyter.env | indent 12 }} +{{- end }} + volumes: + - name: config-volume + configMap: + name: {{ template "dask.fullname" . }}-jupyter-config + +{{ end }} diff --git a/stable/dask/templates/dask-jupyter-service.yaml b/stable/dask/templates/dask-jupyter-service.yaml new file mode 100644 index 0000000000..ea30766a27 --- /dev/null +++ b/stable/dask/templates/dask-jupyter-service.yaml @@ -0,0 +1,23 @@ +{{ if .Values.jupyter.enabled -}} + +apiVersion: v1 +kind: Service +metadata: + name: {{ template "dask.fullname" . }}-jupyter + labels: + app: {{ template "dask.name" . }} + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: {{ template "dask.chart" . }} + component: jupyter +spec: + ports: + - port: {{ .Values.jupyter.servicePort }} + targetPort: 8888 + selector: + app: {{ template "dask.name" . }} + release: {{ .Release.Name | quote }} + component: jupyter + type: {{ .Values.jupyter.serviceType }} + +{{ end }} diff --git a/stable/dask/templates/dask-scheduler-deployment.yaml b/stable/dask/templates/dask-scheduler-deployment.yaml new file mode 100644 index 0000000000..7c65b8063a --- /dev/null +++ b/stable/dask/templates/dask-scheduler-deployment.yaml @@ -0,0 +1,43 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: {{ template "dask.fullname" . }}-scheduler + labels: + app: {{ template "dask.name" . }} + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: {{ template "dask.chart" . }} + component: scheduler +spec: + replicas: {{ .Values.scheduler.replicas }} + selector: + matchLabels: + app: {{ template "dask.name" . }} + release: {{ .Release.Name | quote }} + component: scheduler + strategy: + type: RollingUpdate + template: + metadata: + labels: + app: {{ template "dask.name" . }} + release: {{ .Release.Name | quote }} + component: scheduler + spec: + containers: + - name: {{ template "dask.fullname" . }}-scheduler + image: "{{ .Values.scheduler.image.repository }}:{{ .Values.scheduler.image.tag }}" + imagePullPolicy: {{ .Values.scheduler.image.pullPolicy }} + args: + - dask-scheduler + - --port + - "{{ .Values.scheduler.servicePort }}" + - --bokeh-port + - "8787" + ports: + - containerPort: 8786 + - containerPort: 8787 + resources: +{{ toYaml .Values.scheduler.resources | indent 12 }} + env: +{{ toYaml .Values.scheduler.env | indent 12 }} diff --git a/stable/dask/templates/dask-scheduler-service.yaml b/stable/dask/templates/dask-scheduler-service.yaml new file mode 100644 index 0000000000..83d8d6761e --- /dev/null +++ b/stable/dask/templates/dask-scheduler-service.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "dask.fullname" . }}-scheduler + labels: + app: {{ template "dask.name" . }} + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: {{ template "dask.chart" . }} + component: scheduler +spec: + ports: + - name: {{ template "dask.fullname" . }}-scheduler + port: {{ .Values.scheduler.servicePort }} + targetPort: 8786 + - name: {{ template "dask.fullname" . }}-webui + port: {{ .Values.webUI.servicePort }} + targetPort: 8787 + selector: + app: {{ template "dask.name" . }} + release: {{ .Release.Name | quote }} + component: scheduler + type: {{ .Values.scheduler.serviceType }} diff --git a/stable/dask/templates/dask-worker-deployment.yaml b/stable/dask/templates/dask-worker-deployment.yaml new file mode 100644 index 0000000000..c0560ebedb --- /dev/null +++ b/stable/dask/templates/dask-worker-deployment.yaml @@ -0,0 +1,46 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: {{ template "dask.fullname" . }}-worker + labels: + app: {{ template "dask.name" . }} + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: {{ template "dask.chart" . }} + component: worker +spec: + replicas: {{ .Values.worker.replicas }} + selector: + matchLabels: + app: {{ template "dask.name" . }} + release: {{ .Release.Name | quote }} + component: worker + strategy: + type: RollingUpdate + template: + metadata: + labels: + app: {{ template "dask.name" . }} + release: {{ .Release.Name | quote }} + component: worker + spec: + containers: + - name: {{ template "dask.fullname" . }}-worker + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + args: + - dask-worker + - {{ template "dask.fullname" . }}-scheduler:{{ .Values.scheduler.servicePort }} + {{- if .Values.worker.limits }} + - --nthreads + - {{ default .Values.worker.limits.cpu .Values.workers.default_resources.cpu | quote }} + - --memory-limit + - {{ default .Values.worker.limits.memory .Values.workers.default_resources.memory | quote }} + {{- end }} + - --no-bokeh + ports: + - containerPort: 8789 + resources: +{{ toYaml .Values.worker.resources | indent 12 }} + env: +{{ toYaml .Values.worker.env | indent 12 }} diff --git a/stable/dask/values.yaml b/stable/dask/values.yaml new file mode 100644 index 0000000000..5ae0267157 --- /dev/null +++ b/stable/dask/values.yaml @@ -0,0 +1,71 @@ +# nameOverride: dask +# fullnameOverride: dask + +scheduler: + name: scheduler + image: + repository: "daskdev/dask" + tag: "0.17.1" + pullPolicy: IfNotPresent + replicas: 1 + serviceType: "LoadBalancer" + servicePort: 8786 + resources: {} + # limits: + # cpu: 1.8 + # memory: 6G + # requests: + # cpu: 1.8 + # memory: 6G + +webUI: + name: webui + servicePort: 80 + +worker: + name: worker + image: + repository: "daskdev/dask" + tag: "0.17.1" + pullPolicy: IfNotPresent + replicas: 3 + aptPackages: >- + default_resources: # overwritten by resource limits if they exist + cpu: 1 + memory: "4GiB" + env: + # - name: EXTRA_CONDA_PACKAGES + # value: numba xarray -c conda-forge + # - name: EXTRA_PIP_PACKAGES + # value: s3fs dask-ml --upgrade + resources: {} + # limits: + # cpu: 1 + # memory: 3G + # requests: + # cpu: 1 + # memory: 3G + +jupyter: + name: jupyter + enabled: true + image: + repository: "daskdev/dask-notebook" + tag: "0.17.1" + pullPolicy: IfNotPresent + replicas: 1 + serviceType: "LoadBalancer" + servicePort: 80 + password: 'sha1:aae8550c0a44:9507d45e087d5ee481a5ce9f4f16f37a0867318c' # 'dask' + env: + # - name: EXTRA_CONDA_PACKAGES + # value: numba xarray -c conda-forge + # - name: EXTRA_PIP_PACKAGES + # value: s3fs dask-ml --upgrade + resources: {} + # limits: + # cpu: 2 + # memory: 6G + # requests: + # cpu: 2 + # memory: 6G