Add dask-distributed chart (#1224)

2026-08-23 22:37:45 +00:00 · 2017-06-26 22:10:57 +02:00
parent 868eedebe1
commit b5dd55d7c7
12 changed files with 379 additions and 0 deletions
@@ -0,0 +1,21 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
@@ -0,0 +1,10 @@
+name: dask-distributed
+version: 1.16.3
+description: Distributed computation in Python
+home: https://github.com/dask/distributed
+icon: https://avatars3.githubusercontent.com/u/17131925?v=3&s=200
+sources:
+  - https://github.com/dask/distributed
+maintainers:
+  - name: danielfrg
+    email: df.rodriguez143@gmail.com
@@ -0,0 +1,83 @@
+# Dask Distributed Helm Chart
+
+Dask Distributed allows distributed computation in Python the chart also includes a single user Jupyter Notebook.
+
+* https://github.com/dask/distributed
+* http://jupyter.org/
+
+## Chart Details
+This chart will do the following:
+
+* 1 x Dask scheduler with port 8786 (scheduler) and 80 (Web UI) exposed on an external LoadBalancer
+* 3 x Dask workers that connect to the scheduler
+* 1 x Jupyter notebook with port 80 exposed on an external LoadBalancer
+* All using Kubernetes Deployments
+
+## Installing the Chart
+
+To install the chart with the release name `my-release`:
+
+```bash
+$ helm install --name my-release stable/dask
+```
+
+## Configuration
+
+The following tables lists the configurable parameters of the Dask chart and their default values.
+
+### Dask scheduler
+
+| Parameter                  | Description                        | Default                                                    |
+| -------------------------- | ---------------------------------- | ---------------------------------------------------------- |
+| `scheduler.name`           | Dask master name                   | `dask-master`                                              |
+| `scheduler.image`          | Container image name               | `dask2/dask`                                               |
+| `scheduler.imageTag`       | Container image tag                | `latest`                                                   |
+| `scheduler.replicas`       | k8s deployment replicas            | `1`                                                        |
+| `scheduler.component`      | k8s selector key                   | `dask-scheduler`                                           |
+| `scheduler.cpu`            | container requested cpu            | `500m`                                                     |
+| `scheduler.containerPort`  | Container listening port           | `8786`                                                     |
+| `scheduler.resources`      | Container resources                | `{}`                                                       |
+
+### Dask webUI
+
+|       Parameter       |           Description            |                         Default                          |
+|-----------------------|----------------------------------|----------------------------------------------------------|
+| `webUI.name`          | Dask webui name                  | `dask-webui`                                             |
+| `webUI.servicePort`   | k8s service port                 | `8787`                                                   |
+| `webUI.containerPort` | Container listening port         | `8787`                                                   |
+
+### Dask worker
+
+| Parameter                    | Description                          | Default                                                    |
+| -----------------------      | ------------------------------------ | ---------------------------------------------------------- |
+| `worker.name`                | Dask worker name                     | `dask-worker`                                              |
+| `worker.image`               | Container image name                 | `daskdev/dask`                                             |
+| `worker.imageTag`            | Container image tag                  | `1.5.1_v3`                                                 |
+| `worker.replicas`            | k8s hpa and deployment replicas      | `3`                                                        |
+| `worker.replicasMax`         | k8s hpa max replicas                 | `10`                                                       |
+| `worker.component`           | k8s selector key                     | `dask-worker`                                              |
+| `worker.containerPort`       | Container listening port             | `7077`                                                     |
+| `worker.resources`           | Container resources                  | `{}`                                                       |
+
+### jupyter
+
+|       Parameter         |           Description            |                         Default                          |
+|-------------------------|----------------------------------|----------------------------------------------------------|
+| `jupyter.name`          | jupyter name                     | `jupyter`                                                |
+| `jupyter.image`         | Container image name             | `jupyter/base-notebook`                                  |
+| `jupyter.imageTag`      | Container image tag              | `11be019e4079`                                           |
+| `jupyter.replicas`      | k8s deployment replicas          | `1`                                                      |
+| `jupyter.component`     | k8s selector key                 | `jupyter`                                                |
+| `jupyter.servicePort`   | k8s service port                 | `80`                                                     |
+| `jupyter.containerPort` | Container listening port         | `8888`                                                   |
+| `jupyter.resources`     | Container resources              | `{}`                                                     |
+
+Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`.
+
+Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example,
+
+```bash
+$ helm install --name my-release -f values.yaml stable/dask
+```
+
+> **Tip**: You can use the default [values.yaml](values.yaml)
@@ -0,0 +1,25 @@
+1. Get the Dask Scheduler connection string by running this commands in the same shell:
+
+  NOTE: It may take a few minutes for the LoadBalancer IP to be available, until that the commands below will not work.  
+  You can watch the status by running 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ template "scheduler-fullname" . }}'
+
+  export DASK_SCHEDULER=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "scheduler-fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+  echo http://$DASK_SCHEDULER:{{ .Values.scheduler.servicePort }}
+
+2. Get the Dask Scheduler Web UI by running this commands in the same shell:
+
+  NOTE: It may take a few minutes for the LoadBalancer IP to be available, until that the commands below will not work.  
+  You can watch the status by running 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ template "scheduler-fullname" . }}'
+
+  export DASK_SCHEDULER_UI_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "scheduler-fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+  echo http://$DASK_SCHEDULER_UI_IP:{{ .Values.webUI.servicePort }}
+
+3. Get the Jupyter Notebook URL to visit by running these commands in the same shell:
+
+  NOTE: It may take a few minutes for the LoadBalancer IP to be available, until that the commands below will not work.  
+  You can watch the status by running 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ template "jupyter-fullname" . }}'
+
+  NOTE: The default password to login is `dask`.
+  
+  export JUPYTER_NOTEBOOK_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "jupyter-fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+  echo http://$JUPYTER_NOTEBOOK_IP:{{ .Values.jupyter.servicePort }}
@@ -0,0 +1,31 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 24 -}}
+{{- end -}}
+
+{{/*
+Create fully qualified names.
+We truncate at 24 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "scheduler-fullname" -}}
+{{- $name := default .Chart.Name .Values.scheduler.name -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 24 -}}
+{{- end -}}
+
+{{- define "webui-fullname" -}}
+{{- $name := default .Chart.Name .Values.webUI.name -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 24 -}}
+{{- end -}}
+
+{{- define "worker-fullname" -}}
+{{- $name := default .Chart.Name .Values.worker.name -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 24 -}}
+{{- end -}}
+
+{{- define "jupyter-fullname" -}}
+{{- $name := default .Chart.Name .Values.jupyter.name -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 24 -}}
+{{- end -}}
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "jupyter-fullname" . }}-config
+  labels:
+    app: {{ template "name" . }}
+    heritage: {{ .Release.Service | quote }}
+    release: {{ .Release.Name | quote }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    component: "{{ .Release.Name }}-{{ .Values.jupyter.component }}"
+data:
+  jupyter_notebook_config.py: |
+    c = get_config()
+    c.NotebookApp.password = '{{ .Values.jupyter.password }}'
@@ -0,0 +1,35 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: {{ template "jupyter-fullname" . }}
+  labels:
+    app: {{ template "name" . }}
+    heritage: {{ .Release.Service | quote }}
+    release: {{ .Release.Name | quote }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    component: "{{ .Release.Name }}-{{ .Values.jupyter.component }}"
+spec:
+  replicas: {{ .Values.jupyter.replicas }}
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: {{ template "name" . }}
+        release: {{ .Release.Name | quote }}
+        component: "{{ .Release.Name }}-{{ .Values.jupyter.component }}"
+    spec:
+      containers:
+        - name: {{ template "jupyter-fullname" . }}
+          image: "{{ .Values.jupyter.image }}:{{ .Values.jupyter.imageTag }}"
+          ports:
+            - containerPort: {{ .Values.jupyter.containerPort }}
+          resources:
+{{ toYaml .Values.jupyter.resources | indent 12 }}
+          volumeMounts:
+          - name: config-volume
+            mountPath: /home/jovyan/.jupyter
+      volumes:
+      - name: config-volume
+        configMap:
+          name: {{ template "jupyter-fullname" . }}-config
@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "jupyter-fullname" . }}
+  labels:
+    app: {{ template "name" . }}
+    heritage: {{ .Release.Service | quote }}
+    release: {{ .Release.Name | quote }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    component: "{{ .Release.Name }}-{{ .Values.jupyter.component }}"
+spec:
+  ports:
+    - port: {{ .Values.jupyter.servicePort }}
+      targetPort: {{ .Values.jupyter.containerPort }}
+  selector:
+    app: {{ template "name" . }}
+    release: {{ .Release.Name | quote }}
+    component: "{{ .Release.Name }}-{{ .Values.jupyter.component }}"
+  type: "LoadBalancer"
@@ -0,0 +1,30 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: {{ template "scheduler-fullname" . }}
+  labels:
+    app: {{ template "name" . }}
+    heritage: {{ .Release.Service | quote }}
+    release: {{ .Release.Name | quote }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    component: "{{ .Release.Name }}-{{ .Values.scheduler.component }}"
+spec:
+  replicas: {{ .Values.scheduler.replicas }}
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: {{ template "name" . }}
+        release: {{ .Release.Name | quote }}
+        component: "{{ .Release.Name }}-{{ .Values.scheduler.component }}"
+    spec:
+      containers:
+        - name: {{ template "scheduler-fullname" . }}
+          image: "{{ .Values.scheduler.image }}:{{ .Values.scheduler.imageTag }}"
+          command: ["dask-scheduler", "--port", "{{ .Values.scheduler.servicePort }}", "--bokeh-port", "{{ .Values.webUI.containerPort }}"]
+          ports:
+            - containerPort: {{ .Values.scheduler.containerPort }}
+            - containerPort: {{ .Values.webUI.containerPort }}
+          resources:
+{{ toYaml .Values.scheduler.resources | indent 12 }}
@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "scheduler-fullname" . }}
+  labels:
+    app: {{ template "name" . }}
+    heritage: {{ .Release.Service | quote }}
+    release: {{ .Release.Name | quote }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    component: "{{ .Release.Name }}-{{ .Values.scheduler.component }}"
+spec:
+  ports:
+    - name: {{ template "scheduler-fullname" . }}
+      port: {{ .Values.scheduler.servicePort }}
+      targetPort: {{ .Values.scheduler.containerPort }}
+    - name: {{ template "webui-fullname" . }}
+      port: {{ .Values.webUI.servicePort }}
+      targetPort: {{ .Values.webUI.containerPort }}
+  selector:
+    app: {{ template "name" . }}
+    release: {{ .Release.Name | quote }}
+    component: "{{ .Release.Name }}-{{ .Values.scheduler.component }}"
+  type: "LoadBalancer"
@@ -0,0 +1,29 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: {{ template "worker-fullname" . }}
+  labels:
+    app: {{ template "name" . }}
+    heritage: {{ .Release.Service | quote }}
+    release: {{ .Release.Name | quote }}
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    component: "{{ .Release.Name }}-{{ .Values.worker.component }}"
+spec:
+  replicas: {{ .Values.worker.replicas }}
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: {{ template "name" . }}
+        release: {{ .Release.Name | quote }}
+        component: "{{ .Release.Name }}-{{ .Values.worker.component }}"
+    spec:
+      containers:
+        - name: {{ template "worker-fullname" . }}
+          image: "{{ .Values.worker.image }}:{{ .Values.worker.imageTag }}"
+          command: ["dask-worker", "{{ template "scheduler-fullname" . }}:{{ .Values.scheduler.servicePort }}"]
+          ports:
+            - containerPort: {{ .Values.worker.containerPort }}
+          resources:
+{{ toYaml .Values.worker.resources | indent 12 }}
@@ -0,0 +1,59 @@
+# Default values for dask.
+# This is a YAML-formatted file.
+# Declare name/value pairs to be passed into your templates.
+# name: value
+
+# nameOverride: dask
+
+scheduler:
+  name: scheduler
+  image: "daskdev/dask"
+  imageTag: "latest"
+  replicas: 1
+  component: "dask-scheduler"
+  servicePort: 8786
+  containerPort: 8786
+  resources: {}
+  # limits:
+  #   cpu: 500m
+  #   memory: 512Mi
+  # requests:
+  #   cpu: 500m
+  #   memory: 512Mi
+
+webUI:
+  name: webui
+  servicePort: 80
+  containerPort: 8787
+
+worker:
+  name: worker
+  image: "daskdev/dask"
+  imageTag: "latest"
+  replicas: 3
+  component: "dask-worker"
+  containerPort: 8081
+  resources: {}
+  # limits:
+  #   cpu: 500m
+  #   memory: 512Mi
+  # requests:
+  #   cpu: 500m
+  #   memory: 512Mi
+
+jupyter:
+  name: jupyter
+  image: "jupyter/base-notebook"
+  imageTag: "11be019e4079"
+  replicas: 1
+  component: "jupyter-notebook"
+  servicePort: 80
+  containerPort: 8888
+  password: 'sha1:aae8550c0a44:9507d45e087d5ee481a5ce9f4f16f37a0867318c'  # 'dask'
+  resources: {}
+  # limits:
+  #   cpu: 500m
+  #   memory: 512Mi
+  # requests:
+  #   cpu: 500m
+  #   memory: 512Mi