diff --git a/stable/distributed-tensorflow/.helmignore b/stable/distributed-tensorflow/.helmignore new file mode 100644 index 0000000000..9a24b62c84 --- /dev/null +++ b/stable/distributed-tensorflow/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +docker +*.jpg +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/stable/distributed-tensorflow/Chart.yaml b/stable/distributed-tensorflow/Chart.yaml new file mode 100644 index 0000000000..59d7cd5a24 --- /dev/null +++ b/stable/distributed-tensorflow/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +description: A Helm chart for running distributed TensorFlow on Kubernetes +name: distributed-tensorflow +version: 0.1.0 +appVersion: 1.6.0 +sources: +- https://github.com/tensorflow/tensorflow +- https://www.tensorflow.org/deploy/distributed +home: https://www.tensorflow.org +maintainers: +- name: cheyang + email: cheyang@163.com diff --git a/stable/distributed-tensorflow/README.md b/stable/distributed-tensorflow/README.md new file mode 100644 index 0000000000..55692a8e3c --- /dev/null +++ b/stable/distributed-tensorflow/README.md @@ -0,0 +1,85 @@ +# Distributed TensorFlow + +TensorFlow is an open source software library for numerical computation using data flow graphs, and it supports distributed computing, allowing 'data parallel' or 'model parallel' on different servers. This means data scientists can now scale out distributed training to 100s of GPUs using TensorFlow. + +- https://www.tensorflow.org +- https://www.tensorflow.org/deploy/distributed + +![](distributed-tf.jpg) + +## Prerequisites + +- Kubernetes cluster v1.8+ + +## Chart Details + +This chart will create a TensorFlow cluster, and distribute a computation graph across that cluster. + +## Installing the Chart + +* To install the chart with the release name `mnist`: + + ```bash + $ helm install --name mnist incubator/distributed-tensorflow + ``` + +* To install with custom values via file: + + ``` + $ helm install --values values.yaml --name mnist incubator/distributed-tensorflow + ``` + + Below is an example of the custom value file values.yaml with GPU support. + + ``` + worker: + number: 2 + podManagementPolicy: Parallel + image: + repository: cheyang/distributed-tf + tag: 1.6.0-gpu + port: 9090 + gpuCount: 1 + + ps: + number: 2 + podManagementPolicy: Parallel + image: + repository: cheyang/distributed-tf + tag: 1.6.0 + pullPolicy: IfNotPresent + port: 8080 + + # optimize for training + hyperparams: + batchsize: 20 + learningrate: 0.001 + ``` + +> Notice: you can check the details of docker image from [Docker hub](https://hub.docker.com/r/cheyang/distributed-tf/) + +## Uninstalling the Chart + +* To uninstall/delete the `mnist` deployment: + + ```bash + $ helm delete mnist + ``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Configuration + +The following tables lists the configurable parameters of the Service Distributed Tensorflow +chart and their default values. + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `worker.image.repository` | TensorFlow Worker Server's image repository | `cheyang/distributed-tf` | +| `worker.image.tag` | TensorFlow Worker Server's image tag | `gpu` | +| `worker.image.pullPolicy` | image pullPolicy for the worker | `IfNotPresent` | +| `worker.gpuCount` | Set the gpu to be allocated and allowed for the Pods | `0` | +| `ps.image.repository` | TensorFlow Parameter Server's image repository | `cheyang/distributed-tf` | +| `ps.image.tag` | TensorFlow Parameter Server's image tag | `1.6.0-gpu` | +| `ps.image.pullPolicy` | image pullPolicy for the ps | `IfNotPresent` | + diff --git a/stable/distributed-tensorflow/distributed-tf.jpg b/stable/distributed-tensorflow/distributed-tf.jpg new file mode 100644 index 0000000000..0a74809e4e Binary files /dev/null and b/stable/distributed-tensorflow/distributed-tf.jpg differ diff --git a/stable/distributed-tensorflow/templates/NOTES.txt b/stable/distributed-tensorflow/templates/NOTES.txt new file mode 100644 index 0000000000..613adabb97 --- /dev/null +++ b/stable/distributed-tensorflow/templates/NOTES.txt @@ -0,0 +1,4 @@ +1. Get the application Status by running these commands: + + echo "Get the statefulset of distributed tensorflow" + kubectl get sts --namespace {{ .Release.Namespace }} -l "app={{ template "distributed-tensorflow.name" . }},release={{ .Release.Name }}" \ No newline at end of file diff --git a/stable/distributed-tensorflow/templates/_helpers.tpl b/stable/distributed-tensorflow/templates/_helpers.tpl new file mode 100644 index 0000000000..effc4e83a0 --- /dev/null +++ b/stable/distributed-tensorflow/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "distributed-tensorflow.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "distributed-tensorflow.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "distributed-tensorflow.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/stable/distributed-tensorflow/templates/config.yaml b/stable/distributed-tensorflow/templates/config.yaml new file mode 100644 index 0000000000..ad97c244f8 --- /dev/null +++ b/stable/distributed-tensorflow/templates/config.yaml @@ -0,0 +1,24 @@ +{{- $workerNum := .Values.worker.number -}} +{{- $workerPort := .Values.worker.port -}} +{{- $psNum := .Values.ps.number -}} +{{- $psPort := .Values.ps.port -}} +{{- $tfService := include "distributed-tensorflow.fullname" . }} +{{- $releaseName := .Release.Name -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "distributed-tensorflow.fullname" . }} + labels: + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: {{ template "distributed-tensorflow.chart" . }} + app: {{ template "distributed-tensorflow.name" . }} +data: + ps.hostList: " + {{- range $i, $none := until (int $psNum) }} + {{- if gt $i 0}},{{- end }}{{ $releaseName }}-ps-{{ $i }}.{{ $tfService }}-ps:{{ $psPort }} + {{- end }}" + worker.hostList: " + {{- range $i, $none := until (int $workerNum) }} + {{- if gt $i 0}},{{- end }}{{ $releaseName }}-worker-{{ $i }}.{{ $tfService }}-worker:{{ $workerPort }} + {{- end }}" \ No newline at end of file diff --git a/stable/distributed-tensorflow/templates/service-ps.yaml b/stable/distributed-tensorflow/templates/service-ps.yaml new file mode 100644 index 0000000000..631383ae52 --- /dev/null +++ b/stable/distributed-tensorflow/templates/service-ps.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "distributed-tensorflow.fullname" . }}-ps + labels: + app: {{ template "distributed-tensorflow.name" . }} + chart: {{ template "distributed-tensorflow.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + clusterIP: None + ports: + - port: {{ .Values.ps.port }} + targetPort: {{ .Values.ps.port }} + protocol: TCP + name: ps + selector: + app: {{ template "distributed-tensorflow.name" . }} + release: {{ .Release.Name }} + role: ps diff --git a/stable/distributed-tensorflow/templates/service-worker.yaml b/stable/distributed-tensorflow/templates/service-worker.yaml new file mode 100644 index 0000000000..1af3582d32 --- /dev/null +++ b/stable/distributed-tensorflow/templates/service-worker.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "distributed-tensorflow.fullname" . }}-worker + labels: + app: {{ template "distributed-tensorflow.name" . }} + chart: {{ template "distributed-tensorflow.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + clusterIP: None + ports: + - port: {{ .Values.worker.port }} + targetPort: {{ .Values.worker.port }} + protocol: TCP + name: worker + selector: + app: {{ template "distributed-tensorflow.name" . }} + release: {{ .Release.Name }} + role: worker diff --git a/stable/distributed-tensorflow/templates/statefulset-ps.yaml b/stable/distributed-tensorflow/templates/statefulset-ps.yaml new file mode 100644 index 0000000000..e53e2268ab --- /dev/null +++ b/stable/distributed-tensorflow/templates/statefulset-ps.yaml @@ -0,0 +1,83 @@ +{{- $lr := .Values.hyperparams.learningrate -}} +{{- $batchsize := .Values.hyperparams.batchsize -}} + +apiVersion: apps/v1beta2 +kind: StatefulSet +metadata: + name: {{ .Release.Name }}-ps + labels: + app: {{ template "distributed-tensorflow.name" . }} + chart: {{ template "distributed-tensorflow.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + role: ps +spec: + selector: + matchLabels: + app: {{ template "distributed-tensorflow.name" . }} + release: {{ .Release.Name }} + role: ps + serviceName: {{ template "distributed-tensorflow.fullname" . }}-ps + podManagementPolicy: {{ .Values.ps.podManagementPolicy }} + replicas: {{.Values.ps.number}} + template: + metadata: + labels: + app: {{ template "distributed-tensorflow.name" . }} + chart: {{ template "distributed-tensorflow.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + role: ps + spec: +{{- if .Values.volumes }} + volumes: +{{ toYaml .Values.volumes | indent 6 }} +{{- end }} + containers: + - name: ps + image: "{{ .Values.ps.image.repository }}:{{ .Values.ps.image.tag }}" + imagePullPolicy: {{ .Values.ps.image.pullPolicy }} + command: + - "python" + - "train_distributed.py" + - --learning_rate + - {{ $lr | quote }} + - --batch_size + - {{ $batchsize | quote }} + env: + - name: WORKER_HOSTS + valueFrom: + configMapKeyRef: + name: {{ template "distributed-tensorflow.fullname" . }} + key: worker.hostList + - name: PS_HOSTS + valueFrom: + configMapKeyRef: + name: {{ template "distributed-tensorflow.fullname" . }} + key: ps.hostList + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: JOB_NAME + value: ps + {{- if .Values.ps.env }} + {{- range $key, $value := .Values.ps.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- end }} +{{- if .Values.ps.privileged }} + securityContext: + privileged: true +{{- end }} + ports: + - containerPort: {{ .Values.ps.port }} +{{- if .Values.volumeMounts }} + volumeMounts: +{{ toYaml .Values.volumeMounts | indent 8 }} +{{- end }} +{{- if .Values.ps.resources }} + resources: +{{ toYaml .Values.ps.resources | indent 10 }} +{{- end }} \ No newline at end of file diff --git a/stable/distributed-tensorflow/templates/statefulset-worker.yaml b/stable/distributed-tensorflow/templates/statefulset-worker.yaml new file mode 100644 index 0000000000..69d4f51e8d --- /dev/null +++ b/stable/distributed-tensorflow/templates/statefulset-worker.yaml @@ -0,0 +1,86 @@ +{{- $lr := .Values.hyperparams.learningrate -}} +{{- $batchsize := .Values.hyperparams.batchsize -}} + +apiVersion: apps/v1beta2 +kind: StatefulSet +metadata: + name: {{ .Release.Name }}-worker + labels: + app: {{ template "distributed-tensorflow.name" . }} + chart: {{ template "distributed-tensorflow.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + role: worker +spec: + selector: + matchLabels: + app: {{ template "distributed-tensorflow.name" . }} + release: {{ .Release.Name }} + role: worker + serviceName: {{ template "distributed-tensorflow.fullname" . }}-worker + podManagementPolicy: {{ .Values.worker.podManagementPolicy }} + replicas: {{.Values.worker.number}} + template: + metadata: + labels: + app: {{ template "distributed-tensorflow.name" . }} + chart: {{ template "distributed-tensorflow.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + role: worker + spec: +{{- if .Values.volumes }} + volumes: +{{ toYaml .Values.volumes | indent 6 }} +{{- end }} + containers: + - name: worker + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + command: + - "python" + - "train_distributed.py" +{{- if gt (int .Values.worker.gpuCount) 0 }} + - --num_gpus + - "{{ .Values.worker.gpuCount }}" +{{- end }} + - --learning_rate + - {{ $lr | quote }} + - --batch_size + - {{ $batchsize | quote }} + env: + - name: WORKER_HOSTS + valueFrom: + configMapKeyRef: + name: {{ template "distributed-tensorflow.fullname" . }} + key: worker.hostList + - name: PS_HOSTS + valueFrom: + configMapKeyRef: + name: {{ template "distributed-tensorflow.fullname" . }} + key: ps.hostList + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: JOB_NAME + value: worker + {{- if .Values.worker.env }} + {{- range $key, $value := .Values.worker.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- end }} + ports: + - containerPort: {{ .Values.worker.port }} +{{- if .Values.volumeMounts }} + volumeMounts: +{{ toYaml .Values.volumeMounts | indent 8 }} +{{- end }} +{{- if gt (int .Values.worker.gpuCount) 0 }} + resources: + limits: + nvidia.com/gpu: {{- .Values.worker.gpuCount }} + request: + nvidia.com/gpu: {{- .Values.worker.gpuCount }} +{{- end }} diff --git a/stable/distributed-tensorflow/values.yaml b/stable/distributed-tensorflow/values.yaml new file mode 100644 index 0000000000..29c872de6f --- /dev/null +++ b/stable/distributed-tensorflow/values.yaml @@ -0,0 +1,24 @@ +# Default values for distributed-tensorflow. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. +worker: + number: 2 + podManagementPolicy: Parallel + image: + repository: cheyang/distributed-tf + tag: 1.6.0 + pullPolicy: IfNotPresent + port: 9000 + # gpuCount: 2 +ps: + number: 2 + podManagementPolicy: Parallel + image: + repository: cheyang/distributed-tf + tag: 1.6.0 + pullPolicy: IfNotPresent + port: 8000 +# optimize for training +hyperparams: + batchsize: 20 + learningrate: 0.001