From dd642bfec5e43f56f28b3c2d9991c7772ab0ba2b Mon Sep 17 00:00:00 2001 From: cheyang Date: Fri, 18 May 2018 13:02:50 +0800 Subject: [PATCH] [stable/horovod] Add chart for horovod (#5415) * [incubator/horovod] support horovod * fix lint check * update the readme * Separate files per resource * add use secrets * add env for using secrets * move to stable * change to offical docker image --- stable/horovod/.helmignore | 21 +++ stable/horovod/Chart.yaml | 12 ++ stable/horovod/README.md | 111 +++++++++++++++ stable/horovod/templates/NOTES.txt | 5 + stable/horovod/templates/_helpers.tpl | 32 +++++ stable/horovod/templates/config.yaml | 115 ++++++++++++++++ stable/horovod/templates/job-service.yaml | 19 +++ stable/horovod/templates/job.yaml | 126 ++++++++++++++++++ stable/horovod/templates/secrets.yaml | 15 +++ .../templates/statefulset-service.yaml | 19 +++ stable/horovod/templates/statefulset.yaml | 111 +++++++++++++++ stable/horovod/values.yaml | 37 +++++ 12 files changed, 623 insertions(+) create mode 100644 stable/horovod/.helmignore create mode 100644 stable/horovod/Chart.yaml create mode 100644 stable/horovod/README.md create mode 100644 stable/horovod/templates/NOTES.txt create mode 100644 stable/horovod/templates/_helpers.tpl create mode 100644 stable/horovod/templates/config.yaml create mode 100644 stable/horovod/templates/job-service.yaml create mode 100644 stable/horovod/templates/job.yaml create mode 100644 stable/horovod/templates/secrets.yaml create mode 100644 stable/horovod/templates/statefulset-service.yaml create mode 100644 stable/horovod/templates/statefulset.yaml create mode 100644 stable/horovod/values.yaml diff --git a/stable/horovod/.helmignore b/stable/horovod/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/stable/horovod/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/stable/horovod/Chart.yaml b/stable/horovod/Chart.yaml new file mode 100644 index 0000000000..e0986652bf --- /dev/null +++ b/stable/horovod/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +description: A Helm chart for deploying Horovod +name: horovod +version: "0.1.1" +appVersion: "0.12.1" +sources: + - https://github.com/uber/horovod + - https://github.com/uber/horovod/blob/master/docs/docker.md +home: https://eng.uber.com/horovod/ +maintainers: + - name: cheyang + email: cheyang@163.com diff --git a/stable/horovod/README.md b/stable/horovod/README.md new file mode 100644 index 0000000000..75d04f722e --- /dev/null +++ b/stable/horovod/README.md @@ -0,0 +1,111 @@ +# Horovod + +[Horovod](https://eng.uber.com/horovod/) is a distributed training framework for TensorFlow, and it's provided by UBER. The goal of Horovod is to make distributed Deep Learning fast and easy to use. And it provides [Horovod in Docker](https://github.com/uber/horovod/blob/master/docs/docker.md) to streamline the installation process. + +## Introduction + +This chart bootstraps Horovod which is a Distributed TensorFlow Framework on a Kubernetes cluster using the Helm Package manager. It deploys Horovod workers as statefulsets, and the Horovod master as a job, then discover the the host list automatically.ß + +## Prerequisites + +- Kubernetes cluster v1.8+ + +## Build Docker Image + +You can download [offical Horovod Dockerfile](https://github.com/uber/horovod/blob/master/Dockerfile), then modify it according to your requirement, e.g. select a different CUDA, TensorFlow or Python version. + +``` +# mkdir horovod-docker +# wget -O horovod-docker/Dockerfile https://raw.githubusercontent.com/uber/horovod/master/Dockerfile +# docker build -t horovod:latest horovod-docker +``` + +## Define the values.yaml + +To deploy Horovod with GPU, you can create `values.yaml` like + +``` +worker: + number: 3 + podManagementPolicy: Parallel + image: + repository: uber/horovod + tag: 0.12.1-tf1.8.0-py3.5 + pullPolicy: IfNotPresent + resources: + limits: + nvidia.com/gpu: 1 + requests: + nvidia.com/gpu: 1 + +master: + image: + repository: uber/horovod + tag: 0.12.1-tf1.8.0-py3.5 + pullPolicy: IfNotPresent + args: + - "mpiexec -n ${WORKERS} --hostfile /kubeflow/openmpi/assets/hostfile --mca orte_keep_fqdn_hostnames t --allow-run-as-root --display-map --tag-output --timestamp-output sh -c 'python /examples/tensorflow_mnist.py'" +``` + +To deploy Horovod without GPU, you can create `values.yaml` like + +``` +worker: + number: 3 + podManagementPolicy: Parallel + image: + repository: uber/horovod + tag: 0.12.1-tf1.8.0-py3.5 + pullPolicy: IfNotPresent + +master: + image: + repository: uber/horovod + tag: 0.12.1-tf1.8.0-py3.5 + pullPolicy: IfNotPresent + args: + - "mpiexec -n 3 --hostfile /horovod/generated/hostfile --mca orte_keep_fqdn_hostnames t --allow-run-as-root --display-map --tag-output --timestamp-output sh -c 'LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-9.0/targets/x86_64-linux/lib/stubs python /examples/tensorflow_mnist.py'" +``` + + + +## Installing the Chart + +To install the chart with the release name `mnist`: + +```bash +$ helm install --values values.yaml --name mnist incubator/horovod +``` + +## Uninstalling the Chart + +To uninstall/delete the `mnist` deployment: + +```bash +$ helm delete mnist +``` + +The command removes all the Kubernetes components associated with the chart and +deletes the release. + +## Configuration + +The following tables lists the configurable parameters of the Horovod +chart and their default values. + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `ssh.port` | The ssh port | `22` | +| `ssh.useSecrets` | Determine if using the secrets for ssh | `false` | +| `worker.number`| The worker's number | `5` | +| `worker.image.repository` | horovod worker image | `uber/horovod` | +| `worker.image.pullPolicy` | `pullPolicy` for the worker | `IfNotPresent` | +| `worker.image.tag` | `tag` for the worker | `0.12.1-tf1.8.0-py3.5` | +| `worker.resources`| worker's pod resource requests & limits| `{}`| +| `worker.env` | worker's environment varaibles | `{}` | +| `master.image.repository` | horovod master image | `uber/horovod` | +| `master.image.tag` | `tag` for the master | `0.12.1-tf1.8.0-py3.5` | +| `master.image.pullPolicy` | image pullPolicy for the master image| `IfNotPresent` | +| `master.args` | master's args | `{}` | +| `master.resources`| master's pod resource requests & limits| `{}`| +| `master.env` | master's environment varaibles | `{}` | diff --git a/stable/horovod/templates/NOTES.txt b/stable/horovod/templates/NOTES.txt new file mode 100644 index 0000000000..da3d1b5834 --- /dev/null +++ b/stable/horovod/templates/NOTES.txt @@ -0,0 +1,5 @@ +1. Get the application URL by running these commands: + +*** NOTE: It may take a few minutes for the statefulset to be avaialble + +*** you can watch the status of statefulset by running 'kubectl get sts --namespace {{ .Release.Namespace }} -w {{ template "horovod.fullname" . }}' *** \ No newline at end of file diff --git a/stable/horovod/templates/_helpers.tpl b/stable/horovod/templates/_helpers.tpl new file mode 100644 index 0000000000..02071c0f54 --- /dev/null +++ b/stable/horovod/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "horovod.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "horovod.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "horovod.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/stable/horovod/templates/config.yaml b/stable/horovod/templates/config.yaml new file mode 100644 index 0000000000..558b1c5604 --- /dev/null +++ b/stable/horovod/templates/config.yaml @@ -0,0 +1,115 @@ +{{- $workerNum := .Values.worker.number -}} +{{- $name := include "horovod.fullname" . }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "horovod.fullname" . }} + labels: + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: {{ template "horovod.chart" . }} + app: {{ template "horovod.fullname" . }} +data: + hostfile.config: | + {{ $name }}-master + {{- range $i, $none := until (int $workerNum) }} + {{ $name }}-{{ $i }}.{{ $name }} + {{- end }} + ssh.readiness: | + #!/bin/bash + set -xev + ssh localhost ls + master.run: | + #!/bin/bash + set -x + sleep 5 + + mkdir -p /root/.ssh + rm -f /root/.ssh/config + touch /root/.ssh/config + + if [ "$USESECRETS" == "true" ];then + ln -s /etc/secret-volume/id_rsa /root/.ssh/id_rsa + ln -s /etc/secret-volume/authorized_keys /root/.ssh/authorized_keys + fi + + if [ -n "$SSHPORT" ]; then + echo "Port $SSHPORT" > /root/.ssh/config + sed -ri "s/#Port 22/Port ${SSH_PORT}/g" /etc/ssh/sshd_config + fi + echo "StrictHostKeyChecking no" >> /root/.ssh/config + /usr/sbin/sshd + + if [ $# -eq 0 ]; then + sleep infinity + else + bash -c "$*" + fi + master.waitWorkerReady: | + #!/bin/bash + set -xev + function updateSSHPort() { + if [ -n "$SSH_PORT" ]; then + sed -i "s/^Port.*/Port $SSH_PORT /g" /root/.ssh/config + echo "StrictHostKeyChecking no" >> /root/.ssh/config + fi + } + + function runCheckSSH() { + if [ "$USESECRETS" == "true" ];then + ln -s /etc/secret-volume/id_rsa /root/.ssh/id_rsa + ln -s /etc/secret-volume/authorized_keys /root/.ssh/authorized_keys + fi + + for i in `cat $1`;do + if [[ "$i" != *"master" ]];then + retry 30 ssh -o ConnectTimeout=2 -q $i exit + fi + done + } + + function retry() + { + local n=0;local try=$1 + local cmd="${@: 2}" + [[ $# -le 1 ]] && { + echo "Usage $0 "; + } + set +e + until [[ $n -ge $try ]] + do + $cmd && break || { + echo "Command Fail.." + ((n++)) + echo "retry $n :: [$cmd]" + sleep 1; + } + done + $cmd + if [ $? -ne 0 ]; then + exit 1 + fi + set -e + } + updateSSHPort + runCheckSSH $1 + worker.run: | + #! /bin/sh + set -x + + mkdir -p /root/.ssh + rm -f /root/.ssh/config + touch /root/.ssh/config + + if [ "$USESECRETS" == "true" ];then + ln -s /etc/secret-volume/id_rsa /root/.ssh/id_rsa + ln -s /etc/secret-volume/authorized_keys /root/.ssh/authorized_keys + fi + + if [ -n "$SSHPORT" ]; then + echo "Port $SSHPORT" > /root/.ssh/config + sed -ri "s/#Port 22/Port ${SSH_PORT}/g" /etc/ssh/sshd_config + fi + echo "StrictHostKeyChecking no" >> /root/.ssh/config + + /usr/sbin/sshd -D \ No newline at end of file diff --git a/stable/horovod/templates/job-service.yaml b/stable/horovod/templates/job-service.yaml new file mode 100644 index 0000000000..e7b05c26bf --- /dev/null +++ b/stable/horovod/templates/job-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "horovod.fullname" . }}-master + labels: + app: {{ template "horovod.name" . }} + chart: {{ template "horovod.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + clusterIP: None + ports: + - name: ssh + port: {{ .Values.ssh.port }} + targetPort: {{ .Values.ssh.port }} + selector: + app: {{ template "horovod.name" . }} + release: {{ .Release.Name }} + role: master diff --git a/stable/horovod/templates/job.yaml b/stable/horovod/templates/job.yaml new file mode 100644 index 0000000000..e48b8c9568 --- /dev/null +++ b/stable/horovod/templates/job.yaml @@ -0,0 +1,126 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "horovod.fullname" . }} + labels: + app: {{ template "horovod.name" . }} + chart: {{ template "horovod.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + role: master +spec: + template: + metadata: + labels: + app: {{ template "horovod.name" . }} + release: {{ .Release.Name }} + role: master + spec: + {{- if .Values.useHostNetwork }} + hostNetwork: {{ .Values.useHostNetwork }} + dnsPolicy: ClusterFirstWithHostNet + {{- end }} + {{- if .Values.useHostPID }} + hostPID: {{ .Values.useHostPID }} + {{- end }} + restartPolicy: OnFailure + volumes: + - name: {{ template "horovod.fullname" . }}-cm + configMap: + name: {{ template "horovod.fullname" . }} + items: + - key: hostfile.config + path: hostfile + mode: 438 + - key: master.waitWorkerReady + path: waitWorkersReady.sh + mode: 365 + - key: master.run + path: run.sh + mode: 365 + {{- if .Values.ssh.useSecrets }} + - name: {{ template "horovod.fullname" . }}-secret + secret: + secretName: {{ template "horovod.fullname" . }} + defaultMode: 448 + items: + - key: host-key + path: id_rsa + - key: host-key-pub + path: authorized_keys + {{- end }} +{{- if .Values.volumes }} +{{ toYaml .Values.volumes | indent 6 }} +{{- end }} + containers: + - name: horovod-master + image: "{{ .Values.master.image.repository }}:{{ .Values.master.image.tag }}" + imagePullPolicy: {{ .Values.master.image.pullPolicy }} + env: + - name: SSHPORT + value: "{{ .Values.ssh.port }}" + {{- if .Values.ssh.useSecrets }} + - name: USESECRETS + value: "{{ .Values.ssh.useSecrets }}" + {{- end }} + {{- if .Values.master.env }} + {{- range $key, $value := .Values.master.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- end }} +{{- if .Values.master.privileged }} + securityContext: + privileged: true +{{- end }} + ports: + - containerPort: {{ .Values.ssh.port }} + volumeMounts: + - name: {{ template "horovod.fullname" . }}-cm + mountPath: /horovod/generated + {{- if .Values.ssh.useSecrets }} + - name: {{ template "horovod.fullname" . }}-secret + readOnly: true + mountPath: "/etc/secret-volume" + {{- end }} +{{- if .Values.volumeMounts }} +{{ toYaml .Values.volumeMounts | indent 8 }} +{{- end }} + command: + - /horovod/generated/run.sh + args: +{{ toYaml .Values.master.args | indent 10 }} + resources: +{{ toYaml .Values.master.resources | indent 10 }} +{{- if .Values.ssh.useSecrets }} + initContainers: + - name: wait-workers + image: "{{ .Values.master.image.repository }}:{{ .Values.master.image.tag }}" + imagePullPolicy: {{ .Values.master.image.pullPolicy }} + env: + - name: SSHPORT + value: "{{ .Values.ssh.port }}" + {{- if .Values.ssh.useSecrets }} + - name: USESECRETS + value: "{{ .Values.ssh.useSecrets }}" + {{- end }} + {{- if .Values.master.env }} + {{- range $key, $value := .Values.master.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- end }} + command: + - /horovod/generated/waitWorkersReady.sh + args: + - /horovod/generated/hostfile + volumeMounts: + - name: {{ template "horovod.fullname" . }}-cm + mountPath: /horovod/generated + {{- if .Values.ssh.useSecrets }} + - name: {{ template "horovod.fullname" . }}-secret + readOnly: true + mountPath: "/etc/secret-volume" + {{- end }} +{{- end }} \ No newline at end of file diff --git a/stable/horovod/templates/secrets.yaml b/stable/horovod/templates/secrets.yaml new file mode 100644 index 0000000000..c9853ed0c9 --- /dev/null +++ b/stable/horovod/templates/secrets.yaml @@ -0,0 +1,15 @@ +{{- if .Values.ssh.useSecrets }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "horovod.fullname" . }} + labels: + app: {{ template "horovod.name" . }} + chart: {{ template "horovod.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +type: Opaque +data: + host-key: {{ .Values.ssh.hostKey | b64enc | quote }} + host-key-pub: {{ .Values.ssh.hostKeyPub | b64enc | quote }} +{{- end }} \ No newline at end of file diff --git a/stable/horovod/templates/statefulset-service.yaml b/stable/horovod/templates/statefulset-service.yaml new file mode 100644 index 0000000000..d0216a86b5 --- /dev/null +++ b/stable/horovod/templates/statefulset-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "horovod.fullname" . }} + labels: + app: {{ template "horovod.name" . }} + chart: {{ template "horovod.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + clusterIP: None + ports: + - name: ssh + port: {{ .Values.ssh.port }} + targetPort: {{ .Values.ssh.port }} + selector: + app: {{ template "horovod.name" . }} + release: {{ .Release.Name }} + role: worker diff --git a/stable/horovod/templates/statefulset.yaml b/stable/horovod/templates/statefulset.yaml new file mode 100644 index 0000000000..9db64c3d74 --- /dev/null +++ b/stable/horovod/templates/statefulset.yaml @@ -0,0 +1,111 @@ +apiVersion: apps/v1beta2 +kind: StatefulSet +metadata: + name: {{ template "horovod.fullname" . }} + labels: + app: {{ template "horovod.name" . }} + chart: {{ template "horovod.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + role: worker +spec: + selector: + matchLabels: + app: {{ template "horovod.name" . }} + chart: {{ template "horovod.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + role: worker + serviceName: {{ template "horovod.fullname" . }} + podManagementPolicy: {{ .Values.worker.podManagementPolicy }} + replicas: {{.Values.worker.number}} + template: + metadata: + labels: + app: {{ template "horovod.name" . }} + chart: {{ template "horovod.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + role: worker + spec: + {{- if .Values.useHostNetwork }} + hostNetwork: {{ .Values.useHostNetwork }} + dnsPolicy: ClusterFirstWithHostNet + {{- end }} + {{- if .Values.useHostPID }} + hostPID: {{ .Values.useHostPID }} + {{- end }} + volumes: + - name: {{ template "horovod.fullname" . }}-cm + configMap: + name: {{ template "horovod.fullname" . }} + items: + - key: hostfile.config + path: hostfile + mode: 438 + - key: ssh.readiness + path: check.sh + mode: 365 + - key: worker.run + path: run.sh + mode: 365 + {{- if .Values.ssh.useSecrets }} + - name: {{ template "horovod.fullname" . }}-secret + secret: + secretName: {{ template "horovod.fullname" . }} + defaultMode: 448 + items: + - key: host-key + path: id_rsa + - key: host-key-pub + path: authorized_keys + {{- end }} +{{- if .Values.volumes }} +{{ toYaml .Values.volumes | indent 6 }} +{{- end }} + containers: + - name: worker + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + env: + - name: SSHPORT + value: "{{ .Values.ssh.port }}" + {{- if .Values.ssh.useSecrets }} + - name: USESECRETS + value: "{{ .Values.ssh.useSecrets }}" + {{- end }} + {{- if .Values.master.env }} + {{- range $key, $value := .Values.master.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- end }} +{{- if .Values.worker.privileged }} + securityContext: + privileged: true +{{- end }} + ports: + - containerPort: {{ .Values.ssh.port }} + volumeMounts: + - name: {{ template "horovod.fullname" . }}-cm + mountPath: /horovod/generated + {{- if .Values.ssh.useSecrets }} + - name: {{ template "horovod.fullname" . }}-secret + readOnly: true + mountPath: "/etc/secret-volume" + {{- end }} +{{- if .Values.volumeMounts }} +{{ toYaml .Values.volumeMounts | indent 8 }} +{{- end }} + command: + - /horovod/generated/run.sh +{{- if .Values.ssh.useSecrets }} + readinessProbe: + exec: + command: + - /horovod/generated/check.sh + initialDelaySeconds: 1 + periodSeconds: 2 +{{- end }} + resources: +{{ toYaml .Values.worker.resources | indent 10 }} diff --git a/stable/horovod/values.yaml b/stable/horovod/values.yaml new file mode 100644 index 0000000000..83493ce44d --- /dev/null +++ b/stable/horovod/values.yaml @@ -0,0 +1,37 @@ +# Default values for horovod. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +useHostNetwork: false +useHostPID: false + +ssh: + port: 22 + useSecrets: false +# hostKey: |- +# -----BEGIN RSA PRIVATE KEY----- +# YourPrivateKey +# -----END RSA PRIVATE KEY----- +# hostKeyPub: |- +# ssh-rsa YourPublicKey + +worker: + number: 3 + podManagementPolicy: Parallel + image: + repository: uber/horovod + tag: 0.12.1-tf1.8.0-py3.5 + pullPolicy: IfNotPresent + # resources: + # limits: + # nvidia.com/gpu: 1 + # requests: + # nvidia.com/gpu: 1 + +master: + image: + repository: uber/horovod + tag: 0.12.1-tf1.8.0-py3.5 + pullPolicy: IfNotPresent + # args: + # - "mpiexec -n 3 --hostfile /horovod/generated/hostfile --mca orte_keep_fqdn_hostnames t --allow-run-as-root --display-map --tag-output --timestamp-output sh -c 'LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-9.0/targets/x86_64-linux/lib/stubs python /examples/tensorflow_mnist.py'"