From a7f323c9c4a28c64fd7a42dd8a60db74c009a628 Mon Sep 17 00:00:00 2001 From: jackgr Date: Sun, 15 Nov 2015 17:10:05 -0800 Subject: [PATCH 1/4] Code changes to support terminology rationalization. --- README.md | 29 +++ redis/v1/redis.jinja | 32 ++++ redis/v1/redis.jinja.schema | 10 + redis/v1/redis.yaml | 6 + replicatedservice/v1/replicatedservice.py | 181 ++++++++++++++++++ .../v1/replicatedservice.py.schema | 57 ++++++ 6 files changed, 315 insertions(+) create mode 100644 README.md create mode 100644 redis/v1/redis.jinja create mode 100644 redis/v1/redis.jinja.schema create mode 100644 redis/v1/redis.yaml create mode 100644 replicatedservice/v1/replicatedservice.py create mode 100644 replicatedservice/v1/replicatedservice.py.schema diff --git a/README.md b/README.md new file mode 100644 index 0000000000..c9435a2dec --- /dev/null +++ b/README.md @@ -0,0 +1,29 @@ +# Kubernetes Template Registry + +Welcome to the Kubernetes Template Registry! + +This registry holds Deployment Manager +[templates](https://github.com/kubernetes/deployment-manager/tree/master/docs/design/design.md#templates) +that you can use to deploy Kubernetes resources. + +For more information about installing and using Deployment Manager, see its +[README.md](https://github.com/kubernetes/deployment-manager/tree/master/README.md). + +## Organization + +The Kubernetes Template Registry is organized as a flat list of template +directories. Templates are versioned. The versions of a template live in version +directories under its template directory. + +For more information about Deployment Manager template registries, including +directory structure and template versions, see the +[template registry documentation](https://github.com/kubernetes/deployment-manager/tree/master/docs/templates/registry.md) + +## Contributing + +The Kubernetes Template Registry follows the +[git setup](https://github.com/kubernetes/kubernetes/blob/master/docs/devel/development.md#git-setup) +used by Kubernetes. + +You will also need to have a Contributor License Agreement on file, as described +in [CONTRIBUTING.md](../CONTRIBUTING.md). diff --git a/redis/v1/redis.jinja b/redis/v1/redis.jinja new file mode 100644 index 0000000000..fdcbfb7623 --- /dev/null +++ b/redis/v1/redis.jinja @@ -0,0 +1,32 @@ +{% set REDIS_PORT = 6379 %} +{% set WORKERS = properties['workers'] if properties and properties['workers'] else 2 %} + +resources: +- name: redis-master + type: https://raw.githubusercontent.com/kubernetes/deployment-manager/master/templates/replicatedservice/v1/replicatedservice.py + properties: + # This has to be overwritten since service names are hard coded in the code + service_name: redis-master + service_port: {{ REDIS_PORT }} + target_port: {{ REDIS_PORT }} + container_port: {{ REDIS_PORT }} + replicas: 1 + container_name: master + image: redis + +- name: redis-slave + type: https://raw.githubusercontent.com/kubernetes/deployment-manager/master/templates/replicatedservice/v1/replicatedservice.py + properties: + # This has to be overwritten since service names are hard coded in the code + service_name: redis-slave + service_port: {{ REDIS_PORT }} + container_port: {{ REDIS_PORT }} + replicas: {{ WORKERS }} + container_name: worker + image: kubernetes/redis-slave:v2 + # An example of how to specify env variables. + env: + - name: GET_HOSTS_FROM + value: env + - name: REDIS_MASTER_SERVICE_HOST + value: redis-master diff --git a/redis/v1/redis.jinja.schema b/redis/v1/redis.jinja.schema new file mode 100644 index 0000000000..cd550d65a0 --- /dev/null +++ b/redis/v1/redis.jinja.schema @@ -0,0 +1,10 @@ +info: + title: Redis cluster + description: Defines a redis cluster, using a single replica + replicatedservice for master and replicatedservice for workers. + +properties: + workers: + type: int + default: 2 + description: Number of worker replicas. diff --git a/redis/v1/redis.yaml b/redis/v1/redis.yaml new file mode 100644 index 0000000000..0b1b87b697 --- /dev/null +++ b/redis/v1/redis.yaml @@ -0,0 +1,6 @@ +imports: +- path: redis.jinja + +resources: +- name: redis + type: redis.jinja diff --git a/replicatedservice/v1/replicatedservice.py b/replicatedservice/v1/replicatedservice.py new file mode 100644 index 0000000000..6c226955e8 --- /dev/null +++ b/replicatedservice/v1/replicatedservice.py @@ -0,0 +1,181 @@ +###################################################################### +# Copyright 2015 The Kubernetes Authors All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +###################################################################### + +"""Defines a ReplicatedService type by creating both a Service and an RC. + +This module creates a typical abstraction for running a service in a +Kubernetes cluster, namely a replication controller and a service packaged +together into a single unit. +""" + +import yaml + +SERVICE_TYPE_COLLECTION = 'Service' +RC_TYPE_COLLECTION = 'ReplicationController' + + +def GenerateConfig(context): + """Generates a Replication Controller and a matching Service. + + Args: + context: Template context. See schema for context properties. + + Returns: + A Container Manifest as a YAML string. + """ + # YAML config that we're going to create for both RC & Service + config = {'resources': []} + + name = context.env['name'] + container_name = context.properties.get('container_name', name) + namespace = context.properties.get('namespace', 'default') + + # Define things that the Service cares about + service_name = context.properties.get('service_name', name + '-service') + service_type = SERVICE_TYPE_COLLECTION + + # Define things that the Replication Controller (rc) cares about + rc_name = name + '-rc' + rc_type = RC_TYPE_COLLECTION + + service = { + 'name': service_name, + 'type': service_type, + 'properties': { + 'apiVersion': 'v1', + 'kind': 'Service', + 'metadata': { + 'name': service_name, + 'namespace': namespace, + 'labels': GenerateLabels(context, service_name), + }, + 'spec': { + 'ports': [GenerateServicePorts(context, container_name)], + 'selector': GenerateLabels(context, name) + } + } + } + set_up_external_lb = context.properties.get('external_service', None) + if set_up_external_lb: + service['properties']['spec']['type'] = 'LoadBalancer' + config['resources'].append(service) + + rc = { + 'name': rc_name, + 'type': rc_type, + 'properties': { + 'apiVersion': 'v1', + 'kind': 'ReplicationController', + 'metadata': { + 'name': rc_name, + 'namespace': namespace, + 'labels': GenerateLabels(context, rc_name), + }, + 'spec': { + 'replicas': context.properties['replicas'], + 'selector': GenerateLabels(context, name), + 'template': { + 'metadata': { + 'labels': GenerateLabels(context, name), + }, + 'spec': { + 'containers': [ + { + 'env': GenerateEnv(context), + 'name': container_name, + 'image': context.properties['image'], + 'ports': [ + { + 'name': container_name, + 'containerPort': context.properties['container_port'], + } + ] + } + ] + } + } + } + } + } + + config['resources'].append(rc) + return yaml.dump(config) + + +def GenerateLabels(context, name): + """Generates labels either from the context.properties['labels'] or + generates a default label 'app':name + + We make a deep copy of the context.properties['labels'] section to avoid + linking in the yaml document, which I believe reduces readability of the + expanded template. If no labels are given, generate a default 'app':name. + + Args: + context: Template context, which can contain the following properties: + labels - Labels to generate + + Returns: + A dict containing labels in a name:value format + """ + tmp_labels = context.properties.get('labels', None) + ret_labels = {'app': name} + if isinstance(tmp_labels, dict): + for key, value in tmp_labels.iteritems(): + ret_labels[key] = value + return ret_labels + + +def GenerateServicePorts(context, name): + """Generates a ports section for a service. + + Args: + context: Template context, which can contain the following properties: + service_port - Port to use for the service + target_port - Target port for the service + protocol - Protocol to use. + + Returns: + A dict containing a port definition + """ + service_port = context.properties.get('service_port', None) + target_port = context.properties.get('target_port', None) + protocol = context.properties.get('protocol') + + ports = {} + if name: + ports['name'] = name + if service_port: + ports['port'] = service_port + if target_port: + ports['targetPort'] = target_port + if protocol: + ports['protocol'] = protocol + + return ports + +def GenerateEnv(context): + """Generates environmental variables for a pod. + + Args: + context: Template context, which can contain the following properties: + env - Environment variables to set. + + Returns: + A list containing env variables in dict format {name: 'name', value: 'value'} + """ + env = [] + tmp_env = context.properties.get('env', []) + for entry in tmp_env: + if isinstance(entry, dict): + env.append({'name': entry.get('name'), 'value': entry.get('value')}) + return env diff --git a/replicatedservice/v1/replicatedservice.py.schema b/replicatedservice/v1/replicatedservice.py.schema new file mode 100644 index 0000000000..c9ce310ab3 --- /dev/null +++ b/replicatedservice/v1/replicatedservice.py.schema @@ -0,0 +1,57 @@ +info: + title: Replicated Service + description: | + Defines a ReplicatedService type by creating both a Service and an RC. + + This module creates a typical abstraction for running a service in a + Kubernetes cluster, namely a replication controller and a service packaged + together into a single unit. + +required: +- image + +properties: + container_name: + type: string + description: Name to use for container. If omitted, name is used. + service_name: + type: string + description: Name to use for service. If omitted, name-service is used. + namespace: + type: string + description: Namespace to create resources in. If omitted, 'default' is + used. + default: default + protocol: + type: string + description: Protocol to use for the service. + service_port: + type: int + description: Port to use for the service. + target_port: + type: int + description: Target port to use for the service. + container_port: + type: int + description: Port to use for the container. + replicas: + type: int + description: Number of replicas to create in RC. + image: + type: string + description: Docker image to use for replicas. + labels: + type: object + description: Labels to apply. + env: + type: object + description: Environment variables to apply. + properties: + name: + type: string + value: + type: string + external_service: + type: boolean + description: If set to true, enable external load balancer. + From 73dd5257bf707a72dc4cf6f4f239fdaeb70c5248 Mon Sep 17 00:00:00 2001 From: Robert Leenders Date: Tue, 17 Nov 2015 15:25:24 -0800 Subject: [PATCH 2/4] Adds Wordpress example --- nfs/v1/nfs.jinja | 50 +++++ nfs/v1/nfs.jinja.schema | 25 +++ nfs/v1/nfs.yaml | 2 + replicatedservice/v2/replicatedservice.py | 194 ++++++++++++++++++ .../v2/replicatedservice.py.schema | 91 ++++++++ 5 files changed, 362 insertions(+) create mode 100644 nfs/v1/nfs.jinja create mode 100644 nfs/v1/nfs.jinja.schema create mode 100644 nfs/v1/nfs.yaml create mode 100644 replicatedservice/v2/replicatedservice.py create mode 100644 replicatedservice/v2/replicatedservice.py.schema diff --git a/nfs/v1/nfs.jinja b/nfs/v1/nfs.jinja new file mode 100644 index 0000000000..d8aca31178 --- /dev/null +++ b/nfs/v1/nfs.jinja @@ -0,0 +1,50 @@ +{% set PROPERTIES = properties or {} %} +{% set SERVER_IP = PROPERTIES['ip'] or '10.0.253.247' %} +{% set SERVER_PORT = PROPERTIES['port'] or 2049 %} +{% set SERVER_DISK = PROPERTIES['disk'] or 'nfs-disk' %} +{% set SERVER_DISK_FSTYPE = PROPERTIES['fstype'] or 'ext4' %} +{% set CLAIM_NAME = PROPERTIES['claim-name'] or 'nfs' %} + +resources: +- name: nfs + type: https://raw.githubusercontent.com/leendersr/deployment-manager/master/templates/replicatedservice/v2/replicatedservice.py + properties: + service_port: {{ SERVER_PORT }} + container_port: {{ SERVER_PORT }} + replicas: 1 # Has to be 1 because of the persistent disk + image: gcr.io/google_containers/volume-nfs + privileged: true + cluster_ip: {{ SERVER_IP }} + volumes: + - mount_path: /mnt/data + gcePersistentDisk: + pdName: {{ SERVER_DISK }} + fsType: {{ SERVER_DISK_FSTYPE }} +- name: nfs-pvc + type: PersistentVolumeClaim + properties: + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: {{ CLAIM_NAME }} + spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Mi +- name: nfs-pv + type: PersistentVolume + properties: + apiVersion: v1 + kind: PersistentVolume + metadata: + name: {{ CLAIM_NAME }} + spec: + capacity: + storage: 1Mi + accessModes: + - ReadWriteMany + nfs: + server: {{ SERVER_IP }} + path: "/" diff --git a/nfs/v1/nfs.jinja.schema b/nfs/v1/nfs.jinja.schema new file mode 100644 index 0000000000..86d26bb9cb --- /dev/null +++ b/nfs/v1/nfs.jinja.schema @@ -0,0 +1,25 @@ +info: + title: NFS + description: Defines an NFS service. + +properties: + ip: + type: string + default: 10.0.253.247 + description: The IP of the NFS service. + port: + type: int + default: 2049 + description: The port of the NFS service. + disk: + type: string + default: nfs-disk + description: The name of the persistent disk the NFS service uses. + fstype: + type: string + default: ext4 + description: The filesystem the disk of the NFS service uses. + claim-name: + type: string + default: nfs + description: The name of the PersistentVolumeClaim diff --git a/nfs/v1/nfs.yaml b/nfs/v1/nfs.yaml new file mode 100644 index 0000000000..4d5c68d8ab --- /dev/null +++ b/nfs/v1/nfs.yaml @@ -0,0 +1,2 @@ +- name: nfs + type: https://raw.githubusercontent.com/leendersr/deployment-manager/master/templates/nfs/v1/nfs.jinja diff --git a/replicatedservice/v2/replicatedservice.py b/replicatedservice/v2/replicatedservice.py new file mode 100644 index 0000000000..a6ef0d71ed --- /dev/null +++ b/replicatedservice/v2/replicatedservice.py @@ -0,0 +1,194 @@ +"""Defines a ReplicatedService type by creating both a Service and an RC. + +This module creates a typical abstraction for running a service in a +Kubernetes cluster, namely a replication controller and a service packaged +together into a single unit. +""" + +import yaml + +SERVICE_TYPE_COLLECTION = 'Service' +RC_TYPE_COLLECTION = 'ReplicationController' + + +def GenerateConfig(context): + """Generates a Replication Controller and a matching Service. + + Args: + context: Template context. See schema for context properties + + Returns: + A Container Manifest as a YAML string. + """ + # YAML config that we're going to create for both RC & Service + config = {'resources': []} + + name = context.env['name'] + container_name = context.properties.get('container_name', name) + namespace = context.properties.get('namespace', 'default') + + # Define things that the Service cares about + service_name = context.properties.get('service_name', name + '-service') + service_type = SERVICE_TYPE_COLLECTION + + # Define things that the Replication Controller (rc) cares about + rc_name = context.properties.get('rc_name', name + '-rc') + rc_type = RC_TYPE_COLLECTION + + service = { + 'name': service_name, + 'type': service_type, + 'properties': { + 'apiVersion': 'v1', + 'kind': 'Service', + 'namespace': namespace, + 'metadata': { + 'name': service_name, + 'labels': GenerateLabels(context, service_name), + }, + 'spec': { + 'ports': [GenerateServicePorts(context, container_name)], + 'selector': GenerateLabels(context, name) + } + } + } + set_up_external_lb = context.properties.get('external_service', None) + if set_up_external_lb: + service['properties']['spec']['type'] = 'LoadBalancer' + cluster_ip = context.properties.get('cluster_ip', None) + if cluster_ip: + service['properties']['spec']['clusterIP'] = cluster_ip + config['resources'].append(service) + + rc = { + 'name': rc_name, + 'type': rc_type, + 'properties': { + 'apiVersion': 'v1', + 'kind': 'ReplicationController', + 'namespace': namespace, + 'metadata': { + 'name': rc_name, + 'labels': GenerateLabels(context, rc_name), + }, + 'spec': { + 'replicas': context.properties['replicas'], + 'selector': GenerateLabels(context, name), + 'template': { + 'metadata': { + 'labels': GenerateLabels(context, name), + }, + 'spec': { + 'containers': [ + { + 'env': GenerateEnv(context), + 'name': container_name, + 'image': context.properties['image'], + 'ports': [ + { + 'name': container_name, + 'containerPort': context.properties['container_port'], + } + ], + } + ], + } + } + } + } + } + + # Set up volume mounts + if context.properties.get('volumes', None): + rc['properties']['spec']['template']['spec']['containers'][0]['volumeMounts'] = [] + rc['properties']['spec']['template']['spec']['volumes'] = [] + for volume in context.properties['volumes']: + # mountPath should be unique + volume_name = volume['mount_path'].replace('/', '-').lstrip('-') + '-storage' + rc['properties']['spec']['template']['spec']['containers'][0]['volumeMounts'].append( + { + 'name': volume_name, + 'mountPath': volume['mount_path'] + } + ) + del volume['mount_path'] + volume['name'] = volume_name + rc['properties']['spec']['template']['spec']['volumes'].append(volume) + + if context.properties.get('privileged', False): + rc['properties']['spec']['template']['spec']['containers'][0]['securityContext'] = { + 'privileged': True + } + + config['resources'].append(rc) + return yaml.dump(config) + + +# Generates labels either from the context.properties['labels'] or generates +# a default label 'name':name +def GenerateLabels(context, name): + """Generates labels from context.properties['labels'] or creates default. + + We make a deep copy of the context.properties['labels'] section to avoid + linking in the yaml document, which I believe reduces readability of the + expanded template. If no labels are given, generate a default 'name':name. + + Args: + context: Template context, which can contain the following properties: + labels - Labels to generate + + Returns: + A dict containing labels in a name:value format + """ + tmp_labels = context.properties.get('labels', None) + ret_labels = {'name': name} + if isinstance(tmp_labels, dict): + for key, value in tmp_labels.iteritems(): + ret_labels[key] = value + return ret_labels + + +def GenerateServicePorts(context, name): + """Generates a ports section for a service. + + Args: + context: Template context, which can contain the following properties: + service_port - Port to use for the service + target_port - Target port for the service + protocol - Protocol to use. + + Returns: + A dict containing a port definition + """ + service_port = context.properties.get('service_port', None) + target_port = context.properties.get('target_port', None) + protocol = context.properties.get('protocol') + + ports = {} + if name: + ports['name'] = name + if service_port: + ports['port'] = service_port + if target_port: + ports['targetPort'] = target_port + if protocol: + ports['protocol'] = protocol + + return ports + +def GenerateEnv(context): + """Generates environmental variables for a pod. + + Args: + context: Template context, which can contain the following properties: + env - Environment variables to set. + + Returns: + A list containing env variables in dict format {name: 'name', value: 'value'} + """ + env = [] + tmp_env = context.properties.get('env', []) + for entry in tmp_env: + if isinstance(entry, dict): + env.append({'name': entry.get('name'), 'value': entry.get('value')}) + return env diff --git a/replicatedservice/v2/replicatedservice.py.schema b/replicatedservice/v2/replicatedservice.py.schema new file mode 100644 index 0000000000..712ffd3159 --- /dev/null +++ b/replicatedservice/v2/replicatedservice.py.schema @@ -0,0 +1,91 @@ +info: + title: Replicated Service + description: | + Defines a ReplicatedService type by creating both a Service and an RC. + + This module creates a typical abstraction for running a service in a + Kubernetes cluster, namely a replication controller and a service packaged + together into a single unit. + +required: +- image + +properties: + container_name: + type: string + description: Name to use for container. If omitted, name is used. + service_name: + type: string + description: Name to use for service. If omitted, name-service is used. + namespace: + type: string + description: Namespace to create resources in. If omitted, 'default' is + used. + default: default + protocol: + type: string + description: Protocol to use for the service. + service_port: + type: int + description: Port to use for the service. + target_port: + type: int + description: Target port to use for the service. + container_port: + type: int + description: Port to use for the container. + replicas: + type: int + description: Number of replicas to create in RC. + image: + type: string + description: Docker image to use for replicas. + labels: + type: object + description: Labels to apply. + env: + type: array + description: Environment variables to apply. + properties: + name: + type: string + value: + type: string + external_service: + type: boolean + description: If set to true, enable external load balancer. + cluster_ip: + type: string + description: IP to use for the service + privileged: + type: boolean + description: If set to true, enable privileged container + volumes: + type: array + description: Volumes to mount. + items: + type: object + properties: + mounth_path: + type: string + description: Path to mount volume + # See https://cloud.google.com/container-engine/docs/spec-schema?hl=en for possible volumes. Since we only use gcePersistentDisk and NFS in our examples we have only added these ones. + oneOf: + gcePersistentDisk: + pdName: + type: string + description: Persistent's disk name + fsType: + type: string + description: Filesystem type of the persistent disk + nfs: + server: + type: string + description: The hostname or IP address of the NFS server + path: + type: string + description: The path that is exported by the NFS server + readOnly: + type: boolean + description: Forces the NFS export to be mounted with read-only permissions + From 2b96d9145ca6dc71437280f485141bbc14c4a04b Mon Sep 17 00:00:00 2001 From: Robert Leenders Date: Thu, 19 Nov 2015 15:30:14 -0800 Subject: [PATCH 3/4] Updates type references --- nfs/v1/nfs.jinja | 2 +- nfs/v1/nfs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nfs/v1/nfs.jinja b/nfs/v1/nfs.jinja index d8aca31178..12da6e3bfa 100644 --- a/nfs/v1/nfs.jinja +++ b/nfs/v1/nfs.jinja @@ -7,7 +7,7 @@ resources: - name: nfs - type: https://raw.githubusercontent.com/leendersr/deployment-manager/master/templates/replicatedservice/v2/replicatedservice.py + type: https://raw.githubusercontent.com/kubernetes/deployment-manager/master/templates/replicatedservice/v2/replicatedservice.py properties: service_port: {{ SERVER_PORT }} container_port: {{ SERVER_PORT }} diff --git a/nfs/v1/nfs.yaml b/nfs/v1/nfs.yaml index 4d5c68d8ab..650a12f856 100644 --- a/nfs/v1/nfs.yaml +++ b/nfs/v1/nfs.yaml @@ -1,2 +1,2 @@ - name: nfs - type: https://raw.githubusercontent.com/leendersr/deployment-manager/master/templates/nfs/v1/nfs.jinja + type: https://raw.githubusercontent.com/kubernetes/deployment-manager/master/templates/nfs/v1/nfs.jinja From 4b4453d9f75160b273f2b30d885b38aba1efe860 Mon Sep 17 00:00:00 2001 From: Zach Loafman Date: Fri, 20 Nov 2015 09:33:41 -0800 Subject: [PATCH 4/4] Spark: Add DM template for Spark This is a first-cut template based on https://github.com/kubernetes/kubernetes/tree/master/examples/spark with minor templating. As I iterate on that example to add PDs, etc. (which will add much more variance), I'll iterate here, too. --- spark/v1/example.yaml | 6 ++ spark/v1/spark.jinja | 113 ++++++++++++++++++++++++++++++++++++ spark/v1/spark.jinja.schema | 50 ++++++++++++++++ 3 files changed, 169 insertions(+) create mode 100644 spark/v1/example.yaml create mode 100644 spark/v1/spark.jinja create mode 100644 spark/v1/spark.jinja.schema diff --git a/spark/v1/example.yaml b/spark/v1/example.yaml new file mode 100644 index 0000000000..c9f6b4c0b8 --- /dev/null +++ b/spark/v1/example.yaml @@ -0,0 +1,6 @@ +imports: +- path: spark.jinja + +resources: +- name: spark + type: spark.jinja diff --git a/spark/v1/spark.jinja b/spark/v1/spark.jinja new file mode 100644 index 0000000000..ec0a777aea --- /dev/null +++ b/spark/v1/spark.jinja @@ -0,0 +1,113 @@ +{% set PROPERTIES = properties or {} %} +{% set WORKERS = PROPERTIES['workers'] or 2 %} +{% set REPO = PROPERTIES['repository'] or 'gcr.io/google_containers' %} +{% set SPARK_VERSION = PROPERTIES['spark_version'] or '1.5.1_v2' %} +{% set ZEPPELIN_VERSION = PROPERTIES['zeppelin_version'] or 'v0.5.5_v2' %} +{% set MASTER_CPU = PROPERTIES['master_cpu'] or '100m' %} +{% set ZEPPELIN_CPU = PROPERTIES['zeppelin_cpu'] or '100m' %} +{% set WORKER_CPU = PROPERTIES['worker_cpu'] or '100m' %} + +# TODO(zmerlynn): This can't be a standard replicatedservice (yet) because of: +# - the double containerPort +# - the cpu request +# (neither are possible in the type as of v2) +resources: +- name: spark-master + type: ReplicationController + properties: + kind: ReplicationController + apiVersion: v1 + metadata: + name: spark-master-controller + spec: + replicas: 1 + selector: + component: spark-master + template: + metadata: + labels: + component: spark-master + spec: + containers: + - name: spark-master + image: {{ REPO }}/spark-master:{{ SPARK_VERSION }} + ports: + - containerPort: 7077 + - containerPort: 8080 + resources: + requests: + cpu: {{ MASTER_CPU }} +- name: spark-master-service + type: Service + properties: + kind: Service + apiVersion: v1 + metadata: + name: spark-master + spec: + ports: + - port: 7077 + targetPort: 7077 + selector: + component: spark-master +- name: spark-webui + type: Service + properties: + kind: Service + apiVersion: v1 + metadata: + name: spark-webui + spec: + ports: + - port: 8080 + targetPort: 8080 + selector: + component: spark-master +- name: spark-worker-controller + type: ReplicationController + properties: + kind: ReplicationController + apiVersion: v1 + metadata: + name: spark-worker-controller + spec: + replicas: 3 + selector: + component: spark-worker + template: + metadata: + labels: + component: spark-worker + spec: + containers: + - name: spark-worker + image: {{ REPO }}/spark-worker:{{ SPARK_VERSION }} + ports: + - containerPort: 8081 + resources: + requests: + cpu: {{ WORKER_CPU }} +- name: zeppelin-controller + type: ReplicationController + properties: + kind: ReplicationController + apiVersion: v1 + metadata: + name: zeppelin-controller + spec: + replicas: 1 + selector: + component: zeppelin + template: + metadata: + labels: + component: zeppelin + spec: + containers: + - name: zeppelin + image: {{ REPO }}/zeppelin:{{ ZEPPELIN_VERSION }} + ports: + - containerPort: 8080 + resources: + requests: + cpu: {{ ZEPPELIN_CPU }} diff --git a/spark/v1/spark.jinja.schema b/spark/v1/spark.jinja.schema new file mode 100644 index 0000000000..f0f2f1f1b2 --- /dev/null +++ b/spark/v1/spark.jinja.schema @@ -0,0 +1,50 @@ +info: + title: Spark cluster with Zeppelin front-end + description: | + Defines a Spark cluster with a single master in standalone mode, + and a Zeppelin web notebook front-end. After deploying the + cluster, the Spark WebUI can be accessed by starting: + + kubectl proxy --port=8001 & + + Then visiting: + + http://localhost:8001/api/v1/proxy/namespaces/default/services/spark-webui/ + + The Zeppelin WebUI can be access by finding the Zeppelin pod and port-forwarding: + + kubectl get pods -lcomponent=zeppelin # Take the pod from here + kubectl port-forward zeppelin-controller-abcef :8080 + + Then visit the forwarded port. + +properties: + master_cpu: + type: string + default: 100m + description: CPU request for the master (in KCUs) (see http://kubernetes.io/v1.1/docs/design/resources.html) + + workers: + type: int + default: 3 + description: Number of Spark workers. + + worker_cpu: + type: string + default: 100m + description: CPU request for each worker (in KCUs) (see http://kubernetes.io/v1.1/docs/design/resources.html) + + repository: + type: string + default: gcr.io/google_containers + description: Docker repo that houses the {spark-worker, spark-master, zeppelin} images. + + spark_version: + type: string + default: 1.5.1_v2 + description: spark-worker / spark-master image version to use + + zeppelin_version: + type: string + default: 0.5.5_v2 + description: zeppelin image version to use