From 4b5c62dc2ce0238d4021657238d7710c12db1b7d Mon Sep 17 00:00:00 2001 From: Lachlan Evenson Date: Mon, 31 Oct 2016 22:19:54 -0700 Subject: [PATCH 1/3] enable memory configuration on master and worker --- incubator/spark/templates/spark-master-deployment.yaml | 3 +++ incubator/spark/templates/spark-worker-deployment.yaml | 5 +++++ incubator/spark/values.yaml | 6 ++++++ 3 files changed, 14 insertions(+) diff --git a/incubator/spark/templates/spark-master-deployment.yaml b/incubator/spark/templates/spark-master-deployment.yaml index 96eefd17e5..a83e42d8ba 100644 --- a/incubator/spark/templates/spark-master-deployment.yaml +++ b/incubator/spark/templates/spark-master-deployment.yaml @@ -73,3 +73,6 @@ spec: requests: cpu: "{{.Values.Master.Cpu}}" memory: "{{.Values.Master.Memory}}" + env: + - name: SPARK_DAEMON_MEMORY + value: {{ default "1g" .Values.Master.DaemonMemory | quote }} \ No newline at end of file diff --git a/incubator/spark/templates/spark-worker-deployment.yaml b/incubator/spark/templates/spark-worker-deployment.yaml index 816fde9902..87ad8a838c 100644 --- a/incubator/spark/templates/spark-worker-deployment.yaml +++ b/incubator/spark/templates/spark-worker-deployment.yaml @@ -34,3 +34,8 @@ spec: requests: cpu: "{{.Values.Worker.Cpu}}" memory: "{{.Values.Worker.Memory}}" + env: + - name: SPARK_DAEMON_MEMORY + value: {{ default "1g" .Values.Worker.DaemonMemory | quote }} + - name: SPARK_WORKER_MEMORY + value: {{ default "1g" .Values.Worker.WorkerMemory | quote }} diff --git a/incubator/spark/values.yaml b/incubator/spark/values.yaml index 43490a4831..0fcec93090 100644 --- a/incubator/spark/values.yaml +++ b/incubator/spark/values.yaml @@ -13,6 +13,8 @@ Master: Memory: "512Mi" ServicePort: 7077 ContainerPort: 7077 + # Set Master JVM memory. Default 1g + #DaemonMemory: 2g WebUi: Name: spark-webui @@ -28,6 +30,10 @@ Worker: Cpu: "100m" Memory: "512Mi" ContainerPort: 8081 + # Set Worker JVM memory. Default 1g + #DaemonMemory: 2g + # Set how much total memory workers have to give executors + #ExecutorMemory: 2g Zeppelin: Name: zeppelin-controller From f4ded183919d3474f91be635a4dc8fcf0d2a7468 Mon Sep 17 00:00:00 2001 From: Lachlan Evenson Date: Mon, 31 Oct 2016 22:22:55 -0700 Subject: [PATCH 2/3] Update README with new values --- incubator/spark/README.md | 3 +++ incubator/spark/values.yaml | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/incubator/spark/README.md b/incubator/spark/README.md index 73f58aebbb..7a909c69f2 100644 --- a/incubator/spark/README.md +++ b/incubator/spark/README.md @@ -45,6 +45,7 @@ The following tables lists the configurable parameters of the Spark chart and th | `Master.Memory` | container requested memory | `512Mi` | | `Master.ServicePort` | k8s service port | `7077` | | `Master.ContainerPort` | Container listening port | `7077` | +| `Master.DaemonMemory` | Master JVM Xms and Xmx option | `1g` | ### Spark WebUi @@ -68,6 +69,8 @@ The following tables lists the configurable parameters of the Spark chart and th | `Worker.Memory` | container requested memory | `512Mi` | | `Worker.ContainerPort` | Container listening port | `7077` | | `Worker.CpuTargetPercentage` | k8s hpa cpu targetPercentage | `50` | +| `Worker.DaemonMemory` | Worker JVM Xms and Xmx setting | `1g` | +| `Worker.ExecutorMemory` | Worker memory available for executor | `1g` | diff --git a/incubator/spark/values.yaml b/incubator/spark/values.yaml index 0fcec93090..a984e61877 100644 --- a/incubator/spark/values.yaml +++ b/incubator/spark/values.yaml @@ -14,7 +14,7 @@ Master: ServicePort: 7077 ContainerPort: 7077 # Set Master JVM memory. Default 1g - #DaemonMemory: 2g + #DaemonMemory: 1g WebUi: Name: spark-webui @@ -31,9 +31,9 @@ Worker: Memory: "512Mi" ContainerPort: 8081 # Set Worker JVM memory. Default 1g - #DaemonMemory: 2g + #DaemonMemory: 1g # Set how much total memory workers have to give executors - #ExecutorMemory: 2g + #ExecutorMemory: 1g Zeppelin: Name: zeppelin-controller From 0e6ffef1011266ff11c437630be6f66534e6c9f5 Mon Sep 17 00:00:00 2001 From: Lachlan Evenson Date: Tue, 1 Nov 2016 10:53:13 -0700 Subject: [PATCH 3/3] Update invalid WorkerMemory variable name --- incubator/spark/templates/spark-worker-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/incubator/spark/templates/spark-worker-deployment.yaml b/incubator/spark/templates/spark-worker-deployment.yaml index 87ad8a838c..6d3734c987 100644 --- a/incubator/spark/templates/spark-worker-deployment.yaml +++ b/incubator/spark/templates/spark-worker-deployment.yaml @@ -38,4 +38,4 @@ spec: - name: SPARK_DAEMON_MEMORY value: {{ default "1g" .Values.Worker.DaemonMemory | quote }} - name: SPARK_WORKER_MEMORY - value: {{ default "1g" .Values.Worker.WorkerMemory | quote }} + value: {{ default "1g" .Values.Worker.ExecutorMemory | quote }}