Merge pull request #173 from lachie83/feat-add-spark-memory-settings

(feat) Add memory settings to Spark chart

incubator/spark/README.md

@@ -45,6 +45,7 @@ The following tables lists the configurable parameters of the Spark chart and th
 | `Master.Memory`         | container requested memory         | `512Mi`                                                    |
 | `Master.ServicePort`    | k8s service port                   | `7077`                                                     |
 | `Master.ContainerPort`  | Container listening port           | `7077`                                                     |
+| `Master.DaemonMemory`   | Master JVM Xms and Xmx option      | `1g`                                                       |
 
 ### Spark WebUi
 
@@ -68,6 +69,8 @@ The following tables lists the configurable parameters of the Spark chart and th
 | `Worker.Memory`              | container requested memory         | `512Mi`                                                    |
 | `Worker.ContainerPort`       | Container listening port           | `7077`                                                     |
 | `Worker.CpuTargetPercentage` | k8s hpa cpu targetPercentage       | `50`                                                       |
+| `Worker.DaemonMemory`        | Worker JVM Xms and Xmx setting     | `1g`                                                       |
+| `Worker.ExecutorMemory`      | Worker memory available for executor | `1g`                                                       |
 
 
 

incubator/spark/templates/spark-master-deployment.yaml

@@ -73,3 +73,6 @@ spec:
             requests:
               cpu: "{{.Values.Master.Cpu}}"
               memory: "{{.Values.Master.Memory}}"
+          env:
+          - name: SPARK_DAEMON_MEMORY
+            value: {{ default "1g" .Values.Master.DaemonMemory | quote }}

incubator/spark/templates/spark-worker-deployment.yaml

@@ -34,3 +34,8 @@ spec:
             requests:
               cpu: "{{.Values.Worker.Cpu}}"
               memory: "{{.Values.Worker.Memory}}"
+          env:
+          - name: SPARK_DAEMON_MEMORY
+            value: {{ default "1g" .Values.Worker.DaemonMemory | quote }}
+          - name: SPARK_WORKER_MEMORY
+            value: {{ default "1g" .Values.Worker.ExecutorMemory | quote }}

incubator/spark/values.yaml

@@ -13,6 +13,8 @@ Master:
   Memory: "512Mi"
   ServicePort: 7077
   ContainerPort: 7077
+  # Set Master JVM memory. Default 1g
+  #DaemonMemory: 1g
 
 WebUi:
   Name: spark-webui
@@ -28,6 +30,10 @@ Worker:
   Cpu: "100m"
   Memory: "512Mi"
   ContainerPort: 8081
+  # Set Worker JVM memory. Default 1g
+  #DaemonMemory: 1g
+  # Set how much total memory workers have to give executors
+  #ExecutorMemory: 1g
 
 Zeppelin:
   Name: zeppelin-controller