From c9d5a723bbab778d1cf821aa3690fb044a94d552 Mon Sep 17 00:00:00 2001 From: Chaoran Yu Date: Fri, 9 Nov 2018 08:45:27 -0500 Subject: [PATCH] Update to Spark 2.4.0 (#9135) Signed-off-by: Chaoran Yu --- stable/spark-history-server/Chart.yaml | 4 ++-- stable/spark-history-server/README.md | 10 +++++----- stable/spark-history-server/values.yaml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/stable/spark-history-server/Chart.yaml b/stable/spark-history-server/Chart.yaml index 72cfd93c68..8a1bf80510 100644 --- a/stable/spark-history-server/Chart.yaml +++ b/stable/spark-history-server/Chart.yaml @@ -1,6 +1,6 @@ name: spark-history-server -version: 0.1.0 -appVersion: 2.3.1 +version: 0.1.1 +appVersion: 2.4.0 description: A Helm chart for Spark History Server home: https://spark.apache.org sources: diff --git a/stable/spark-history-server/README.md b/stable/spark-history-server/README.md index e8087a8477..5a3535c93c 100644 --- a/stable/spark-history-server/README.md +++ b/stable/spark-history-server/README.md @@ -80,8 +80,8 @@ The following tables lists the configurable parameters of the Spark History Seve | hdfs.hdfsSiteConfigMap |The name of the ConfigMap for hdfs-site.xml|hdfs-site| | hdfs.coreSiteConfigMap |The name of the ConfigMap for core-site.xml|core-site| | hdfs.HADOOP_CONF_DIR |The directory containing core-site.xml and hdfs-site.xml in the image|/etc/hadoop| -| image.repository |The Docker image used to start the history server daemon|lightbend/spark| -| image.tag |The tag of the image|2.4.0| +| image.repository |The Docker image used to start the history server daemon|gcr.io/spark-operator/spark| +| image.tag |The tag of the image|v2.4.0| | service.type |The type of history server service that exposes the UI|LoadBalancer| | service.port |The port on which the service UI can be accessed.|18080| | pvc.enablePVC |Whether to use PVC storage|true| @@ -136,7 +136,7 @@ bin/spark-submit \ --conf spark.eventLog.enabled=true \ --conf spark.eventLog.dir=file:/mnt \ --conf spark.executor.instances=2 \ - --conf spark.kubernetes.container.image=lightbend/spark:k8s-rc-2.4 \ + --conf spark.kubernetes.container.image=gcr.io/spark-operator/spark:v2.4.0 \ --conf spark.kubernetes.container.image.pullPolicy=Always \ --conf spark.kubernetes.driver.volumes.persistentVolumeClaim.checkpointpvc.options.claimName=nfs-pvc \ --conf spark.kubernetes.driver.volumes.persistentVolumeClaim.checkpointpvc.mount.path=/mnt \ @@ -178,8 +178,8 @@ bin/spark-submit \ --conf spark.hadoop.google.cloud.auth.service.account.json.keyfile=/etc/secrets/sparkonk8s.json \ --conf spark.kubernetes.driver.secrets.history-secrets=/etc/secrets \ --conf spark.kubernetes.executor.secrets.history-secrets=/etc/secrets \ - --conf spark.kubernetes.container.image=lightbend/spark:k8s-daily-gcs \ - local:///opt/spark/examples/jars/spark-examples_2.11-2.5.0-SNAPSHOT.jar + --conf spark.kubernetes.container.image=lightbend/spark:2.4.0-gcs \ + local:///opt/spark/examples/jars/spark-examples_2.11-2.4.0.jar ``` Note that the image for your Spark job (i.e. `spark.kubernetes.container.image`, `spark.kubernetes.driver.container.image` and `spark.kubernetes.executor.container.image`) needs to have the [GCS connector](https://cloud.google.com/dataproc/docs/concepts/connectors/cloud-storage) dependency, otherwise the `gs://` scheme won't be recognized. diff --git a/stable/spark-history-server/values.yaml b/stable/spark-history-server/values.yaml index fd3d5d0de0..93d9eb516e 100644 --- a/stable/spark-history-server/values.yaml +++ b/stable/spark-history-server/values.yaml @@ -9,7 +9,7 @@ serviceAccount: image: repository: gcr.io/spark-operator/spark - tag: v2.3.1 + tag: v2.4.0 pullPolicy: IfNotPresent service: