diff --git a/incubator/sparkoperator/Chart.yaml b/incubator/sparkoperator/Chart.yaml index c5cfe7f0a5..60244012b4 100644 --- a/incubator/sparkoperator/Chart.yaml +++ b/incubator/sparkoperator/Chart.yaml @@ -1,6 +1,6 @@ name: sparkoperator description: A Helm chart for Spark on Kubernetes operator -version: 0.1.13 +version: 0.2.0 appVersion: v1beta1-0.8.1-2.4.0 kubeVersion: ">=1.8.0-0" keywords: diff --git a/incubator/sparkoperator/README.md b/incubator/sparkoperator/README.md index ba0f05bcde..88ebb8375f 100644 --- a/incubator/sparkoperator/README.md +++ b/incubator/sparkoperator/README.md @@ -1,6 +1,6 @@ ### Helm Chart for Spark Operator -This is the Helm chart for the [Spark-on-Kubernetes Operator](https://github.com/GoogleCloudPlatform/spark-on-k8s-operator). +This is the Helm chart for the [Kubernetes Operator for Apache Spark](https://github.com/GoogleCloudPlatform/spark-on-k8s-operator). #### Prerequisites @@ -12,7 +12,7 @@ The chart can be installed by running: ```bash $ helm repo add incubator http://storage.googleapis.com/kubernetes-charts-incubator -$ helm install incubator/sparkoperator --namespace spark-operator +$ helm install incubator/sparkoperator --namespace spark-operator --set sparkJobNamespace=default ``` Note that you need to use the `--namespace` flag during `helm install` to specify in which namespace you want to install the operator. The namespace can be existing or not. When it's not available, Helm would take care of creating the namespace. Note that this namespace has no relation to the namespace where you would like to deploy Spark jobs (i.e. the setting `sparkJobNamespace` shown in the table below). They can be the same namespace or different ones. @@ -24,13 +24,14 @@ The following table lists the configurable parameters of the Spark operator char | Parameter | Description | Default | | ------------------------- | ------------------------------------------------------------ | -------------------------------------- | | `operatorImageName` | The name of the operator image | `gcr.io/spark-operator/spark-operator` | -| `operatorVersion` | The version of the operator to install | `v2.4.0-v1beta1-latest` | +| `operatorVersion` | The version of the operator to install | `v2.4.0-v1beta1-0.8.1` | | `imagePullPolicy` | Docker image pull policy | `IfNotPresent` | -| `sparkJobNamespace` | K8s namespace where Spark jobs are to be deployed | `default` | -| `enableWebhook` | Whether to enable mutating admission webhook | false | +| `sparkJobNamespace` | K8s namespace where Spark jobs are to be deployed | `` | +| `enableWebhook` | Whether to enable mutating admission webhook | false | | `enableMetrics` | Whether to expose metrics to be scraped by Premetheus | true | -| `controllerThreads` | Number of worker threads used by the SparkApplication controller | 10 | -| `ingressUrlFormat` | Ingress URL format | "" | +| `controllerThreads` | Number of worker threads used by the SparkApplication controller | 10 | +| `ingressUrlFormat` | Ingress URL format | "" | +| `logLevel` | Logging verbosity level | 2 | | `installCrds` | Whether to install CRDs | true | | `metricsPort` | Port for the metrics endpoint | 10254 | | `metricsEndpoint` | Metrics endpoint | "/metrics" | diff --git a/incubator/sparkoperator/templates/crd-cleanup-job.yaml b/incubator/sparkoperator/templates/crd-cleanup-job.yaml new file mode 100644 index 0000000000..6f3f3585ed --- /dev/null +++ b/incubator/sparkoperator/templates/crd-cleanup-job.yaml @@ -0,0 +1,44 @@ +{{ if .Values.installCrds }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "sparkoperator.fullname" . }}-crd-cleanup + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": hook-succeeded + labels: + app.kubernetes.io/name: {{ include "sparkoperator.name" . }} + helm.sh/chart: {{ include "sparkoperator.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + template: + spec: + serviceAccountName: {{ include "sparkoperator.serviceAccountName" . }} + restartPolicy: OnFailure + containers: + - name: delete-sparkapp-crd + image: {{ .Values.operatorImageName }}:{{ .Values.operatorVersion }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + command: + - "/bin/sh" + - "-c" + - "curl -ik \ + -X DELETE \ + -H \"Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\" \ + -H \"Accept: application/json\" \ + -H \"Content-Type: application/json\" \ + https://kubernetes.default.svc/apis/apiextensions.k8s.io/v1beta1/customresourcedefinitions/sparkapplications.sparkoperator.k8s.io" + - name: delete-scheduledsparkapp-crd + image: {{ .Values.operatorImageName }}:{{ .Values.operatorVersion }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + command: + - "/bin/sh" + - "-c" + - "curl -ik \ + -X DELETE \ + -H \"Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\" \ + -H \"Accept: application/json\" \ + -H \"Content-Type: application/json\" \ + https://kubernetes.default.svc/apis/apiextensions.k8s.io/v1beta1/customresourcedefinitions/scheduledsparkapplications.sparkoperator.k8s.io" +{{ end }} diff --git a/incubator/sparkoperator/templates/spark-operator-deployment.yaml b/incubator/sparkoperator/templates/spark-operator-deployment.yaml index 4b5f0e4e21..6530f45b05 100644 --- a/incubator/sparkoperator/templates/spark-operator-deployment.yaml +++ b/incubator/sparkoperator/templates/spark-operator-deployment.yaml @@ -56,7 +56,7 @@ spec: - containerPort: {{ .Values.metricsPort }} {{ end }} args: - - -v=2 + - -v={{ .Values.logLevel }} - -namespace={{ .Values.sparkJobNamespace }} - -ingress-url-format={{ .Values.ingressUrlFormat }} - -install-crds={{ .Values.installCrds }} diff --git a/incubator/sparkoperator/templates/webhook-cleanup-job.yaml b/incubator/sparkoperator/templates/webhook-cleanup-job.yaml index d6d9df7cae..b6f1a971e1 100644 --- a/incubator/sparkoperator/templates/webhook-cleanup-job.yaml +++ b/incubator/sparkoperator/templates/webhook-cleanup-job.yaml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: {{ include "sparkoperator.fullname" . }}-cleanup + name: {{ include "sparkoperator.fullname" . }}-webhook-cleanup annotations: "helm.sh/hook": pre-delete, pre-upgrade "helm.sh/hook-delete-policy": hook-succeeded diff --git a/incubator/sparkoperator/values.yaml b/incubator/sparkoperator/values.yaml index 6c69cfedf3..197e255623 100644 --- a/incubator/sparkoperator/values.yaml +++ b/incubator/sparkoperator/values.yaml @@ -31,3 +31,5 @@ webhookPort: 8080 ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ ## nodeSelector: {} + +logLevel: 2