[incubating/sparkoperator] Introduce optional remove before install CRDs flag (#18397)

* Introduce optional remove before install CRDs flag

Signed-off-by: Roland Johann <rmmjohann@gmail.com>

* Update docs

Signed-off-by: Roland Johann <rmmjohann@gmail.com>

* Bump version

Signed-off-by: Roland Johann <rmmjohann@gmail.com>

* Bump version

Signed-off-by: Roland Johann <rmmjohann@gmail.com>

* Bump version to 0.5.0

Signed-off-by: Roland Johann <rmmjohann@gmail.com>

incubator/sparkoperator/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 name: sparkoperator
 description: A Helm chart for Spark on Kubernetes operator
-version: 0.4.7
+version: 0.5.0
 appVersion: v1beta2-1.0.1-2.4.4
 keywords:
   - spark

incubator/sparkoperator/README.md

@@ -35,6 +35,7 @@ The following table lists the configurable parameters of the Spark operator char
 | `ingressUrlFormat`        | Ingress URL format                                           | ""                                     |
 | `logLevel`                | Logging verbosity level                                      | 2                                      |
 | `installCrds`             | Whether to install CRDs                                      | true                                   |
+| `cleanupCrdsBeforeInstall` | Remove CRDs before running the crd-install hook on changes. | `false` |
 | `metricsPort`             | Port for the metrics endpoint                                | 10254                                  |
 | `metricsEndpoint`         | Metrics endpoint                                             | "/metrics"                             |
 | `metricsPrefix`           | Prefix for the metrics                                       | ""                                     |
@@ -48,3 +49,7 @@ The following table lists the configurable parameters of the Spark operator char
 | `securityContext`         | Defines security context for operator container               | `{}`
 
 Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`.
+
+#### Contributing
+
+When making changes to values.yaml, update the files in `ci/` by running `hack/update-ci.sh`.

incubator/sparkoperator/ci/test-values.yaml

@@ -0,0 +1,60 @@
+operatorImageName: gcr.io/spark-operator/spark-operator
+operatorVersion: v1beta2-1.0.1-2.4.4
+imagePullPolicy: IfNotPresent
+imagePullSecrets: []
+replicas: 1
+
+rbac:
+  create: true
+
+serviceAccounts:
+  spark:
+    create: true
+    name:
+  sparkoperator:
+    create: true
+    name:
+
+sparkJobNamespace: ""
+installCrds: true
+cleanupCrdsBeforeInstall: true
+controllerThreads: 10
+resyncInterval: 30
+ingressUrlFormat: ""
+logLevel: 2
+
+securityContext: {}
+
+enableWebhook: false
+webhookPort: 8080
+
+enableMetrics: true
+metricsPort: 10254
+metricsEndpoint: "/metrics"
+metricsPrefix: ""
+
+## Node labels for pod assignment
+## Ref: https://kubernetes.io/docs/user-guide/node-selection/
+##
+nodeSelector: {}
+
+podAnnotations: {}
+
+## Resources for the sparkoperator deployment
+## Ref: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/
+##
+resources: {}
+
+## Whether to enable batch scheduler for pod scheduling,
+## if enabled, end user can specify batch scheduler name in spark application.
+enableBatchScheduler: false
+
+## Whether to enable the ResourceQuota enforcement for SparkApplication resources.
+## Requires the webhook to be enabled by setting enableWebhook to true.
+## Ref: https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/docs/user-guide.md#enabling-resource-quota-enforcement.
+enableResourceQuotaEnforcement: false
+
+## Whether to enable leader election when the operator Deployment has more than one replica.
+## Only applicable when `replicas` is set to a value greater than 1.
+## Ref: https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/docs/user-guide.md#enabling-leader-election-for-high-availability.
+enableLeaderElection: false

incubator/sparkoperator/hack/update-ci.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+sed \
+  -e 's/cleanupCrdsBeforeInstall: false/cleanupCrdsBeforeInstall: true/' \
+  values.yaml > ci/test-values.yaml

incubator/sparkoperator/templates/crds.yaml

@@ -4,6 +4,10 @@ kind: CustomResourceDefinition
 metadata:
   creationTimestamp: null
   name: sparkapplications.sparkoperator.k8s.io
+annotations:
+{{- if .Values.cleanupCrdsBeforeInstall }}
+  "helm.sh/hook-delete-policy": "before-hook-creation"
+{{- end }}
 spec:
   group: sparkoperator.k8s.io
   names:
@@ -2540,6 +2544,10 @@ kind: CustomResourceDefinition
 metadata:
   creationTimestamp: null
   name: scheduledsparkapplications.sparkoperator.k8s.io
+annotations:
+{{- if .Values.cleanupCrdsBeforeInstall }}
+  "helm.sh/hook-delete-policy": "before-hook-creation"
+{{- end }}
 spec:
   group: sparkoperator.k8s.io
   names:

incubator/sparkoperator/values.yaml

@@ -17,6 +17,7 @@ serviceAccounts:
 
 sparkJobNamespace: ""
 installCrds: true
+cleanupCrdsBeforeInstall: false
 controllerThreads: 10
 resyncInterval: 30
 ingressUrlFormat: ""