From d3eb35acd67b3465ff585d94236537d09f30407b Mon Sep 17 00:00:00 2001 From: Chris Schaefer Date: Mon, 18 Mar 2019 05:27:23 -0400 Subject: [PATCH] [stable/spring-cloud-data-flow] Updates for Spring Cloud Data Flow 2.0 (#12285) - Single server consolidations - Update Skipper to 2.0 - Remove metrics - Add ability to use Kafka or RabbitMQ - Remove deprecated functionality Signed-off-by: Chris Schaefer --- stable/spring-cloud-data-flow/Chart.yaml | 6 +- stable/spring-cloud-data-flow/README.md | 52 +++++++-------- .../spring-cloud-data-flow/requirements.lock | 10 +-- .../spring-cloud-data-flow/requirements.yaml | 8 ++- .../templates/NOTES.txt | 5 ++ .../templates/metrics-deployment-rabbit.yaml | 66 ------------------- .../templates/metrics-service.yaml | 20 ------ .../templates/secrets.yaml | 21 ------ .../templates/server-config-rabbit.yaml | 40 ----------- .../templates/server-config.yaml | 30 +++++++++ .../templates/server-deployment.yaml | 53 ++------------- .../templates/server-rbac.yaml | 4 +- .../templates/skipper-config.yaml | 27 +++++++- .../templates/skipper-deployment.yaml | 47 ++++++++----- stable/spring-cloud-data-flow/values.yaml | 40 ++++------- 15 files changed, 152 insertions(+), 277 deletions(-) delete mode 100644 stable/spring-cloud-data-flow/templates/metrics-deployment-rabbit.yaml delete mode 100644 stable/spring-cloud-data-flow/templates/metrics-service.yaml delete mode 100644 stable/spring-cloud-data-flow/templates/secrets.yaml delete mode 100644 stable/spring-cloud-data-flow/templates/server-config-rabbit.yaml create mode 100644 stable/spring-cloud-data-flow/templates/server-config.yaml diff --git a/stable/spring-cloud-data-flow/Chart.yaml b/stable/spring-cloud-data-flow/Chart.yaml index 75df32d790..7e200110ca 100644 --- a/stable/spring-cloud-data-flow/Chart.yaml +++ b/stable/spring-cloud-data-flow/Chart.yaml @@ -1,14 +1,12 @@ apiVersion: v1 description: Toolkit for building data processing pipelines. name: spring-cloud-data-flow -version: 1.0.3 -appVersion: 1.7.3.RELEASE +version: 2.0.0 +appVersion: 2.0.0.RELEASE home: http://cloud.spring.io/spring-cloud-dataflow/ sources: - https://github.com/spring-cloud/spring-cloud-dataflow -- https://github.com/spring-cloud/spring-cloud-dataflow-server-kubernetes - https://github.com/spring-cloud/spring-cloud-skipper -- https://github.com/spring-cloud/spring-cloud-dataflow-metrics-collector icon: https://raw.githubusercontent.com/spring-cloud/spring-cloud-dataflow/master/spring-cloud-dataflow-docs/src/main/docbook/images/logo.png maintainers: - name: trisberg diff --git a/stable/spring-cloud-data-flow/README.md b/stable/spring-cloud-data-flow/README.md index c2fe50c65b..f862bf0510 100644 --- a/stable/spring-cloud-data-flow/README.md +++ b/stable/spring-cloud-data-flow/README.md @@ -8,7 +8,7 @@ Pipelines consist of [Spring Boot](http://projects.spring.io/spring-boot/) apps, This chart will provision a fully functional and fully featured Spring Cloud Data Flow installation that can deploy and manage data processing pipelines in the cluster that it is deployed to. -MySQL and Redis are used as the stores for Spring Cloud Data Flow state and RabbitMQ is used for the pipelines' messaging layer. +MySQL is used as the data store for Spring Cloud Data Flow state and either RabbitMQ or Kafka is used as the messaging layer for streaming apps to communicate with one another. For more information on Spring Cloud Data Flow and its capabilities, see it's [documentation](http://docs.spring.io/spring-cloud-dataflow/docs/current/reference/htmlsingle/). @@ -30,6 +30,12 @@ If you are using a cluster that does not have a load balancer (like Minikube) th $ helm install --name my-release --set server.service.type=NodePort stable/spring-cloud-data-flow ``` +RabbitMQ is the default messaging layer, Kafka can be enabled instead by using the following `set` flags to the `helm` command, for example: + +`--set kafka.enabled=true,rabbitmq.enabled=false` + +Only one messaging layer can be used at a given time. If RabbitMQ and Kafka are enabled, both charts will be installed with RabbitMQ being used in the deployment. + Note that this chart pulls in many different Docker images so can take a while to fully install. ## Configuration @@ -49,44 +55,26 @@ The following tables list the configurable parameters and their default values. | serviceAccount.create | Create ServiceAccount | true | serviceAccount.name | ServiceAccount name | (generated if not specified) -### Data Flow User Accounts - -| Parameter | Description | Default | -| ----------------------- | -------------------------------------- | ------------------------- | -| dataflowUsername | The username for the primary user | user -| dataflowPassword | The password for the primary user | password -| dataflowRoles | The roles assigned to the primary user | ROLE_VIEW, ROLE_CREATE -| dataflowAdminUsername | The username for the admin user | admin -| dataflowAdminPassword | The password for the admin user | admin -| dataflowAdminRoles | The roles assigned to the admin user | ROLE_MANAGE, ROLE_VIEW - ### Data Flow Server Configuration | Parameter | Description | Default | | --------------------------------- | -------------------------------------------------- | ---------------- | -| server.version | The version/tag of the Data Flow server | 1.7.3.RELEASE +| server.version | The version/tag of the Data Flow server | 2.0.1.RELEASE | server.imagePullPolicy | The imagePullPolicy of the Data Flow server | IfNotPresent | server.service.type | The service type for the Data Flow server | LoadBalancer +| server.platformName | The name of the configured platform account | default | server.service.externalPort | The external port for the Data Flow server | 80 ### Skipper Server Configuration | Parameter | Description | Default | | ---------------------------------- | ------------------------------------------------- | ---------------- | -| skipper.version | The version/tag of the Skipper server | 1.1.4.RELEASE +| skipper.version | The version/tag of the Skipper server | 2.0.0.RELEASE | skipper.imagePullPolicy | The imagePullPolicy of the Skipper server | IfNotPresent -| skipper.platformName | The name of the configured platform account | minikube +| skipper.platformName | The name of the configured platform account | default | skipper.service.type | The service type for the Skipper server | ClusterIP -### Metrics Server Configuration - -| Parameter | Description | Default | -| ---------------------------------- | ------------------------------------------------- | ---------------- | -| metrics.version | The version/tag of the Metrics server | 2.0.0.RELEASE -| metrics.imagePullPolicy | The imagePullPolicy of the Metrics server | IfNotPresent -| metrics.service.type | The service type for the Metrics server | ClusterIP - -### Spring Cloud Deployer Configuration +### Spring Cloud Deployer for Kubernetes Configuration | Parameter | Description | Default | | ------------------------------------------- | -------------------------------------- | ------------------------- | @@ -97,9 +85,19 @@ The following tables list the configurable parameters and their default values. ### RabbitMQ Configuration -| Parameter | Description | Default | -| -------------------------- | --------------------- | ------------------------- | -| rabbitmq.rabbitmqUsername | RabbitMQ user name | user +| Parameter | Description | Default | +| -------------------------- | ---------------------------------------- | ------------------------- | +| rabbitmq.enabled | Enable RabbitMQ as the middleware to use | true +| rabbitmq.rabbitmqUsername | RabbitMQ user name | user + +### Kafka Configuration + +| Parameter | Description | Default | +| ---------------------------- | ----------------------------------------- | ------------------------------------------- | +| kafka.enabled | Enable RabbitMQ as the middleware to use | false +| kafka.replicas | The number of Kafka replicas to use | 1 +| kafka.configurationOverrides | Kafka deployment configuration overrides | replication.factor=1, metrics.enabled=false +| kafka.zookeeper.replicaCount | The number of ZooKeeper replicates to use | 1 ### MySQL Configuration diff --git a/stable/spring-cloud-data-flow/requirements.lock b/stable/spring-cloud-data-flow/requirements.lock index 3c0091040d..75f7013912 100644 --- a/stable/spring-cloud-data-flow/requirements.lock +++ b/stable/spring-cloud-data-flow/requirements.lock @@ -2,11 +2,11 @@ dependencies: - name: mysql repository: https://kubernetes-charts.storage.googleapis.com/ version: 0.3.4 -- name: redis - repository: https://kubernetes-charts.storage.googleapis.com/ - version: 1.1.11 - name: rabbitmq repository: https://kubernetes-charts.storage.googleapis.com/ version: 0.6.16 -digest: sha256:b16ffdc5da316b3a7e677d3a83670341b13670d7e59f8ce74d023a01df05f135 -generated: 2018-01-31T09:21:26.470351-05:00 +- name: kafka + repository: https://kubernetes-charts-incubator.storage.googleapis.com/ + version: 0.13.11 +digest: sha256:f637f0a710a29e8c4d052faf43224cf5818cdcfcd6130ea0c931d8a6311ca42e +generated: 2019-03-08T14:41:35.841588-05:00 diff --git a/stable/spring-cloud-data-flow/requirements.yaml b/stable/spring-cloud-data-flow/requirements.yaml index f5ec71f953..1af04efadb 100644 --- a/stable/spring-cloud-data-flow/requirements.yaml +++ b/stable/spring-cloud-data-flow/requirements.yaml @@ -2,9 +2,11 @@ dependencies: - name: mysql version: 0.3.4 repository: https://kubernetes-charts.storage.googleapis.com/ -- name: redis - version: 1.1.11 - repository: https://kubernetes-charts.storage.googleapis.com/ - name: rabbitmq version: 0.6.16 repository: https://kubernetes-charts.storage.googleapis.com/ + condition: rabbitmq.enabled +- name: kafka + version: 0.13.11 + repository: https://kubernetes-charts-incubator.storage.googleapis.com/ + condition: kafka.enabled diff --git a/stable/spring-cloud-data-flow/templates/NOTES.txt b/stable/spring-cloud-data-flow/templates/NOTES.txt index f462838939..ec347ae019 100644 --- a/stable/spring-cloud-data-flow/templates/NOTES.txt +++ b/stable/spring-cloud-data-flow/templates/NOTES.txt @@ -13,3 +13,8 @@ echo "Visit http://127.0.0.1:8080 to use your application" kubectl port-forward $POD_NAME 8080:{{ .Values.server.service.externalPort }} {{- end }} +{{ if and .Values.rabbitmq.enabled .Values.kafka.enabled -}} +WARNING: both RabbitMQ and Kafka are enabled and will be installed as +part of this chart, but only RabbitMQ is being used. Only one middleware +should be enabled at a given time. See the README.md for more details. +{{ end -}} diff --git a/stable/spring-cloud-data-flow/templates/metrics-deployment-rabbit.yaml b/stable/spring-cloud-data-flow/templates/metrics-deployment-rabbit.yaml deleted file mode 100644 index 57c7aad046..0000000000 --- a/stable/spring-cloud-data-flow/templates/metrics-deployment-rabbit.yaml +++ /dev/null @@ -1,66 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "scdf.fullname" . }}-metrics - labels: - app: {{ template "scdf.name" . }} - component: metrics - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -spec: - selector: - matchLabels: - app: {{ template "scdf.name" . }} - component: metrics - release: "{{ .Release.Name }}" - replicas: 1 - template: - metadata: - labels: - app: {{ template "scdf.name" . }} - component: metrics - release: "{{ .Release.Name }}" - spec: - containers: - - name: {{ template "scdf.fullname" . }}-metrics - image: {{ .Values.metrics.image }}:{{ .Values.metrics.version }} - imagePullPolicy: {{ .Values.metrics.imagePullPolicy }} - ports: - - containerPort: 8080 - name: http - resources: -{{ toYaml .Values.metrics.resources | indent 10 }} - livenessProbe: - httpGet: - path: /actuator/health - port: http - initialDelaySeconds: 90 - periodSeconds: 15 - timeoutSeconds: 5 - readinessProbe: - httpGet: - path: /actuator/health - port: http - initialDelaySeconds: 60 - periodSeconds: 15 - timeoutSeconds: 5 - env: - - name: SERVER_PORT - value: '8080' - - name: SPRING_CLOUD_CONFIG_ENABLED - value: 'false' - - name: SPRING_AUTOCONFIGURE_EXCLUDE - value: 'org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration' - - name: SPRING_RABBITMQ_HOST - value: '${{ printf "{" }}{{ template "scdf.envrelease" . }}_RABBITMQ_SERVICE_HOST}' - - name: SPRING_RABBITMQ_PORT - value: '${{ printf "{" }}{{ template "scdf.envrelease" . }}_RABBITMQ_SERVICE_PORT_AMQP}' - - name: SPRING_RABBITMQ_USERNAME - value: '{{ .Values.rabbitmq.rabbitmqUsername }}' - - name: SPRING_RABBITMQ_PASSWORD - valueFrom: - secretKeyRef: - name: {{ printf "%s-%s" .Release.Name "rabbitmq" | trunc 63 | trimSuffix "-" }} - key: rabbitmq-password - serviceAccountName: {{ template "scdf.serviceAccountName" . }} diff --git a/stable/spring-cloud-data-flow/templates/metrics-service.yaml b/stable/spring-cloud-data-flow/templates/metrics-service.yaml deleted file mode 100644 index 2a348c1d11..0000000000 --- a/stable/spring-cloud-data-flow/templates/metrics-service.yaml +++ /dev/null @@ -1,20 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: {{ template "scdf.fullname" . }}-metrics - labels: - app: {{ template "scdf.name" . }} - component: metrics - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -spec: - type: {{ .Values.metrics.service.type }} - ports: - - port: 80 - targetPort: http - name: http - selector: - app: {{ template "scdf.name" . }} - component: metrics - release: {{ .Release.Name }} diff --git a/stable/spring-cloud-data-flow/templates/secrets.yaml b/stable/spring-cloud-data-flow/templates/secrets.yaml deleted file mode 100644 index 97eb9d6c4b..0000000000 --- a/stable/spring-cloud-data-flow/templates/secrets.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: {{ template "scdf.fullname" . }} - labels: - app: {{ template "scdf.name" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -type: Opaque -data: - {{- if .Values.dataflowAdminPassword }} - data-flow-admin-password: {{ .Values.dataflowAdminPassword | b64enc | quote }} - {{- else }} - data-flow-admin-password: {{ randAlphaNum 10 | b64enc | quote }} - {{- end }} - {{- if .Values.dataflowPassword }} - data-flow-password: {{ .Values.dataflowPassword | b64enc | quote }} - {{- else }} - data-flow-password: {{ randAlphaNum 10 | b64enc | quote }} - {{- end }} diff --git a/stable/spring-cloud-data-flow/templates/server-config-rabbit.yaml b/stable/spring-cloud-data-flow/templates/server-config-rabbit.yaml deleted file mode 100644 index 5ba444beaf..0000000000 --- a/stable/spring-cloud-data-flow/templates/server-config-rabbit.yaml +++ /dev/null @@ -1,40 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ template "scdf.fullname" . }}-server - labels: - app: {{ template "scdf.name" . }} - component: server - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -data: - application.yaml: |- - security: - basic: - enabled: true - realm: Spring Cloud Data Flow - spring: - cloud: - dataflow: - security: - authentication: - file: - enabled: true - users: - {{ .Values.dataflowAdminUsername }}: ${data-flow-admin-password}, {{ .Values.dataflowAdminRoles }} - {{ .Values.dataflowUsername }}: ${data-flow-password}, {{ .Values.dataflowRoles }} - deployer: - kubernetes: - environmentVariables: 'SPRING_RABBITMQ_HOST=${{ printf "{" }}{{ template "scdf.envrelease" . }}_RABBITMQ_SERVICE_HOST},SPRING_RABBITMQ_PORT=${{ printf "{" }}{{ template "scdf.envrelease" . }}_RABBITMQ_SERVICE_PORT_AMQP},SPRING_RABBITMQ_USERNAME={{ .Values.rabbitmq.rabbitmqUsername }},SPRING_RABBITMQ_PASSWORD=${rabbitmq-password},SPRING_REDIS_HOST=${{ printf "{" }}{{ template "scdf.envrelease" . }}_REDIS_SERVICE_HOST},SPRING_REDIS_PORT=${{ printf "{" }}{{ template "scdf.envrelease" . }}_REDIS_SERVICE_PORT},SPRING_REDIS_PASSWORD=${redis-password}' - datasource: - url: 'jdbc:mysql://${{ printf "{" }}{{ template "scdf.envrelease" . }}_MYSQL_SERVICE_HOST}:3306/dataflow' - driverClassName: org.mariadb.jdbc.Driver - username: root - password: ${mysql-root-password} - testOnBorrow: true - validationQuery: "SELECT 1" - redis: - host: ${{ printf "{" }}{{ template "scdf.envrelease" . }}_REDIS_SERVICE_HOST} - port: ${{ printf "{" }}{{ template "scdf.envrelease" . }}_REDIS_SERVICE_PORT} - password: ${redis-password} diff --git a/stable/spring-cloud-data-flow/templates/server-config.yaml b/stable/spring-cloud-data-flow/templates/server-config.yaml new file mode 100644 index 0000000000..aa79a6dea2 --- /dev/null +++ b/stable/spring-cloud-data-flow/templates/server-config.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "scdf.fullname" . }}-server + labels: + app: {{ template "scdf.name" . }} + component: server + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +data: + application.yaml: |- + spring: + cloud: + dataflow: + task: + platform: + kubernetes: + accounts: + {{ .Values.server.platformName }}: + limits: + memory: {{ .Values.deployer.resourceLimits.memory }} + cpu: {{ .Values.deployer.resourceLimits.cpu }} + datasource: + url: 'jdbc:mysql://${{ printf "{" }}{{ template "scdf.envrelease" . }}_MYSQL_SERVICE_HOST}:${{ printf "{" }}{{ template "scdf.envrelease" . }}_MYSQL_SERVICE_PORT}/dataflow' + driverClassName: org.mariadb.jdbc.Driver + username: root + password: ${mysql-root-password} + testOnBorrow: true + validationQuery: "SELECT 1" diff --git a/stable/spring-cloud-data-flow/templates/server-deployment.yaml b/stable/spring-cloud-data-flow/templates/server-deployment.yaml index 85a6795630..f34ab54ef1 100644 --- a/stable/spring-cloud-data-flow/templates/server-deployment.yaml +++ b/stable/spring-cloud-data-flow/templates/server-deployment.yaml @@ -27,18 +27,9 @@ spec: image: {{ .Values.server.image }}:{{ .Values.server.version }} imagePullPolicy: {{ .Values.server.imagePullPolicy }} volumeMounts: - - name: data-flow - mountPath: /etc/secrets/data-flow - readOnly: true - - name: rabbitmq - mountPath: /etc/secrets/rabbitmq - readOnly: true - name: mysql mountPath: /etc/secrets/mysql readOnly: true - - name: redis - mountPath: /etc/secrets/redis - readOnly: true ports: - containerPort: 8080 name: http @@ -48,9 +39,6 @@ spec: httpGet: path: /management/health port: http - httpHeaders: - - name: Authorization - value: 'Basic {{ printf "%s:%s" .Values.dataflowAdminUsername .Values.dataflowAdminPassword | b64enc }}' initialDelaySeconds: 120 periodSeconds: 15 timeoutSeconds: 5 @@ -58,9 +46,6 @@ spec: httpGet: path: /management/health port: http - httpHeaders: - - name: Authorization - value: 'Basic {{ printf "%s:%s" .Values.dataflowAdminUsername .Values.dataflowAdminPassword | b64enc }}' initialDelaySeconds: 60 periodSeconds: 15 timeoutSeconds: 5 @@ -73,48 +58,24 @@ spec: value: '8080' - name: SPRING_CLOUD_CONFIG_ENABLED value: 'false' - - name: SPRING_CLOUD_DATAFLOW_FEATURES_ANALYTICS_ENABLED - value: 'true' - - name: SPRING_CLOUD_DATAFLOW_FEATURES_SCHEDULES_ENABLED - value: 'true' - - name: SPRING_CLOUD_DEPLOYER_KUBERNETES_CREATE_DEPLOYMENT - value: 'true' - - name: SPRING_CLOUD_DEPLOYER_KUBERNETES_CPU - value: '{{ .Values.deployer.resourceLimits.cpu }}' - - name: SPRING_CLOUD_DEPLOYER_KUBERNETES_MEMORY - value: '{{ .Values.deployer.resourceLimits.memory }}' - - name: SPRING_CLOUD_DEPLOYER_KUBERNETES_READINESS_PROBE_DELAY - value: '{{ .Values.deployer.readinessProbe.initialDelaySeconds }}' - - name: SPRING_CLOUD_DEPLOYER_KUBERNETES_LIVENESS_PROBE_DELAY - value: '{{ .Values.deployer.livenessProbe.initialDelaySeconds }}' - - name: SPRING_CLOUD_KUBERNETES_CONFIG_NAME - value: {{ template "scdf.fullname" . }}-server - name: SPRING_CLOUD_KUBERNETES_SECRETS_ENABLE_API value: 'true' - name: SPRING_CLOUD_KUBERNETES_SECRETS_PATHS value: /etc/secrets - - name: SPRING_CLOUD_DATAFLOW_FEATURES_SKIPPER_ENABLED - value: 'true' + - name: SPRING_CLOUD_KUBERNETES_CONFIG_NAME + value: {{ template "scdf.fullname" . }}-server - name: SPRING_CLOUD_SKIPPER_CLIENT_SERVER_URI value: 'http://${{ printf "{" }}{{ template "scdf.envname" . }}_SKIPPER_SERVICE_HOST}/api' - - name: SPRING_CLOUD_DATAFLOW_METRICS_COLLECTOR_URI - value: 'http://${{ printf "{" }}{{ template "scdf.envname" . }}_METRICS_SERVICE_HOST}' - name: SPRING_CLOUD_DATAFLOW_SERVER_URI value: 'http://${{ printf "{" }}{{ template "scdf.envname" . }}_SERVER_SERVICE_HOST}:${{ printf "{" }}{{ template "scdf.envname" . }}_SERVER_SERVICE_PORT}' - # Maven repo for metadata artifact resolution plus metrics destination for all stream apps - name: SPRING_APPLICATION_JSON - value: "{ \"maven\": { \"local-repository\": null, \"remote-repositories\": { \"repo1\": { \"url\": \"https://repo.spring.io/libs-snapshot\"} } }, \"spring.cloud.dataflow.application-properties.stream.spring.cloud.stream.bindings.applicationMetrics.destination\": \"metrics\" }" + value: "{ \"maven\": { \"local-repository\": null, \"remote-repositories\": { \"repo1\": { \"url\": \"https://repo.spring.io/libs-snapshot\"} } } }" volumes: - - name: data-flow - secret: - secretName: {{ template "scdf.fullname" . }} - - name: rabbitmq - secret: - secretName: {{ printf "%s-%s" .Release.Name "rabbitmq" | trunc 63 | trimSuffix "-" }} - name: mysql secret: secretName: {{ printf "%s-%s" .Release.Name "mysql" | trunc 63 | trimSuffix "-" }} - - name: redis - secret: - secretName: {{ printf "%s-%s" .Release.Name "redis" | trunc 63 | trimSuffix "-" }} + initContainers: + - name: init-mysql-wait + image: busybox + command: ['sh', '-c', 'until nc -w3 -z {{ printf "%s-%s" .Release.Name "mysql" | trunc 63 | trimSuffix "-" }} 3306; do echo waiting for mysql; sleep 3; done;'] serviceAccountName: {{ template "scdf.serviceAccountName" . }} diff --git a/stable/spring-cloud-data-flow/templates/server-rbac.yaml b/stable/spring-cloud-data-flow/templates/server-rbac.yaml index 1aad05de29..23471cb060 100644 --- a/stable/spring-cloud-data-flow/templates/server-rbac.yaml +++ b/stable/spring-cloud-data-flow/templates/server-rbac.yaml @@ -22,8 +22,8 @@ rules: resources: ["deployments", "replicasets", "jobs"] verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - apiGroups: ["batch"] - resources: ["cronjobs"] - verbs: ["create", "delete", "get", "list", "watch"] + resources: ["cronjobs", "jobs"] + verbs: ["create", "delete", "get", "list", "watch", "update", "patch"] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/stable/spring-cloud-data-flow/templates/skipper-config.yaml b/stable/spring-cloud-data-flow/templates/skipper-config.yaml index af55ab22b5..82eea79083 100644 --- a/stable/spring-cloud-data-flow/templates/skipper-config.yaml +++ b/stable/spring-cloud-data-flow/templates/skipper-config.yaml @@ -9,4 +9,29 @@ metadata: release: "{{ .Release.Name }}" heritage: "{{ .Release.Service }}" data: - spring.application.json: '{"spring.cloud.skipper.server.enableLocalPlatform" : false, "spring.cloud.skipper.server.platform.kubernetes.accounts.{{ .Values.skipper.platformName }}.environmentVariables" : "SPRING_RABBITMQ_HOST=${{ printf "{" }}{{ template "scdf.envrelease" . }}_RABBITMQ_SERVICE_HOST},SPRING_RABBITMQ_PORT=${{ printf "{" }}{{ template "scdf.envrelease" . }}_RABBITMQ_SERVICE_PORT_AMQP},SPRING_RABBITMQ_USERNAME={{ .Values.rabbitmq.rabbitmqUsername }},SPRING_RABBITMQ_PASSWORD=${RABBITMQ_PASSWORD}","spring.cloud.skipper.server.platform.kubernetes.accounts.{{ .Values.skipper.platformName }}.memory" : "{{ .Values.deployer.resourceLimits.memory }}","spring.cloud.skipper.server.platform.kubernetes.accounts.{{ .Values.skipper.platformName }}.cpu" : "{{ .Values.deployer.resourceLimits.cpu }}","spring.cloud.skipper.server.platform.kubernetes.accounts.{{ .Values.skipper.platformName }}.createDeployment" : true, "spring.cloud.skipper.server.platform.kubernetes.accounts.{{ .Values.skipper.platformName }}.readinessProbeDelay" : "{{ .Values.deployer.readinessProbe.initialDelaySeconds }}", "spring.cloud.skipper.server.platform.kubernetes.accounts.{{ .Values.skipper.platformName }}.livenessProbeDelay" : "{{ .Values.deployer.livenessProbe.initialDelaySeconds }}"}' + application.yaml: |- + spring: + cloud: + skipper: + server: + platform: + kubernetes: + accounts: + {{ .Values.skipper.platformName }}: + {{- if .Values.rabbitmq.enabled}} + environmentVariables: 'SPRING_RABBITMQ_HOST=${{ printf "{" }}{{ template "scdf.envrelease" . }}_RABBITMQ_SERVICE_HOST},SPRING_RABBITMQ_PORT=${{ printf "{" }}{{ template "scdf.envrelease" . }}_RABBITMQ_SERVICE_PORT_AMQP},SPRING_RABBITMQ_USERNAME={{ .Values.rabbitmq.rabbitmqUsername }},SPRING_RABBITMQ_PASSWORD=${RABBITMQ_PASSWORD}' + {{- else if .Values.kafka.enabled}} + environmentVariables: 'SPRING_CLOUD_STREAM_KAFKA_BINDER_BROKERS=${{ printf "{" }}{{ template "scdf.envrelease" . }}_KAFKA_SERVICE_HOST}:${{ printf "{" }}{{ template "scdf.envrelease" . }}_KAFKA_SERVICE_PORT},SPRING_CLOUD_STREAM_KAFKA_BINDER_ZK_NODES=${{ printf "{" }}{{ template "scdf.envrelease" . }}_ZOOKEEPER_SERVICE_HOST}:${{ printf "{" }}{{ template "scdf.envrelease" . }}_ZOOKEEPER_SERVICE_PORT}' + {{- end}} + limits: + memory: {{ .Values.deployer.resourceLimits.memory }} + cpu: {{ .Values.deployer.resourceLimits.cpu }} + readinessProbeDelay: {{ .Values.deployer.readinessProbe.initialDelaySeconds }} + livenessProbeDelay: {{ .Values.deployer.livenessProbe.initialDelaySeconds }} + datasource: + url: 'jdbc:mysql://${{ printf "{" }}{{ template "scdf.envrelease" . }}_MYSQL_SERVICE_HOST}:${{ printf "{" }}{{ template "scdf.envrelease" . }}_MYSQL_SERVICE_PORT}/skipper' + driverClassName: org.mariadb.jdbc.Driver + username: root + password: ${mysql-root-password} + testOnBorrow: true + validationQuery: "SELECT 1" diff --git a/stable/spring-cloud-data-flow/templates/skipper-deployment.yaml b/stable/spring-cloud-data-flow/templates/skipper-deployment.yaml index d2cd50654d..b7e2b8ccc7 100644 --- a/stable/spring-cloud-data-flow/templates/skipper-deployment.yaml +++ b/stable/spring-cloud-data-flow/templates/skipper-deployment.yaml @@ -26,6 +26,15 @@ spec: - name: {{ template "scdf.fullname" . }}-skipper image: {{ .Values.skipper.image }}:{{ .Values.skipper.version }} imagePullPolicy: {{ .Values.skipper.imagePullPolicy }} + volumeMounts: + {{- if .Values.rabbitmq.enabled}} + - name: rabbitmq + mountPath: /etc/secrets/rabbitmq + readOnly: true + {{- end}} + - name: mysql + mountPath: /etc/secrets/mysql + readOnly: true ports: - containerPort: 7577 name: http @@ -46,28 +55,34 @@ spec: periodSeconds: 15 timeoutSeconds: 5 env: - - name: SPRING_DATASOURCE_URL - value: 'jdbc:mysql://${{ printf "{" }}{{ template "scdf.envrelease" . }}_MYSQL_SERVICE_HOST}:3306/skipper' - - name: SPRING_DATASOURCE_USERNAME - value: root - - name: SPRING_DATASOURCE_PASSWORD + - name: KUBERNETES_NAMESPACE valueFrom: - secretKeyRef: - name: {{ printf "%s-%s" .Release.Name "mysql" | trunc 63 | trimSuffix "-" }} - key: mysql-root-password - - name: SPRING_DATASOURCE_DRIVER_CLASS_NAME - value: org.mariadb.jdbc.Driver + fieldRef: + fieldPath: "metadata.namespace" + - name: SERVER_PORT + value: '7577' + - name: SPRING_CLOUD_KUBERNETES_SECRETS_ENABLE_API + value: 'true' + - name: SPRING_CLOUD_KUBERNETES_SECRETS_PATHS + value: /etc/secrets + - name: SPRING_CLOUD_KUBERNETES_CONFIG_NAME + value: {{ template "scdf.fullname" . }}-skipper + {{- if .Values.rabbitmq.enabled}} - name: RABBITMQ_PASSWORD valueFrom: secretKeyRef: name: {{ printf "%s-%s" .Release.Name "rabbitmq" | trunc 63 | trimSuffix "-" }} key: rabbitmq-password - - name: SPRING_APPLICATION_JSON - valueFrom: - configMapKeyRef: - name: {{ template "scdf.fullname" . }}-skipper - key: spring.application.json - value: + {{- end}} + volumes: + {{- if .Values.rabbitmq.enabled}} + - name: rabbitmq + secret: + secretName: {{ printf "%s-%s" .Release.Name "rabbitmq" | trunc 63 | trimSuffix "-" }} + {{- end}} + - name: mysql + secret: + secretName: {{ printf "%s-%s" .Release.Name "mysql" | trunc 63 | trimSuffix "-" }} initContainers: - name: init-mysql-wait image: busybox diff --git a/stable/spring-cloud-data-flow/values.yaml b/stable/spring-cloud-data-flow/values.yaml index 75184ec640..364a8a341f 100644 --- a/stable/spring-cloud-data-flow/values.yaml +++ b/stable/spring-cloud-data-flow/values.yaml @@ -1,13 +1,4 @@ # Default values for spring-cloud-data-flow. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. -dataflowUsername: user -dataflowPassword: password -dataflowRoles: ROLE_VIEW, ROLE_CREATE -dataflowAdminUsername: admin -dataflowAdminPassword: admin -dataflowAdminRoles: ROLE_MANAGE, ROLE_VIEW - rbac: # Specifies whether RBAC resources should be created create: true @@ -20,9 +11,10 @@ serviceAccount: name: server: - image: springcloud/spring-cloud-dataflow-server-kubernetes - version: 1.7.3.RELEASE + image: springcloud/spring-cloud-dataflow-server + version: 2.0.1.RELEASE imagePullPolicy: IfNotPresent + platformName: default service: type: LoadBalancer externalPort: 80 @@ -36,23 +28,9 @@ server: skipper: image: springcloud/spring-cloud-skipper-server - version: 1.1.4.RELEASE - imagePullPolicy: IfNotPresent - platformName: minikube - service: - type: ClusterIP - resources: {} - # limits: - # cpu: 1.0 - # memory: 1024Mi - # requests: - # cpu: 0.5 - # memory: 640Mi - -metrics: - image: springcloud/metrics-collector-rabbit version: 2.0.0.RELEASE imagePullPolicy: IfNotPresent + platformName: default service: type: ClusterIP resources: {} @@ -73,7 +51,17 @@ deployer: initialDelaySeconds: 90 rabbitmq: + enabled: true rabbitmqUsername: user +kafka: + enabled: false + replicas: 1 + configurationOverrides: + "offsets.topic.replication.factor": 1 + "confluent.support.metrics.enable": false + zookeeper: + replicaCount: 1 + mysql: mysqlDatabase: dataflow