From a7abb2139f38b0b0e4fabf62936e01322bebf0b5 Mon Sep 17 00:00:00 2001 From: Shaun Mansell Date: Tue, 8 Oct 2019 19:43:11 +1100 Subject: [PATCH] [stable/prometheus-operator] Add support for creating a Service and Ingress for each Prometheus replica (#17382) * Add support for creating a Service and Ingress for each Prometheus replica Adding support for creating separate Service and Ingress resources that will map to the standard web port of each Prometheus server replica. We're using the `prometheus.prometheusSpec.replicas` value to determine how many of each resource type to be created, and to vary resource names and values. Because we're using a single set of values to control certain properties, we have to drop support for setting parameters that need to be unique within a each resource. Primarily this means no IP configuration options for the Service resource. The ingress resources are created with a single host entry that is compiled by taking two values (`hostPrefix` and `hostDomain`). The final hostname format is `hostPrefix`-`replicaIndex`.`hostDomain`, where `replicaIndex` will be 0, 1, 2 etc. Signed-off-by: Shaun Mansell * Fix trailing spaces in values and test-values files Signed-off-by: Shaun Mansell * Bump prometheus-operator chart version Signed-off-by: Shaun Mansell * Convert service and ingress per-replica resources to be managed as a list Updating both the service and ingress per-replica resources from a templated list to list api-resource kinds. Signed-off-by: Shaun Mansell --- stable/prometheus-operator/Chart.yaml | 2 +- stable/prometheus-operator/README.md | 14 ++++++ .../prometheus-operator/ci/test-values.yaml | 50 +++++++++++++++++-- .../prometheus/ingressperreplica.yaml | 43 ++++++++++++++++ .../prometheus/serviceperreplica.yaml | 44 ++++++++++++++++ stable/prometheus-operator/values.yaml | 47 +++++++++++++++++ 6 files changed, 195 insertions(+), 5 deletions(-) create mode 100644 stable/prometheus-operator/templates/prometheus/ingressperreplica.yaml create mode 100644 stable/prometheus-operator/templates/prometheus/serviceperreplica.yaml diff --git a/stable/prometheus-operator/Chart.yaml b/stable/prometheus-operator/Chart.yaml index c2c5bcf2c0..c1d77e2905 100644 --- a/stable/prometheus-operator/Chart.yaml +++ b/stable/prometheus-operator/Chart.yaml @@ -10,7 +10,7 @@ name: prometheus-operator sources: - https://github.com/coreos/prometheus-operator - https://coreos.com/operators/prometheus -version: 6.13.0 +version: 6.14.0 appVersion: 0.32.0 home: https://github.com/coreos/prometheus-operator keywords: diff --git a/stable/prometheus-operator/README.md b/stable/prometheus-operator/README.md index 2abe3d57d4..0428fbf4f2 100644 --- a/stable/prometheus-operator/README.md +++ b/stable/prometheus-operator/README.md @@ -223,6 +223,13 @@ The following tables list the configurable parameters of the prometheus-operator | `prometheus.ingress.labels` | Prometheus Ingress additional labels | `{}` | | `prometheus.ingress.paths` | Prometheus Ingress paths | `[]` | | `prometheus.ingress.tls` | Prometheus Ingress TLS configuration (YAML) | `[]` | +| `prometheus.ingressPerReplica.annotations` | Prometheus pre replica Ingress annotations | `{}` | +| `prometheus.ingressPerReplica.enabled` | If true, create an Ingress for each Prometheus server replica in the StatefulSet | `false` | +| `prometheus.ingressPerReplica.hostPrefix` | | `""` | +| `prometheus.ingressPerReplica.hostDomain` | | `""` | +| `prometheus.ingressPerReplica.labels` | Prometheus per replica Ingress additional labels | `{}` | +| `prometheus.ingressPerReplica.paths` | Prometheus per replica Ingress paths | `[]` | +| `prometheus.ingressPerReplica.tlsSecretName` | Secret name containing the TLS certificate for Prometheus per replica ingress | `[]` | | `prometheus.podDisruptionBudget.enabled` | If true, create a pod disruption budget for prometheus pods. The created resource cannot be modified once created - it must be deleted to perform a change | `false` | | `prometheus.podDisruptionBudget.maxUnavailable` | Maximum number / percentage of pods that may be made unavailable | `""` | | `prometheus.podDisruptionBudget.minAvailable` | Minimum number / percentage of pods that should remain scheduled | `1` | @@ -296,6 +303,13 @@ The following tables list the configurable parameters of the prometheus-operator | `prometheus.serviceMonitor.metricRelabelings` | The `metric_relabel_configs` for scraping the prometheus instance. | `` | | `prometheus.serviceMonitor.relabelings` | The `relabel_configs` for scraping the prometheus instance. | `` | | `prometheus.serviceMonitor.selfMonitor` | Create a `serviceMonitor` to automatically monitor the prometheus instance | `true` | +| `prometheus.servicePerReplica.annotations` | Prometheus per replica Service Annotations | `{}` | +| `prometheus.servicePerReplica.enabled` | If true, create a Service for each Prometheus server replica in the StatefulSet | `false` | +| `prometheus.servicePerReplica.labels` | Prometheus per replica Service Labels | `{}` | +| `prometheus.servicePerReplica.loadBalancerSourceRanges` | Prometheus per replica Service Loadbalancer Source Ranges | `[]` | +| `prometheus.servicePerReplica.nodePort` | Prometheus per replica service port for NodePort Service type | `30091` | +| `prometheus.servicePerReplica.targetPort` | Prometheus per replica Service internal port | `9090` | +| `prometheus.servicePerReplica.type` | Prometheus per replica Service type | `ClusterIP` | ### Alertmanager | Parameter | Description | Default | diff --git a/stable/prometheus-operator/ci/test-values.yaml b/stable/prometheus-operator/ci/test-values.yaml index 50be0387a3..3304cc84f3 100644 --- a/stable/prometheus-operator/ci/test-values.yaml +++ b/stable/prometheus-operator/ci/test-values.yaml @@ -1187,6 +1187,27 @@ prometheus: sessionAffinity: "" + ## Configuration for creating a separate Service for each statefulset Prometheus replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## To be used with a proxy extraContainer port + targetPort: 9090 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30091 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "loadbalancer" + loadBalancerSourceRanges: [] + ## Service type + ## + type: ClusterIP + ## Configure pod disruption budgets for Prometheus ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget ## This configuration is immutable once created and will require the PDB to be deleted to be changed @@ -1222,10 +1243,31 @@ prometheus: # hosts: # - prometheus.example.com - ## Configure additional options for default pod security policy for Prometheus - ## ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ - podSecurityPolicy: - allowedCapabilities: [] + ## Configuration for creating an Ingress that will map to each Prometheus replica service + ## prometheus.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## Secret name containing the TLS certificate for Prometheus per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" serviceMonitor: ## Scrape interval. If not set, the Prometheus default scrape interval is used. diff --git a/stable/prometheus-operator/templates/prometheus/ingressperreplica.yaml b/stable/prometheus-operator/templates/prometheus/ingressperreplica.yaml new file mode 100644 index 0000000000..8b1caf1c25 --- /dev/null +++ b/stable/prometheus-operator/templates/prometheus/ingressperreplica.yaml @@ -0,0 +1,43 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled .Values.prometheus.ingressPerReplica.enabled }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $servicePort := 9090 -}} +{{- $ingressValues := .Values.prometheus.ingressPerReplica -}} +apiVersion: extensions/v1beta1 +kind: IngressList +metadata: + name: {{ include "prometheus-operator.fullname" $ }}-prometheus-ingressperreplica +items: +{{ range $i, $e := until $count }} + - apiVersion: extensions/v1beta1 + kind: Ingress + metadata: + name: {{ include "prometheus-operator.fullname" $ }}-prometheus-{{ $i }} + labels: + app: {{ include "prometheus-operator.name" $ }}-prometheus +{{ include "prometheus-operator.labels" $ | indent 8 }} + {{- if $ingressValues.labels }} + {{ toYaml $ingressValues.labels | indent 8 }} + {{- end }} + {{- if $ingressValues.annotations }} + annotations: +{{ toYaml $ingressValues.annotations | indent 8 }} + {{- end }} + spec: + rules: + - host: {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + http: + paths: + {{- range $p := $ingressValues.paths }} + - path: {{ tpl $p $ }} + backend: + serviceName: {{ include "prometheus-operator.fullname" $ }}-prometheus-{{ $i }} + servicePort: {{ $servicePort }} + {{- end -}} + {{- if $ingressValues.tlsSecretName }} + tls: + - hosts: + - {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + secretName: {{ $ingressValues.tlsSecretName }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/stable/prometheus-operator/templates/prometheus/serviceperreplica.yaml b/stable/prometheus-operator/templates/prometheus/serviceperreplica.yaml new file mode 100644 index 0000000000..8058bb6132 --- /dev/null +++ b/stable/prometheus-operator/templates/prometheus/serviceperreplica.yaml @@ -0,0 +1,44 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $serviceValues := .Values.prometheus.servicePerReplica -}} +apiVersion: v1 +kind: ServiceList +metadata: + name: {{ include "prometheus-operator.fullname" $ }}-prometheus-serviceperreplica +items: +{{ range $i, $e := until $count }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "prometheus-operator.fullname" $ }}-prometheus-{{ $i }} + labels: + app: {{ include "prometheus-operator.name" $ }}-prometheus +{{ include "prometheus-operator.labels" $ | indent 8 }} + {{- if $serviceValues.annotations }} + annotations: +{{ toYaml $serviceValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $serviceValues.clusterIP }} + clusterIP: {{ $serviceValues.clusterIP }} + {{- end }} + {{- if $serviceValues.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := $serviceValues.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} + {{- end }} + ports: + - name: web + {{- if eq $serviceValues.type "NodePort" }} + nodePort: {{ $serviceValues.nodePort }} + {{- end }} + port: 9090 + targetPort: {{ $serviceValues.targetPort }} + selector: + app: prometheus + prometheus: {{ include "prometheus-operator.fullname" $ }}-prometheus + statefulset.kubernetes.io/pod-name: prometheus-{{ include "prometheus-operator.fullname" $ }}-prometheus-{{ $i }} + type: "{{ $serviceValues.type }}" +{{- end }} +{{- end }} diff --git a/stable/prometheus-operator/values.yaml b/stable/prometheus-operator/values.yaml index 0f450727b5..c4c7825558 100644 --- a/stable/prometheus-operator/values.yaml +++ b/stable/prometheus-operator/values.yaml @@ -1187,6 +1187,27 @@ prometheus: sessionAffinity: "" + ## Configuration for creating a separate Service for each statefulset Prometheus replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## To be used with a proxy extraContainer port + targetPort: 9090 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30091 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "loadbalancer" + loadBalancerSourceRanges: [] + ## Service type + ## + type: ClusterIP + ## Configure pod disruption budgets for Prometheus ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget ## This configuration is immutable once created and will require the PDB to be deleted to be changed @@ -1222,6 +1243,32 @@ prometheus: # hosts: # - prometheus.example.com + ## Configuration for creating an Ingress that will map to each Prometheus replica service + ## prometheus.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## Secret name containing the TLS certificate for Prometheus per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" + ## Configure additional options for default pod security policy for Prometheus ## ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ podSecurityPolicy: