diff --git a/Makefile b/Makefile index 85974a5..111343e 100644 --- a/Makefile +++ b/Makefile @@ -274,7 +274,7 @@ metallb: cert-manager: $(HELM) repo add jetstack https://charts.jetstack.io - $(HELM) upgrade --install cert-manager jetstack/cert-manager --namespace certmanager-system --create-namespace --set "installCRDs=true" + $(HELM) upgrade --install cert-manager jetstack/cert-manager --namespace certmanager-system --create-namespace --set "crds.enabled=true" --version v1.18.3 gateway-api: kubectl apply \ @@ -297,6 +297,28 @@ envoy-gateway: gateway-api helm ## Install Envoy Gateway for Gateway API tests. load: kind $(KIND) load docker-image --name kamaji ${CONTAINER_REPOSITORY}:${VERSION} +.PHONY: install +install: helm cert-manager metallb ## Install Kamaji chart from repository using current commit SHA as image tag. + $(MAKE) VERSION=$$(git rev-parse --short HEAD) build load + $(HELM) repo add clastix https://clastix.github.io/charts + $(HELM) repo update + $(HELM) upgrade --install kamaji clastix/kamaji \ + --namespace kamaji-system \ + --create-namespace \ + --set 'resources=null' \ + --set "image.repository=$(CONTAINER_REPOSITORY)" \ + --set 'image.pullPolicy=Never' \ + --set "image.tag=$(GIT_HEAD_COMMIT)" \ + --version 0.0.0+latest + +.PHONY: prometheus-install +prometheus-install: ## Install Prometheus Operator bundle and Kamaji observability manifests. + kubectl create namespace monitoring --dry-run=client -o yaml | kubectl apply -f - + kubectl -n monitoring create configmap grafana-dashboards --from-file=kamaji-dashboard.json=./config/observability/dashboard/grafana-dashboard-kamaji.json --dry-run=client -o yaml | kubectl apply -f - + curl -fsSL https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.89.0/bundle.yaml | sed 's/namespace: default/namespace: monitoring/g' | kubectl apply --server-side --force-conflicts --field-manager=make-prometheus-install -f - + kubectl -n monitoring rollout status deploy/prometheus-operator --timeout=3m + kubectl apply --server-side --force-conflicts --field-manager=make-prometheus-install -f ./config/observability/ + ##@ e2e .PHONY: env diff --git a/cmd/kubeconfig-generator/cmd.go b/cmd/kubeconfig-generator/cmd.go index 404d231..7654bf5 100644 --- a/cmd/kubeconfig-generator/cmd.go +++ b/cmd/kubeconfig-generator/cmd.go @@ -25,6 +25,7 @@ import ( "github.com/clastix/kamaji/controllers" "github.com/clastix/kamaji/internal" + "github.com/clastix/kamaji/internal/metrics" ) func NewCmd(scheme *runtime.Scheme) *cobra.Command { @@ -83,6 +84,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command { } triggerChan := make(chan event.GenericEvent) + metricsRecorder := metrics.DefaultRecorder() mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrlOpts) if err != nil { @@ -105,7 +107,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command { } } - certController := &controllers.CertificateLifecycle{Channel: triggerChan, Deadline: certificateExpirationDeadline} + certController := &controllers.CertificateLifecycle{Channel: triggerChan, Deadline: certificateExpirationDeadline, Metrics: metricsRecorder} certController.EnqueueFn = certController.EnqueueForKubeconfigGenerator if err = certController.SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CertificateLifecycle") diff --git a/cmd/manager/cmd.go b/cmd/manager/cmd.go index 09bc394..aa264ae 100644 --- a/cmd/manager/cmd.go +++ b/cmd/manager/cmd.go @@ -33,6 +33,7 @@ import ( "github.com/clastix/kamaji/controllers/soot" "github.com/clastix/kamaji/internal" "github.com/clastix/kamaji/internal/builders/controlplane" + "github.com/clastix/kamaji/internal/metrics" "github.com/clastix/kamaji/internal/utilities" "github.com/clastix/kamaji/internal/webhook" "github.com/clastix/kamaji/internal/webhook/handlers" @@ -135,8 +136,9 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command { } tcpChannel, certChannel := make(chan event.GenericEvent), make(chan event.GenericEvent) + metricsRecorder := metrics.DefaultRecorder() - if err = (&controllers.DataStore{Client: mgr.GetClient(), TenantControlPlaneTrigger: tcpChannel}).SetupWithManager(mgr); err != nil { + if err = (&controllers.DataStore{Client: mgr.GetClient(), Metrics: metricsRecorder, TenantControlPlaneTrigger: tcpChannel}).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "DataStore") return err @@ -152,6 +154,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command { reconciler := &controllers.TenantControlPlaneReconciler{ Client: mgr.GetClient(), APIReader: mgr.GetAPIReader(), + Metrics: metricsRecorder, Config: controllers.TenantControlPlaneReconcilerConfig{ DefaultDataStoreName: datastore, KineContainerImage: kineImage, @@ -198,7 +201,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command { } } - certController := &controllers.CertificateLifecycle{Channel: certChannel, Deadline: certificateExpirationDeadline} + certController := &controllers.CertificateLifecycle{Channel: certChannel, Deadline: certificateExpirationDeadline, Metrics: metricsRecorder} certController.EnqueueFn = certController.EnqueueForTenantControlPlane if err = certController.SetupWithManager(mgr); err != nil { diff --git a/config/observability/dashboard/grafana-dashboard-kamaji.json b/config/observability/dashboard/grafana-dashboard-kamaji.json new file mode 100644 index 0000000..ecb0012 --- /dev/null +++ b/config/observability/dashboard/grafana-dashboard-kamaji.json @@ -0,0 +1,2131 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "barchart", + "name": "Bar chart", + "version": "" + }, + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "12.2.0" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 10, + "panels": [], + "title": "General", + "type": "row" + }, + { + "gridPos": { + "h": 7, + "w": 5, + "x": 0, + "y": 1 + }, + "id": 4, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n \n
", + "mode": "html" + }, + "pluginVersion": "12.2.0", + "title": "Kamaji", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 19, + "x": 5, + "y": 1 + }, + "id": 3, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "kamaji_build_info", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Build Info", + "transformations": [ + { + "id": "organize", + "options": { + "includeByName": { + "commit": true, + "instance": true, + "pod": true, + "repo": true, + "version": true + }, + "indexByName": { + "commit": 3, + "instance": 1, + "pod": 0, + "repo": 2, + "version": 4 + }, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 9, + "panels": [], + "title": "Tenant Control Planes", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 0, + "y": 9 + }, + "id": 1, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "sort": "desc", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(ready) (kamaji_tenant_control_planes_current)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Tenant Control Planes Ready", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 6, + "y": 9 + }, + "id": 6, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "kamaji_tenant_control_plane_info * on(tcp_namespace, tcp_name) group_left(status, ready) kamaji_tenant_control_plane_status", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Tenant Control Plane Info", + "transformations": [ + { + "id": "organize", + "options": { + "includeByName": { + "address": true, + "datastore_driver": true, + "exposure_strategy": true, + "kubernetes_version": true, + "ready": true, + "status": true, + "tcp_name": true, + "tcp_namespace": true + }, + "indexByName": { + "address": 6, + "datastore_driver": 5, + "exposure_strategy": 7, + "kubernetes_version": 4, + "ready": 3, + "status": 2, + "tcp_name": 1, + "tcp_namespace": 0 + }, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 18, + "y": 9 + }, + "id": 5, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "sort": "desc", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (status) (kamaji_certificates_current)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Certificate Statuses", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 8, + "panels": [], + "title": "Datastores", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 16, + "x": 0, + "y": 20 + }, + "id": 7, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "kamaji_datastore_info * on(datastore_name) group_left(status, ready) kamaji_datastore_status", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "DataStore Info", + "transformations": [ + { + "id": "organize", + "options": { + "includeByName": { + "driver": true, + "datastore_name": true, + "ready": true, + "status": true + }, + "indexByName": { + "driver": 3, + "datastore_name": 0, + "ready": 2, + "status": 1 + }, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 20 + }, + "id": 2, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "sort": "desc", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (ready) (kamaji_datastores_current)", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "DataStores Ready", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 11, + "panels": [], + "title": "Controller", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "hidden", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 12, + "options": { + "barRadius": 0, + "barWidth": 0.9, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "topk(10, sum by (handler) (kamaji_handler_time_seconds_sum) / clamp_min(sum by (handler) (kamaji_handler_time_seconds_count), 1))", + "instant": true, + "legendFormat": "{{handler}}", + "range": false, + "refId": "A" + } + ], + "title": "Top 10 Slowest Handlers (avg seconds)", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": false, + "axisLabel": "", + "axisPlacement": "hidden", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 13, + "options": { + "barRadius": 0, + "barWidth": 0.9, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "topk(15, sum by (controller) (rate(controller_runtime_reconcile_total[5m])))", + "instant": true, + "legendFormat": "{{controller}}", + "range": false, + "refId": "A" + } + ], + "title": "Top 15 Reconcile Throughput (req/s)", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "topk(15, sum by (controller) (rate(controller_runtime_reconcile_errors_total[5m])) / clamp_min(sum by (controller) (rate(controller_runtime_reconcile_total[5m])), 1e-9))", + "instant": false, + "legendFormat": "{{controller}}", + "range": true, + "refId": "A" + } + ], + "title": "Reconcile Error Ratio (5m)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "topk(15, histogram_quantile(0.95, sum by (controller, le) (rate(controller_runtime_reconcile_time_seconds_bucket[5m])))) and on(controller) (sum by (controller) (rate(controller_runtime_reconcile_total[5m])) > 0.01)", + "instant": false, + "legendFormat": "{{controller}}", + "range": true, + "refId": "A" + } + ], + "title": "Reconcile Latency p95 by Controller (5m)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (webhook, code) (rate(controller_runtime_webhook_requests_total[5m])) > 0", + "instant": false, + "legendFormat": "{{webhook}} {{code}}", + "range": true, + "refId": "A" + } + ], + "title": "Webhook Requests by Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (webhook) (rate(controller_runtime_webhook_latency_seconds_sum[5m])) / sum by (webhook) (rate(controller_runtime_webhook_latency_seconds_count[5m]))", + "instant": false, + "legendFormat": "{{webhook}}", + "range": true, + "refId": "A" + } + ], + "title": "Webhook Average Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "orange", + "value": 0.9 + }, + { + "color": "yellow", + "value": 0.95 + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 31 + }, + "id": 27, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "1 - (sum(rate(controller_runtime_reconcile_total{result=\"error\"}[5m])) / sum(rate(controller_runtime_reconcile_total[5m])))", + "format": "time_series", + "instant": false, + "legendFormat": "Value", + "range": true, + "refId": "A" + } + ], + "title": "Global Reconcile Success Ratio (5m)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 0.15 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 31 + }, + "id": 28, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(controller_runtime_reconcile_total{result=\"error\"}[5m])) / clamp_min(sum(rate(controller_runtime_reconcile_total{result=~\"success|error\"}[5m])), 1e-9)", + "legendFormat": "Value", + "range": true, + "refId": "A" + } + ], + "title": "Global Reconcile Error Ratio (5m)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 0.15 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 31 + }, + "id": 29, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(controller_runtime_webhook_requests_total{code!~\"2..\"}[5m])) / clamp_min(sum(rate(controller_runtime_webhook_requests_total[5m])), 1e-9)", + "legendFormat": "Value", + "range": true, + "refId": "A" + } + ], + "title": "Webhook Error Ratio (5m)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 1.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 31 + }, + "id": 30, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le) (rate(controller_runtime_reconcile_time_seconds_bucket[5m])))", + "legendFormat": "Value", + "range": true, + "refId": "A" + } + ], + "title": "Global Reconcile Latency p95 (5m)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 17, + "panels": [], + "title": "Runtime", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 70 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "go_goroutines", + "legendFormat": "goroutines", + "range": true, + "refId": "A" + } + ], + "title": "Goroutines", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 70 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "go_memstats_heap_inuse_bytes", + "legendFormat": "heap in use", + "range": true, + "refId": "A" + } + ], + "title": "Go Heap In Use", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 70 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le) (rate(go_gc_pauses_seconds_bucket[5m])))", + "legendFormat": "gc pause p95", + "range": true, + "refId": "A" + } + ], + "title": "GC Pause p95 (5m)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 78 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(process_cpu_seconds_total[5m])", + "legendFormat": "cpu cores", + "range": true, + "refId": "A" + } + ], + "title": "Process CPU Usage (5m rate)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 78 + }, + "id": 22, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "process_resident_memory_bytes", + "legendFormat": "rss", + "range": true, + "refId": "A" + } + ], + "title": "Resident Memory (RSS)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 78 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le) (rate(go_sched_latencies_seconds_bucket[5m])))", + "legendFormat": "sched latency p95", + "range": true, + "refId": "A" + } + ], + "title": "Scheduler Latency p95 (5m)", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 42, + "tags": [ + "kamaji", + "prometheus", + "kubernetes" + ], + "templating": { + "list": [ + { + "current": {}, + "includeAll": false, + "label": "Datasource", + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Kamaji Overview", + "uid": "kamaji-overview", + "weekStart": "", + "id": null +} diff --git a/config/observability/grafana.yaml b/config/observability/grafana.yaml new file mode 100644 index 0000000..52ddbbb --- /dev/null +++ b/config/observability/grafana.yaml @@ -0,0 +1,154 @@ +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: monitoring +spec: + type: ClusterIP + selector: + operator.prometheus.io/name: kamaji + ports: + - name: http + port: 9090 + targetPort: 9090 + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-datasources + namespace: monitoring +data: + prometheus.yaml: | + apiVersion: 1 + datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus.monitoring.svc:9090 + isDefault: true + editable: false + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-providers + namespace: monitoring +data: + dashboards.yaml: | + apiVersion: 1 + providers: + - name: kamaji + orgId: 1 + folder: Kamaji + type: file + disableDeletion: false + editable: true + updateIntervalSeconds: 30 + options: + path: /var/lib/grafana/dashboards + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-config + namespace: monitoring +data: + grafana.ini: | + [server] + root_url = %(protocol)s://%(domain)s/ + + [security] + allow_embedding = true + + [auth.basic] + enabled = true + + [auth.anonymous] + enabled = true + org_name = Main Org. + org_role = Viewer + + [users] + auto_assign_org = true + auto_assign_org_role = Viewer + +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana + namespace: monitoring +spec: + type: ClusterIP + selector: + app: grafana + ports: + - name: http + port: 3000 + targetPort: 3000 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: + app: grafana + template: + metadata: + labels: + app: grafana + spec: + securityContext: + fsGroup: 472 + containers: + - name: grafana + image: grafana/grafana:12.2.0 + ports: + - containerPort: 3000 + name: http + env: + - name: GF_PATHS_CONFIG + value: /etc/grafana/grafana.ini + - name: GF_SECURITY_ADMIN_USER + value: admin + - name: GF_SECURITY_ADMIN_PASSWORD + value: kamaji + volumeMounts: + - name: grafana-config + mountPath: /etc/grafana/grafana.ini + subPath: grafana.ini + - name: grafana-datasources + mountPath: /etc/grafana/provisioning/datasources/prometheus.yaml + subPath: prometheus.yaml + - name: grafana-dashboard-providers + mountPath: /etc/grafana/provisioning/dashboards/dashboards.yaml + subPath: dashboards.yaml + - name: grafana-dashboards + mountPath: /var/lib/grafana/dashboards + readOnly: true + resources: + requests: + cpu: 100m + memory: 256Mi + volumes: + - name: grafana-config + configMap: + name: grafana-config + - name: grafana-datasources + configMap: + name: grafana-datasources + - name: grafana-dashboard-providers + configMap: + name: grafana-dashboard-providers + - name: grafana-dashboards + configMap: + name: grafana-dashboards diff --git a/config/observability/prometheus.yaml b/config/observability/prometheus.yaml new file mode 100644 index 0000000..7027c86 --- /dev/null +++ b/config/observability/prometheus.yaml @@ -0,0 +1,74 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus + namespace: monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus +rules: + - apiGroups: [""] + resources: ["nodes", "nodes/metrics", "services", "endpoints", "pods"] + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingresses"] + verbs: ["get", "list", "watch"] + - nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: + - kind: ServiceAccount + name: prometheus + namespace: monitoring +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: kamaji + namespace: monitoring + labels: + monitoring: kamaji +spec: + namespaceSelector: + matchNames: + - kamaji-system + selector: + matchLabels: + app.kubernetes.io/instance: kamaji + app.kubernetes.io/component: metrics + endpoints: + - port: metrics + path: /metrics + interval: 10s +--- +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + name: kamaji + namespace: monitoring +spec: + replicas: 1 + serviceAccountName: prometheus + serviceMonitorSelector: + matchLabels: + monitoring: kamaji + resources: + requests: + cpu: 100m + memory: 512Mi + retention: 24h diff --git a/controllers/certificate_lifecycle_controller.go b/controllers/certificate_lifecycle_controller.go index 9dc0de2..9797c4d 100644 --- a/controllers/certificate_lifecycle_controller.go +++ b/controllers/certificate_lifecycle_controller.go @@ -12,6 +12,7 @@ import ( corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1" controllerruntime "sigs.k8s.io/controller-runtime" @@ -19,6 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -26,6 +28,7 @@ import ( "github.com/clastix/kamaji/controllers/utils" "github.com/clastix/kamaji/internal/constants" "github.com/clastix/kamaji/internal/crypto" + "github.com/clastix/kamaji/internal/metrics" "github.com/clastix/kamaji/internal/utilities" ) @@ -33,12 +36,21 @@ type CertificateLifecycle struct { Channel chan event.GenericEvent Deadline time.Duration EnqueueFn func(secret *corev1.Secret) + Metrics *metrics.Recorder client client.Client } func (s *CertificateLifecycle) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { logger := log.FromContext(ctx) + defer func(c context.Context) { + metricCtx, cancelMetricCtx := metrics.NewRefreshContextFrom(c) + defer cancelMetricCtx() + + if err := s.refreshCertificatesMetrics(metricCtx); err != nil { + logger.WithName("metrics").Error(err, "cannot refresh certificate status gauges") + } + }(ctx) logger.Info("starting CertificateLifecycle handling") @@ -176,6 +188,19 @@ func (s *CertificateLifecycle) extractCertificateFromKubeconfig(secret corev1.Se func (s *CertificateLifecycle) SetupWithManager(mgr controllerruntime.Manager) error { s.client = mgr.GetClient() + if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { + metricCtx, cancelMetricCtx := metrics.NewRefreshContextFrom(ctx) + defer cancelMetricCtx() + + if err := s.refreshCertificatesMetrics(metricCtx); err != nil { + controllerruntime.Log.WithName("metrics").Error(err, "cannot initialize certificate status gauges") + } + + return nil + })); err != nil { + return err + } + supportedStrategies := sets.New[string](utilities.CertificateX509Label, utilities.CertificateKubeconfigLabel) return controllerruntime.NewControllerManagedBy(mgr). @@ -195,3 +220,104 @@ func (s *CertificateLifecycle) SetupWithManager(mgr controllerruntime.Manager) e }))). Complete(s) } + +func (s *CertificateLifecycle) refreshCertificatesMetrics(ctx context.Context) error { + metricsRecorder := s.metricsRecorder() + metricsRecorder.ResetCertificatesStatusCounts() + + countsByTenantControlPlane := map[k8stypes.NamespacedName]map[string]map[string]int{} + + var tenantControlPlaneList kamajiv1alpha1.TenantControlPlaneList + if err := s.client.List(ctx, &tenantControlPlaneList); err != nil { + return err + } + + for i := range tenantControlPlaneList.Items { + tcp := tenantControlPlaneList.Items[i] + namespacedName := k8stypes.NamespacedName{Namespace: tcp.GetNamespace(), Name: tcp.GetName()} + countsByTenantControlPlane[namespacedName] = metrics.NewCertificateStatusCounts() + } + + var secretList corev1.SecretList + if err := s.client.List(ctx, &secretList); err != nil { + return err + } + + deadline := time.Now().Add(s.Deadline) + + for i := range secretList.Items { + secret := secretList.Items[i] + labels := secret.GetLabels() + if labels == nil { + continue + } + + tenantControlPlaneName, ok := labels[constants.ControlPlaneLabelKey] + if !ok || tenantControlPlaneName == "" { + continue + } + + strategy, ok := certificateStrategyFromLabel(labels[constants.ControllerLabelResource]) + if !ok { + continue + } + + namespacedName := k8stypes.NamespacedName{Namespace: secret.GetNamespace(), Name: tenantControlPlaneName} + if _, exists := countsByTenantControlPlane[namespacedName]; !exists { + countsByTenantControlPlane[namespacedName] = metrics.NewCertificateStatusCounts() + } + + var ( + crt *x509.Certificate + err error + ) + + switch strategy { + case metrics.CertificateStrategyX509: + crt, err = s.extractCertificateFromBareSecret(secret) + case metrics.CertificateStrategyKubeconfig: + crt, err = s.extractCertificateFromKubeconfig(secret) + default: + continue + } + + if err != nil { + countsByTenantControlPlane[namespacedName][metrics.CertificateStatusInvalid][strategy]++ + + continue + } + + if deadline.After(crt.NotAfter) { + countsByTenantControlPlane[namespacedName][metrics.CertificateStatusExpiring][strategy]++ + + continue + } + + countsByTenantControlPlane[namespacedName][metrics.CertificateStatusValid][strategy]++ + } + + for namespacedName, counts := range countsByTenantControlPlane { + metricsRecorder.SetCertificatesStatusCounts(namespacedName.Namespace, namespacedName.Name, counts) + } + + return nil +} + +func certificateStrategyFromLabel(label string) (string, bool) { + switch label { + case utilities.CertificateX509Label: + return metrics.CertificateStrategyX509, true + case utilities.CertificateKubeconfigLabel: + return metrics.CertificateStrategyKubeconfig, true + default: + return "", false + } +} + +func (s *CertificateLifecycle) metricsRecorder() *metrics.Recorder { + if s.Metrics == nil { + s.Metrics = metrics.DefaultRecorder() + } + + return s.Metrics +} diff --git a/controllers/certificate_lifecycle_controller_metrics_test.go b/controllers/certificate_lifecycle_controller_metrics_test.go new file mode 100644 index 0000000..2c2e797 --- /dev/null +++ b/controllers/certificate_lifecycle_controller_metrics_test.go @@ -0,0 +1,137 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package controllers + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/constants" + "github.com/clastix/kamaji/internal/metrics" + "github.com/clastix/kamaji/internal/utilities" +) + +func TestCertificateRefreshMetrics(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := kamajiv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("failed adding kamaji scheme: %v", err) + } + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatalf("failed adding corev1 scheme: %v", err) + } + + validCertPEM, err := testCertificatePEM(time.Now().Add(48 * time.Hour)) + if err != nil { + t.Fatalf("failed generating valid cert: %v", err) + } + expiringCertPEM, err := testCertificatePEM(time.Now().Add(30 * time.Minute)) + if err != nil { + t.Fatalf("failed generating expiring cert: %v", err) + } + + reader := fake.NewClientBuilder().WithScheme(scheme).WithObjects( + &kamajiv1alpha1.TenantControlPlane{ObjectMeta: metav1.ObjectMeta{Name: "tcp-a", Namespace: "default"}}, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cert-valid", + Namespace: "default", + Labels: map[string]string{ + constants.ControlPlaneLabelKey: "tcp-a", + constants.ControllerLabelResource: utilities.CertificateX509Label, + }, + }, + Data: map[string][]byte{"tls.crt": validCertPEM}, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cert-expiring", + Namespace: "default", + Labels: map[string]string{ + constants.ControlPlaneLabelKey: "tcp-a", + constants.ControllerLabelResource: utilities.CertificateX509Label, + }, + }, + Data: map[string][]byte{"tls.crt": expiringCertPEM}, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cert-invalid", + Namespace: "default", + Labels: map[string]string{ + constants.ControlPlaneLabelKey: "tcp-a", + constants.ControllerLabelResource: utilities.CertificateX509Label, + }, + }, + Data: map[string][]byte{"tls.crt": []byte("not-a-certificate")}, + }, + ).Build() + + registry := prometheus.NewRegistry() + recorder := metrics.NewRecorder(registry) + + s := &CertificateLifecycle{ + Deadline: 24 * time.Hour, + client: reader, + Metrics: recorder, + } + + if err := s.refreshCertificatesMetrics(t.Context()); err != nil { + t.Fatalf("refreshCertificatesMetrics returned error: %v", err) + } + + family := mustMetricFamilyFromGatherer(t, registry, "kamaji_certificates_current") + if got := gaugeValueByLabels(t, family, map[string]string{"tcp_namespace": "default", "tcp_name": "tcp-a", "status": metrics.CertificateStatusValid, "strategy": metrics.CertificateStrategyX509}); got != 1 { + t.Fatalf("expected valid/x509 certificates gauge to be 1, got %v", got) + } + if got := gaugeValueByLabels(t, family, map[string]string{"tcp_namespace": "default", "tcp_name": "tcp-a", "status": metrics.CertificateStatusExpiring, "strategy": metrics.CertificateStrategyX509}); got != 1 { + t.Fatalf("expected expiring/x509 certificates gauge to be 1, got %v", got) + } + if got := gaugeValueByLabels(t, family, map[string]string{"tcp_namespace": "default", "tcp_name": "tcp-a", "status": metrics.CertificateStatusInvalid, "strategy": metrics.CertificateStrategyX509}); got != 1 { + t.Fatalf("expected invalid/x509 certificates gauge to be 1, got %v", got) + } + if got := gaugeValueByLabels(t, family, map[string]string{"tcp_namespace": "default", "tcp_name": "tcp-a", "status": metrics.CertificateStatusValid, "strategy": metrics.CertificateStrategyKubeconfig}); got != 0 { + t.Fatalf("expected valid/kubeconfig certificates gauge to be 0, got %v", got) + } +} + +func testCertificatePEM(notAfter time.Time) ([]byte, error) { + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, err + } + + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: "metrics-test", + }, + NotBefore: time.Now().Add(-time.Minute), + NotAfter: notAfter, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + BasicConstraintsValid: true, + } + + derBytes, err := x509.CreateCertificate(rand.Reader, template, template, &privateKey.PublicKey, privateKey) + if err != nil { + return nil, err + } + + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}), nil +} diff --git a/controllers/datastore_controller.go b/controllers/datastore_controller.go index f1632d1..33f200a 100644 --- a/controllers/datastore_controller.go +++ b/controllers/datastore_controller.go @@ -21,14 +21,17 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" "github.com/clastix/kamaji/controllers/utils" + "github.com/clastix/kamaji/internal/metrics" ) type DataStore struct { - Client client.Client + Client client.Client + Metrics *metrics.Recorder // TenantControlPlaneTrigger is the channel used to communicate across the controllers: // if a Data Source is updated, we have to be sure that the reconciliation of the certificates content // for each Tenant Control Plane is put in place properly. @@ -43,6 +46,15 @@ func (r *DataStore) Reconcile(ctx context.Context, request reconcile.Request) (r logger := log.FromContext(ctx) + defer func(c context.Context) { + metricCtx, cancelMetricCtx := metrics.NewRefreshContextFrom(c) + defer cancelMetricCtx() + + if gaugeErr := r.refreshDatastoreMetrics(metricCtx); gaugeErr != nil { + logger.WithName("metrics").Error(gaugeErr, "cannot refresh DataStore metrics") + } + }(ctx) + var ds kamajiv1alpha1.DataStore if dsErr := r.Client.Get(ctx, request.NamespacedName, &ds); dsErr != nil { if k8serrors.IsNotFound(dsErr) { @@ -238,6 +250,19 @@ func (r *DataStore) Reconcile(ctx context.Context, request reconcile.Request) (r } func (r *DataStore) SetupWithManager(mgr controllerruntime.Manager) error { + if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { + metricCtx, cancelMetricCtx := metrics.NewRefreshContextFrom(ctx) + defer cancelMetricCtx() + + if err := r.refreshDatastoreMetrics(metricCtx); err != nil { + controllerruntime.Log.WithName("metrics").Error(err, "cannot initialize DataStore metrics") + } + + return nil + })); err != nil { + return err + } + enqueueFn := func(tcp *kamajiv1alpha1.TenantControlPlane, limitingInterface workqueue.TypedRateLimitingInterface[reconcile.Request]) { if dataStoreName := tcp.Status.Storage.DataStoreName; len(dataStoreName) > 0 { limitingInterface.AddRateLimited(reconcile.Request{ @@ -265,6 +290,65 @@ func (r *DataStore) SetupWithManager(mgr controllerruntime.Manager) error { Complete(r) } +func (r *DataStore) refreshDatastoreMetrics(ctx context.Context) error { + var dataStoreList kamajiv1alpha1.DataStoreList + + if err := r.Client.List(ctx, &dataStoreList); err != nil { + return err + } + + metricsRecorder := r.metricsRecorder() + metricsRecorder.ResetDataStoreInfo() + metricsRecorder.ResetDataStoreStatus() + metricsRecorder.ResetDatastoresReadyAndDriverCounts() + + for i := range dataStoreList.Items { + ds := &dataStoreList.Items[i] + driver := metrics.NormalizeDataStoreDriverLabel(ds.Spec.Driver) + ready := metrics.ReadyLabelFalse + if ds.Status.Ready { + ready = metrics.ReadyLabelTrue + } + + status := classifyDataStoreStatusLabel(ds) + + metricsRecorder.SetDataStoreStatus(ds.GetName(), status, ready) + metricsRecorder.SetDataStoreInfo( + ds.GetName(), + driver, + ) + + counts := metrics.NewSingleDriverReadyCounts(driver) + + if ds.Status.Ready { + counts[metrics.ReadyLabelTrue][driver] = 1 + } else { + counts[metrics.ReadyLabelFalse][driver] = 1 + } + + metricsRecorder.SetDatastoresReadyAndDriverCounts(ds.GetName(), counts) + } + + return nil +} + +func (r *DataStore) metricsRecorder() *metrics.Recorder { + if r.Metrics == nil { + r.Metrics = metrics.DefaultRecorder() + } + + return r.Metrics +} + +func classifyDataStoreStatusLabel(ds *kamajiv1alpha1.DataStore) string { + condition := meta.FindStatusCondition(ds.Status.Conditions, kamajiv1alpha1.DataStoreConditionValidType) + if condition == nil { + return metrics.DataStoreStatusUnknown + } + + return metrics.NormalizeDataStoreConditionStatusLabel(condition.Status) +} + func (r *DataStore) validateBasicAuth(ctx context.Context, ds kamajiv1alpha1.DataStore) error { if err := r.validateContentReference(ctx, ds.Spec.BasicAuth.Password); err != nil { return fmt.Errorf("basic-auth password is not valid, %w", err) diff --git a/controllers/datastore_controller_metrics_test.go b/controllers/datastore_controller_metrics_test.go new file mode 100644 index 0000000..8278cb2 --- /dev/null +++ b/controllers/datastore_controller_metrics_test.go @@ -0,0 +1,124 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package controllers + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + io_prometheus_client "github.com/prometheus/client_model/go" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/metrics" +) + +func TestDataStoreRefreshMetrics(t *testing.T) { + t.Parallel() + + scheme := runtime.NewScheme() + if err := kamajiv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("failed adding kamaji scheme: %v", err) + } + + reader := fake.NewClientBuilder().WithScheme(scheme).WithObjects( + &kamajiv1alpha1.DataStore{ + ObjectMeta: metav1.ObjectMeta{Name: "primary"}, + Spec: kamajiv1alpha1.DataStoreSpec{Driver: kamajiv1alpha1.EtcdDriver}, + Status: kamajiv1alpha1.DataStoreStatus{ + Ready: true, + Conditions: []metav1.Condition{{ + Type: kamajiv1alpha1.DataStoreConditionValidType, + Status: metav1.ConditionTrue, + }}, + }, + }, + &kamajiv1alpha1.DataStore{ + ObjectMeta: metav1.ObjectMeta{Name: "secondary"}, + Spec: kamajiv1alpha1.DataStoreSpec{Driver: kamajiv1alpha1.Driver("unsupported")}, + Status: kamajiv1alpha1.DataStoreStatus{Ready: false}, + }, + ).Build() + + registry := prometheus.NewRegistry() + recorder := metrics.NewRecorder(registry) + + r := &DataStore{ + Client: reader, + Metrics: recorder, + } + + if err := r.refreshDatastoreMetrics(t.Context()); err != nil { + t.Fatalf("refreshDatastoreMetrics returned error: %v", err) + } + + countFamily := mustMetricFamilyFromGatherer(t, registry, "kamaji_datastores_current") + if got := gaugeValueByLabels(t, countFamily, map[string]string{"datastore_name": "primary", "ready": metrics.ReadyLabelTrue, "driver": string(kamajiv1alpha1.EtcdDriver)}); got != 1 { + t.Fatalf("expected primary ready/etcd count to be 1, got %v", got) + } + if got := gaugeValueByLabels(t, countFamily, map[string]string{"datastore_name": "secondary", "ready": metrics.ReadyLabelFalse, "driver": metrics.DataStoreDriverUnknown}); got != 1 { + t.Fatalf("expected secondary not-ready/unknown count to be 1, got %v", got) + } + + statusFamily := mustMetricFamilyFromGatherer(t, registry, "kamaji_datastore_status") + if got := gaugeValueByLabels(t, statusFamily, map[string]string{"datastore_name": "primary", "status": metrics.DataStoreStatusTrue, "ready": metrics.ReadyLabelTrue}); got != 1 { + t.Fatalf("expected primary status metric to be 1, got %v", got) + } + if got := gaugeValueByLabels(t, statusFamily, map[string]string{"datastore_name": "secondary", "status": metrics.DataStoreStatusUnknown, "ready": metrics.ReadyLabelFalse}); got != 1 { + t.Fatalf("expected secondary status metric to be 1, got %v", got) + } +} + +func mustMetricFamilyFromGatherer(t *testing.T, gatherer prometheus.Gatherer, name string) *io_prometheus_client.MetricFamily { + t.Helper() + + families, err := gatherer.Gather() + if err != nil { + t.Fatalf("failed to gather metrics: %v", err) + } + + for _, family := range families { + if family.GetName() == name { + return family + } + } + + t.Fatalf("metric family %s not found", name) + + return nil +} + +func gaugeValueByLabels(t *testing.T, family *io_prometheus_client.MetricFamily, labels map[string]string) float64 { + t.Helper() + + for _, metric := range family.GetMetric() { + allMatch := true + for name, value := range labels { + matched := false + for _, label := range metric.GetLabel() { + if label.GetName() == name && label.GetValue() == value { + matched = true + + break + } + } + + if !matched { + allMatch = false + + break + } + } + + if allMatch { + return metric.GetGauge().GetValue() + } + } + + t.Fatalf("metric with labels %v not found", labels) + + return 0 +} diff --git a/controllers/tenantcontrolplane_controller.go b/controllers/tenantcontrolplane_controller.go index 47df584..cb12687 100644 --- a/controllers/tenantcontrolplane_controller.go +++ b/controllers/tenantcontrolplane_controller.go @@ -7,6 +7,8 @@ import ( "context" "errors" "fmt" + "net" + "strconv" "strings" "time" @@ -29,6 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" @@ -41,6 +44,7 @@ import ( controlplanebuilder "github.com/clastix/kamaji/internal/builders/controlplane" "github.com/clastix/kamaji/internal/datastore" kamajierrors "github.com/clastix/kamaji/internal/errors" + "github.com/clastix/kamaji/internal/metrics" "github.com/clastix/kamaji/internal/resources" "github.com/clastix/kamaji/internal/utilities" ) @@ -49,6 +53,7 @@ import ( type TenantControlPlaneReconciler struct { Client client.Client APIReader client.Reader + Metrics *metrics.Recorder Config TenantControlPlaneReconcilerConfig TriggerChan chan event.GenericEvent KamajiNamespace string @@ -92,6 +97,15 @@ type TenantControlPlaneReconcilerConfig struct { func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := log.FromContext(ctx) + defer func(c context.Context) { + metricCtx, cancelMetricCtx := metrics.NewRefreshContextFrom(c) + defer cancelMetricCtx() + + if err := r.refreshTenantControlPlaneMetrics(metricCtx); err != nil { + log.WithName("metrics").Error(err, "cannot refresh TenantControlPlane metrics") + } + }(ctx) + var cancelFn context.CancelFunc ctx, cancelFn = context.WithTimeout(ctx, r.ReconcileTimeout) defer cancelFn() @@ -304,6 +318,19 @@ func (r *TenantControlPlaneReconciler) mutexSpec(obj client.Object) mutex.Spec { func (r *TenantControlPlaneReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { r.clock = clock.RealClock{} + if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { + metricCtx, cancelMetricCtx := metrics.NewRefreshContextFrom(ctx) + defer cancelMetricCtx() + + if err := r.refreshTenantControlPlaneMetrics(metricCtx); err != nil { + ctrl.Log.WithName("metrics").Error(err, "cannot initialize TenantControlPlane metrics") + } + + return nil + })); err != nil { + return err + } + controllerBuilder := ctrl.NewControllerManagedBy(mgr). WatchesRawSource(source.Channel(r.CertificateChan, handler.Funcs{GenericFunc: func(_ context.Context, genericEvent event.TypedGenericEvent[client.Object], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { w.AddRateLimited(ctrl.Request{ @@ -374,6 +401,93 @@ func (r *TenantControlPlaneReconciler) SetupWithManager(ctx context.Context, mgr Complete(r) } +func (r *TenantControlPlaneReconciler) refreshTenantControlPlaneMetrics(ctx context.Context) error { + var tcpList kamajiv1alpha1.TenantControlPlaneList + + if err := r.Client.List(ctx, &tcpList); err != nil { + return err + } + + metricsRecorder := r.metricsRecorder() + metricsRecorder.ResetTenantControlPlaneInfo() + metricsRecorder.ResetTenantControlPlaneStatus() + + counts := metrics.NewReadyCounts() + + for i := range tcpList.Items { + tcp := &tcpList.Items[i] + status := classifyTenantControlPlaneStatusLabel(tcp) + ready := classifyTenantControlPlaneReadyLabel(status) + counts[ready]++ + + metricsRecorder.SetTenantControlPlaneStatus( + tcp.GetNamespace(), + tcp.GetName(), + status, + ready, + ) + + metricsRecorder.SetTenantControlPlaneInfo( + tcp.GetNamespace(), + tcp.GetName(), + metrics.NormalizeTenantControlPlaneKubernetesVersionLabel(tcp.Spec.Kubernetes.Version), + metrics.NormalizeDataStoreDriverStringLabel(tcp.Status.Storage.Driver), + resolveTenantControlPlaneAddress(tcp), + classifyTenantControlPlaneExposureStrategyLabel(tcp), + ) + } + + metricsRecorder.SetTenantControlPlanesReadyCounts(counts) + + return nil +} + +func (r *TenantControlPlaneReconciler) metricsRecorder() *metrics.Recorder { + if r.Metrics == nil { + r.Metrics = metrics.DefaultRecorder() + } + + return r.Metrics +} + +func classifyTenantControlPlaneStatusLabel(tcp *kamajiv1alpha1.TenantControlPlane) string { + return metrics.NormalizeTenantControlPlaneStatusLabel(tcp.Status.Kubernetes.Version.Status) +} + +func classifyTenantControlPlaneReadyLabel(status string) string { + if status == string(kamajiv1alpha1.VersionReady) { + return metrics.ReadyLabelTrue + } + + return metrics.ReadyLabelFalse +} + +func classifyTenantControlPlaneExposureStrategyLabel(tcp *kamajiv1alpha1.TenantControlPlane) string { + if tcp.Spec.ControlPlane.Gateway != nil { + return metrics.TenantControlPlaneExposureGateway + } + + if tcp.Spec.ControlPlane.Service.ServiceType != "" { + return metrics.TenantControlPlaneExposureService + } + + return metrics.TenantControlPlaneExposureUnknown +} + +func resolveTenantControlPlaneAddress(tcp *kamajiv1alpha1.TenantControlPlane) string { + address, port, err := tcp.AssignedControlPlaneAddress() + if err != nil || strings.TrimSpace(address) == "" || port <= 0 { + // Empty string is used as safe fallback when no externally reachable address can be resolved. + return "" + } + + return formatHTTPSAddress(address, port) +} + +func formatHTTPSAddress(host string, port int32) string { + return "https://" + net.JoinHostPort(host, strconv.FormatInt(int64(port), 10)) +} + func (r *TenantControlPlaneReconciler) getTenantControlPlane(ctx context.Context, namespacedName k8stypes.NamespacedName) utils.TenantControlPlaneRetrievalFn { return func() (*kamajiv1alpha1.TenantControlPlane, error) { tcp := &kamajiv1alpha1.TenantControlPlane{} diff --git a/controllers/tenantcontrolplane_controller_metrics_test.go b/controllers/tenantcontrolplane_controller_metrics_test.go new file mode 100644 index 0000000..f682891 --- /dev/null +++ b/controllers/tenantcontrolplane_controller_metrics_test.go @@ -0,0 +1,40 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package controllers + +import ( + "testing" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" +) + +func TestResolveTenantControlPlaneAddressFromAssignedEndpoint(t *testing.T) { + t.Parallel() + + tcp := &kamajiv1alpha1.TenantControlPlane{ + Status: kamajiv1alpha1.TenantControlPlaneStatus{ + ControlPlaneEndpoint: "203.0.113.10:6443", + }, + } + + address := resolveTenantControlPlaneAddress(tcp) + if address != "https://203.0.113.10:6443" { + t.Fatalf("expected endpoint address https://203.0.113.10:6443, got %q", address) + } +} + +func TestResolveTenantControlPlaneAddressReturnsEmptyWhenUnresolvable(t *testing.T) { + tcp := &kamajiv1alpha1.TenantControlPlane{ + Spec: kamajiv1alpha1.TenantControlPlaneSpec{ + ControlPlane: kamajiv1alpha1.ControlPlane{ + Service: kamajiv1alpha1.ServiceSpec{ServiceType: kamajiv1alpha1.ServiceTypeLoadBalancer}, + }, + }, + } + + address := resolveTenantControlPlaneAddress(tcp) + if address != "" { + t.Fatalf("expected empty address fallback, got %q", address) + } +} diff --git a/docs/content/guides/monitoring.md b/docs/content/guides/monitoring.md index 5e30a21..9d8994a 100644 --- a/docs/content/guides/monitoring.md +++ b/docs/content/guides/monitoring.md @@ -1,11 +1,56 @@ -# Tenant Control Plane Monitoring - -Kamaji exposes a set of metrics that can be used to monitor the health of the Tenant Control Plane (TCP) and its components. The metrics are exposed in Prometheus format and can be scraped by a Prometheus server instance running in the Management Cluster. +# Monitoring +Kamaji exposes metrics for both the management plane (Kamaji controllers) and Tenant Control Plane (TCP) components. Metrics are exposed in Prometheus format and can be scraped from the management cluster. ## Prerequisites Ensure you have installed the [Prometheus Operator](https://prometheus.io/community/) in the Management Cluster and that it is configured properly. You should verify that Service Monitor CRDs are installed in the Management Cluster as they are used to tell Prometheus how to scrape the metrics from the TCP. +## Kamaji management-plane metrics + +Kamaji exposes management-plane metrics from the controller-manager `/metrics` endpoint. You can scrape this endpoint with Prometheus and use the metrics to monitor `TenantControlPlanes`, `DataStores`, certificate lifecycle, handler latency, and build metadata. + +The custom Kamaji metrics are: + +- `kamaji_tenant_control_plane_info` +- `kamaji_tenant_control_plane_status` +- `kamaji_tenant_control_planes_current` +- `kamaji_datastore_info` +- `kamaji_datastore_status` +- `kamaji_datastores_current` +- `kamaji_certificates_current` +- `kamaji_handler_time_seconds` +- `kamaji_build_info` + +In addition, Kamaji also exposes the default Go runtime and controller-runtime metrics. + +To enable scraping, create a `ServiceMonitor` like the following: + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: kamaji + namespace: default +spec: + namespaceSelector: + matchNames: + - kamaji-system + selector: + matchLabels: + app.kubernetes.io/instance: kamaji + app.kubernetes.io/component: metrics + endpoints: + - port: metrics + path: /metrics + interval: 15s +``` + +A ready-to-use Grafana dashboard for these metrics is available [here](https://raw.githubusercontent.com/clastix/kamaji/master/config/observability/dashboard/grafana-dashboard-kamaji.json). + +![Kamaji Monitoring Dashboard](../images/kamaji-monitoring-dashboard.png) + +## Tenant Control Plane component metrics + ## Enable metrics scraping On the Management Cluster, in the same namespace as the Tenant Control Plane, create a Service Monitor that instructs Prometheus how to scrape the metrics from the TCP. @@ -206,4 +251,4 @@ kubeScheduler: !!! tip "Multi-Cluster Mode" In Grafana, enable the "Multi-Cluster Mode" option for improved visualization of metrics. This option is available in the Grafana settings. -That's it! \ No newline at end of file +That's it! diff --git a/docs/content/images/kamaji-monitoring-dashboard.png b/docs/content/images/kamaji-monitoring-dashboard.png new file mode 100644 index 0000000..0c078eb Binary files /dev/null and b/docs/content/images/kamaji-monitoring-dashboard.png differ diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..51a5c35 --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,382 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package metrics + +import ( + "context" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + kamajiversion "github.com/clastix/kamaji/internal" +) + +const ( + ReadyLabelTrue = "true" + ReadyLabelFalse = "false" + + DataStoreDriverUnknown = "unknown" + + DataStoreStatusUnknown = "Unknown" + DataStoreStatusTrue = "True" + DataStoreStatusFalse = "False" + + TenantControlPlaneExposureService = "service" + TenantControlPlaneExposureGateway = "gateway" + TenantControlPlaneExposureUnknown = "unknown" + + TenantControlPlaneKubernetesVersionUnknown = "unknown" + + CertificateStatusValid = "valid" + CertificateStatusExpiring = "expiring" + CertificateStatusInvalid = "invalid" + + CertificateStrategyX509 = "x509" + CertificateStrategyKubeconfig = "kubeconfig" + + kamajiNamespace = "kamaji" + + buildSubsystem = "build" + tenantControlPlaneSubsystem = "tenant_control_plane" + tenantControlPlanesS = "tenant_control_planes" + datastoreSubsystem = "datastore" + datastoresSubsystem = "datastores" + certificatesSubsystem = "certificates" + + metricNameInfo = "info" + metricNameStatus = "status" + metricNameCurrent = "current" + + labelTCPNamespace = "tcp_namespace" + labelTCPName = "tcp_name" + labelKubernetesVer = "kubernetes_version" + labelDatastoreDriver = "datastore_driver" + labelAddress = "address" + labelExposureStrategy = "exposure_strategy" + labelStatus = "status" + labelReady = "ready" + labelDataStoreName = "datastore_name" + labelDriver = "driver" + labelStrategy = "strategy" + labelVersion = "version" + labelCommit = "commit" + labelDirty = "dirty" + labelRepo = "repo" + labelBuildTime = "build_time" + + metricsRefreshTimeout = 5 * time.Second +) + +func NewRefreshContext() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), metricsRefreshTimeout) +} + +func NewRefreshContextFrom(ctx context.Context) (context.Context, context.CancelFunc) { + if ctx == nil { + ctx = context.Background() + } + + return context.WithTimeout(ctx, metricsRefreshTimeout) +} + +var readyLabelValues = []string{ + ReadyLabelTrue, + ReadyLabelFalse, +} + +var tenantControlPlaneStatuses = []string{ + string(kamajiv1alpha1.VersionUnknown), + string(kamajiv1alpha1.VersionProvisioning), + string(kamajiv1alpha1.VersionCARotating), + string(kamajiv1alpha1.VersionUpgrading), + string(kamajiv1alpha1.VersionMigrating), + string(kamajiv1alpha1.VersionReady), + string(kamajiv1alpha1.VersionNotReady), + string(kamajiv1alpha1.VersionSleeping), + string(kamajiv1alpha1.VersionWriteLimited), +} + +var datastoreDrivers = []string{ + string(kamajiv1alpha1.EtcdDriver), + string(kamajiv1alpha1.KineMySQLDriver), + string(kamajiv1alpha1.KinePostgreSQLDriver), + string(kamajiv1alpha1.KineNatsDriver), + DataStoreDriverUnknown, +} + +var certificateStatuses = []string{ + CertificateStatusValid, + CertificateStatusExpiring, + CertificateStatusInvalid, +} + +var certificateStrategies = []string{ + CertificateStrategyX509, + CertificateStrategyKubeconfig, +} + +var ( + defaultRecorderOnce sync.Once + defaultRecorder *Recorder +) + +type Recorder struct { + buildInfo *prometheus.GaugeVec + tenantControlPlaneInfo *prometheus.GaugeVec + tenantControlPlaneStatus *prometheus.GaugeVec + datastoreInfo *prometheus.GaugeVec + datastoreStatus *prometheus.GaugeVec + controlPlanesCount *prometheus.GaugeVec + datastoresCount *prometheus.GaugeVec + certificatesCount *prometheus.GaugeVec +} + +func DefaultRecorder() *Recorder { + defaultRecorderOnce.Do(func() { + defaultRecorder = NewRecorder(ctrlmetrics.Registry) + }) + + return defaultRecorder +} + +func NewRecorder(registerer prometheus.Registerer) *Recorder { + if registerer == nil { + registerer = ctrlmetrics.Registry + } + + recorder := &Recorder{ + buildInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: kamajiNamespace, + Subsystem: buildSubsystem, + Name: metricNameInfo, + Help: "Build information for Kamaji.", + }, []string{labelVersion, labelCommit, labelDirty, labelRepo, labelBuildTime}), + tenantControlPlaneInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: kamajiNamespace, + Subsystem: tenantControlPlaneSubsystem, + Name: metricNameInfo, + Help: "Identity and high-level configuration information for TenantControlPlane resources.", + }, []string{labelTCPNamespace, labelTCPName, labelKubernetesVer, labelDatastoreDriver, labelAddress, labelExposureStrategy}), + tenantControlPlaneStatus: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: kamajiNamespace, + Subsystem: tenantControlPlaneSubsystem, + Name: metricNameStatus, + Help: "Current status and readiness of TenantControlPlane resources.", + }, []string{labelTCPNamespace, labelTCPName, labelStatus, labelReady}), + datastoreInfo: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: kamajiNamespace, + Subsystem: datastoreSubsystem, + Name: metricNameInfo, + Help: "Identity and high-level configuration information for DataStore resources.", + }, []string{labelDataStoreName, labelDriver}), + datastoreStatus: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: kamajiNamespace, + Subsystem: datastoreSubsystem, + Name: metricNameStatus, + Help: "Current status and readiness of DataStore resources.", + }, []string{labelDataStoreName, labelStatus, labelReady}), + controlPlanesCount: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: kamajiNamespace, + Subsystem: tenantControlPlanesS, + Name: metricNameCurrent, + Help: "Current number of TenantControlPlane resources by ready flag.", + }, []string{labelReady}), + datastoresCount: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: kamajiNamespace, + Subsystem: datastoresSubsystem, + Name: metricNameCurrent, + Help: "Current number of DataStore resources by name, ready flag and driver.", + }, []string{labelDataStoreName, labelReady, labelDriver}), + certificatesCount: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: kamajiNamespace, + Subsystem: certificatesSubsystem, + Name: metricNameCurrent, + Help: "Current number of certificate Secrets by tcp_namespace, tcp_name, status and strategy.", + }, []string{labelTCPNamespace, labelTCPName, labelStatus, labelStrategy}), + } + + registerer.MustRegister( + recorder.buildInfo, + recorder.tenantControlPlaneInfo, + recorder.tenantControlPlaneStatus, + recorder.datastoreInfo, + recorder.datastoreStatus, + recorder.controlPlanesCount, + recorder.datastoresCount, + recorder.certificatesCount, + ) + + recorder.buildInfo.WithLabelValues( + kamajiversion.GitTag, + kamajiversion.GitCommit, + kamajiversion.GitDirty, + kamajiversion.GitRepo, + kamajiversion.BuildTime, + ).Set(1) + + recorder.SetTenantControlPlanesReadyCounts(NewReadyCounts()) + + return recorder +} + +func NewReadyCounts() map[string]int { + return map[string]int{ + ReadyLabelTrue: 0, + ReadyLabelFalse: 0, + } +} + +func NewSingleDriverReadyCounts(driver string) map[string]map[string]int { + return map[string]map[string]int{ + ReadyLabelTrue: { + driver: 0, + }, + ReadyLabelFalse: { + driver: 0, + }, + } +} + +func NewCertificateStatusCounts() map[string]map[string]int { + return map[string]map[string]int{ + CertificateStatusValid: { + CertificateStrategyX509: 0, + CertificateStrategyKubeconfig: 0, + }, + CertificateStatusExpiring: { + CertificateStrategyX509: 0, + CertificateStrategyKubeconfig: 0, + }, + CertificateStatusInvalid: { + CertificateStrategyX509: 0, + CertificateStrategyKubeconfig: 0, + }, + } +} + +func (r *Recorder) SetTenantControlPlanesReadyCounts(counts map[string]int) { + for _, ready := range readyLabelValues { + r.controlPlanesCount.WithLabelValues(ready).Set(float64(counts[ready])) + } +} + +func (r *Recorder) ResetTenantControlPlaneStatus() { + r.tenantControlPlaneStatus.Reset() +} + +func (r *Recorder) SetTenantControlPlaneStatus(tcpNamespace, tcpName, status, ready string) { + r.tenantControlPlaneStatus.WithLabelValues(tcpNamespace, tcpName, status, ready).Set(1) +} + +func (r *Recorder) ResetDataStoreStatus() { + r.datastoreStatus.Reset() +} + +func (r *Recorder) SetDataStoreStatus(datastoreName, status, ready string) { + r.datastoreStatus.WithLabelValues(datastoreName, status, ready).Set(1) +} + +func (r *Recorder) ResetTenantControlPlaneInfo() { + r.tenantControlPlaneInfo.Reset() +} + +func (r *Recorder) SetTenantControlPlaneInfo(tcpNamespace, tcpName, kubernetesVersion, dataStoreDriver, address, exposureStrategy string) { + r.tenantControlPlaneInfo.WithLabelValues( + tcpNamespace, + tcpName, + kubernetesVersion, + dataStoreDriver, + address, + exposureStrategy, + ).Set(1) +} + +func (r *Recorder) ResetDataStoreInfo() { + r.datastoreInfo.Reset() +} + +func (r *Recorder) SetDataStoreInfo(datastoreName, dataStoreDriver string) { + r.datastoreInfo.WithLabelValues(datastoreName, dataStoreDriver).Set(1) +} + +func NormalizeTenantControlPlaneStatusLabel(status *kamajiv1alpha1.KubernetesVersionStatus) string { + if status == nil { + return string(kamajiv1alpha1.VersionUnknown) + } + + value := string(*status) + for _, allowedStatus := range tenantControlPlaneStatuses { + if value == allowedStatus { + return value + } + } + + return string(kamajiv1alpha1.VersionUnknown) +} + +func (r *Recorder) ResetDatastoresReadyAndDriverCounts() { + r.datastoresCount.Reset() +} + +func (r *Recorder) SetDatastoresReadyAndDriverCounts(datastoreName string, counts map[string]map[string]int) { + for ready, readyCounts := range counts { + for driver, value := range readyCounts { + r.datastoresCount.WithLabelValues(datastoreName, ready, driver).Set(float64(value)) + } + } +} + +func NormalizeDataStoreDriverLabel(driver kamajiv1alpha1.Driver) string { + for _, knownDriver := range datastoreDrivers { + if string(driver) == knownDriver { + return string(driver) + } + } + + return DataStoreDriverUnknown +} + +func NormalizeDataStoreDriverStringLabel(driver string) string { + return NormalizeDataStoreDriverLabel(kamajiv1alpha1.Driver(driver)) +} + +func NormalizeDataStoreConditionStatusLabel(status metav1.ConditionStatus) string { + switch status { + case metav1.ConditionTrue: + return DataStoreStatusTrue + case metav1.ConditionFalse: + return DataStoreStatusFalse + default: + return DataStoreStatusUnknown + } +} + +func NormalizeTenantControlPlaneKubernetesVersionLabel(version string) string { + if version == "" { + return TenantControlPlaneKubernetesVersionUnknown + } + + return version +} + +func (r *Recorder) ResetCertificatesStatusCounts() { + r.certificatesCount.Reset() +} + +func (r *Recorder) SetCertificatesStatusCounts(tcpNamespace, tcpName string, counts map[string]map[string]int) { + for _, status := range certificateStatuses { + for _, strategy := range certificateStrategies { + value := 0 + if statusCounts, ok := counts[status]; ok { + value = statusCounts[strategy] + } + + r.certificatesCount.WithLabelValues(tcpNamespace, tcpName, status, strategy).Set(float64(value)) + } + } +} diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go new file mode 100644 index 0000000..9c0dd5b --- /dev/null +++ b/internal/metrics/metrics_test.go @@ -0,0 +1,399 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package metrics + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + io_prometheus_client "github.com/prometheus/client_model/go" + "sigs.k8s.io/controller-runtime/pkg/metrics" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" +) + +func testRecorder() *Recorder { + return DefaultRecorder() +} + +func TestKamajiMetricRegistration(t *testing.T) { + t.Helper() + recorder := testRecorder() + + recorder.ResetDatastoresReadyAndDriverCounts() + recorder.SetDatastoresReadyAndDriverCounts("default", map[string]map[string]int{ + ReadyLabelTrue: { + string(kamajiv1alpha1.EtcdDriver): 1, + }, + }) + + recorder.ResetCertificatesStatusCounts() + recorder.SetCertificatesStatusCounts("default", "k8s-133", map[string]map[string]int{ + CertificateStatusValid: { + CertificateStrategyX509: 1, + }, + }) + + wanted := map[string]struct{}{ + "kamaji_build_info": {}, + "kamaji_tenant_control_planes_current": {}, + "kamaji_datastores_current": {}, + "kamaji_certificates_current": {}, + } + + families, err := metrics.Registry.Gather() + if err != nil { + t.Fatalf("failed to gather metrics: %v", err) + } + + seen := map[string]struct{}{} + for _, family := range families { + if _, ok := wanted[family.GetName()]; ok { + seen[family.GetName()] = struct{}{} + } + } + + for name := range wanted { + if _, ok := seen[name]; !ok { + t.Fatalf("metric %s not registered", name) + } + } +} + +func TestNewRecorderWithCustomRegistry(t *testing.T) { + t.Helper() + + registry := prometheus.NewRegistry() + recorder := NewRecorder(registry) + recorder.SetTenantControlPlanesReadyCounts(map[string]int{ + ReadyLabelTrue: 1, + ReadyLabelFalse: 0, + }) + + family := mustMetricFamilyFromGatherer(t, registry, "kamaji_tenant_control_planes_current") + if got := gaugeValueByLabels(t, family, map[string]string{"ready": ReadyLabelTrue}); got != 1 { + t.Fatalf("expected ready control planes gauge to be 1 on custom registry, got %v", got) + } +} + +func TestTenantControlPlanesCountGaugeByStatus(t *testing.T) { + t.Helper() + recorder := testRecorder() + + recorder.SetTenantControlPlanesReadyCounts(map[string]int{ + ReadyLabelTrue: 2, + ReadyLabelFalse: 1, + }) + + family := mustMetricFamily(t, "kamaji_tenant_control_planes_current") + if len(family.GetMetric()) != len(readyLabelValues) { + t.Fatalf("expected %d ready-only gauge metrics, got %d", len(readyLabelValues), len(family.GetMetric())) + } + + if got := gaugeValueByLabels(t, family, map[string]string{"ready": ReadyLabelTrue}); got != 2 { + t.Fatalf("expected ready control planes gauge to be 2, got %v", got) + } + + if got := gaugeValueByLabels(t, family, map[string]string{"ready": ReadyLabelFalse}); got != 1 { + t.Fatalf("expected not-ready control planes gauge to be 1, got %v", got) + } +} + +func TestDatastoresCountGaugeByStatus(t *testing.T) { + t.Helper() + recorder := testRecorder() + + recorder.ResetDatastoresReadyAndDriverCounts() + + recorder.SetDatastoresReadyAndDriverCounts("default", map[string]map[string]int{ + ReadyLabelTrue: { + string(kamajiv1alpha1.EtcdDriver): 1, + }, + ReadyLabelFalse: { + string(kamajiv1alpha1.EtcdDriver): 0, + }, + }) + recorder.SetDatastoresReadyAndDriverCounts("secondary", map[string]map[string]int{ + ReadyLabelTrue: { + string(kamajiv1alpha1.KineMySQLDriver): 1, + }, + ReadyLabelFalse: { + string(kamajiv1alpha1.KineMySQLDriver): 0, + }, + }) + + family := mustMetricFamily(t, "kamaji_datastores_current") + if len(family.GetMetric()) != 4 { + t.Fatalf("expected 4 name-ready-driver gauge metrics, got %d", len(family.GetMetric())) + } + + assertMetricFamilyHasLabels(t, family, "datastore_name", "ready", "driver") + assertMetricFamilyDoesNotHaveLabels(t, family, "namespace") + + if got := gaugeValueByLabels(t, family, map[string]string{"datastore_name": "default", "ready": ReadyLabelTrue, "driver": string(kamajiv1alpha1.EtcdDriver)}); got != 1 { + t.Fatalf("expected ready datastores gauge to be 1, got %v", got) + } + + if got := gaugeValueByLabels(t, family, map[string]string{"datastore_name": "default", "ready": ReadyLabelFalse, "driver": string(kamajiv1alpha1.EtcdDriver)}); got != 0 { + t.Fatalf("expected not_ready/etcd datastores gauge to be 0, got %v", got) + } + + if got := gaugeValueByLabels(t, family, map[string]string{"datastore_name": "secondary", "ready": ReadyLabelTrue, "driver": string(kamajiv1alpha1.KineMySQLDriver)}); got != 1 { + t.Fatalf("expected secondary/mysql ready datastores gauge to be 1, got %v", got) + } + + if got := gaugeValueByLabels(t, family, map[string]string{"datastore_name": "secondary", "ready": ReadyLabelFalse, "driver": string(kamajiv1alpha1.KineMySQLDriver)}); got != 0 { + t.Fatalf("expected secondary/mysql not-ready datastores gauge to be 0, got %v", got) + } +} + +func TestDataStoreInfoMetric(t *testing.T) { + t.Helper() + recorder := testRecorder() + + recorder.ResetDataStoreInfo() + recorder.SetDataStoreInfo("default", string(kamajiv1alpha1.EtcdDriver)) + + family := mustMetricFamily(t, "kamaji_datastore_info") + if len(family.GetMetric()) != 1 { + t.Fatalf("expected one datastore_info sample, got %d", len(family.GetMetric())) + } + + labels := map[string]string{ + "datastore_name": "default", + "driver": string(kamajiv1alpha1.EtcdDriver), + } + + assertMetricFamilyHasLabels(t, family, "datastore_name", "driver") + assertMetricFamilyDoesNotHaveLabels(t, family, "tls", "basic_auth") + + if got := gaugeValueByLabels(t, family, labels); got != 1 { + t.Fatalf("expected datastore_info value to be 1, got %v", got) + } +} + +func TestDataStoreStatusMetric(t *testing.T) { + t.Helper() + recorder := testRecorder() + + recorder.ResetDataStoreStatus() + recorder.SetDataStoreStatus("default", DataStoreStatusTrue, ReadyLabelTrue) + + family := mustMetricFamily(t, "kamaji_datastore_status") + if len(family.GetMetric()) != 1 { + t.Fatalf("expected one datastore_status sample, got %d", len(family.GetMetric())) + } + + labels := map[string]string{ + "datastore_name": "default", + "status": DataStoreStatusTrue, + "ready": ReadyLabelTrue, + } + + if got := gaugeValueByLabels(t, family, labels); got != 1 { + t.Fatalf("expected datastore_status value to be 1, got %v", got) + } +} + +func TestBuildInfoMetricHasSingleSample(t *testing.T) { + t.Helper() + + family := mustMetricFamily(t, "kamaji_build_info") + if len(family.GetMetric()) != 1 { + t.Fatalf("expected a single build info metric, got %d", len(family.GetMetric())) + } + + if got := family.GetMetric()[0].GetGauge().GetValue(); got != 1 { + t.Fatalf("expected build info gauge to be 1, got %v", got) + } +} + +func TestTenantControlPlaneInfoMetric(t *testing.T) { + t.Helper() + recorder := testRecorder() + + recorder.ResetTenantControlPlaneInfo() + recorder.SetTenantControlPlaneInfo("default", "test", "v1.33.0", string(kamajiv1alpha1.EtcdDriver), "https://203.0.113.10:6443", TenantControlPlaneExposureService) + + family := mustMetricFamily(t, "kamaji_tenant_control_plane_info") + if len(family.GetMetric()) != 1 { + t.Fatalf("expected one tenant_control_plane_info sample, got %d", len(family.GetMetric())) + } + + labels := map[string]string{ + "tcp_namespace": "default", + "tcp_name": "test", + "kubernetes_version": "v1.33.0", + "datastore_driver": string(kamajiv1alpha1.EtcdDriver), + "address": "https://203.0.113.10:6443", + "exposure_strategy": TenantControlPlaneExposureService, + } + + assertMetricFamilyHasLabels(t, family, "tcp_namespace", "tcp_name", "kubernetes_version", "datastore_driver", "address", "exposure_strategy") + assertMetricFamilyDoesNotHaveLabels(t, family, "namespace", "name") + + if got := gaugeValueByLabels(t, family, labels); got != 1 { + t.Fatalf("expected tenant_control_plane_info value to be 1, got %v", got) + } +} + +func TestTenantControlPlaneStatusMetric(t *testing.T) { + t.Helper() + recorder := testRecorder() + + recorder.ResetTenantControlPlaneStatus() + recorder.SetTenantControlPlaneStatus("default", "test", string(kamajiv1alpha1.VersionReady), ReadyLabelTrue) + + family := mustMetricFamily(t, "kamaji_tenant_control_plane_status") + if len(family.GetMetric()) != 1 { + t.Fatalf("expected one tenant_control_plane_status sample, got %d", len(family.GetMetric())) + } + + labels := map[string]string{ + "tcp_namespace": "default", + "tcp_name": "test", + "status": string(kamajiv1alpha1.VersionReady), + "ready": ReadyLabelTrue, + } + + assertMetricFamilyHasLabels(t, family, "tcp_namespace", "tcp_name", "status", "ready") + assertMetricFamilyDoesNotHaveLabels(t, family, "namespace", "name") + + if got := gaugeValueByLabels(t, family, labels); got != 1 { + t.Fatalf("expected tenant_control_plane_status value to be 1, got %v", got) + } +} + +func TestCertificatesCountGaugeByStatusAndStrategy(t *testing.T) { + t.Helper() + recorder := testRecorder() + + recorder.ResetCertificatesStatusCounts() + + recorder.SetCertificatesStatusCounts("default", "k8s-133", map[string]map[string]int{ + CertificateStatusValid: { + CertificateStrategyX509: 4, + CertificateStrategyKubeconfig: 1, + }, + CertificateStatusExpiring: { + CertificateStrategyX509: 2, + CertificateStrategyKubeconfig: 0, + }, + CertificateStatusInvalid: { + CertificateStrategyX509: 1, + CertificateStrategyKubeconfig: 3, + }, + }) + + family := mustMetricFamily(t, "kamaji_certificates_current") + if len(family.GetMetric()) != len(certificateStatuses)*len(certificateStrategies) { + t.Fatalf("expected %d tcp_namespace-tcp_name-status-strategy gauge metrics, got %d", len(certificateStatuses)*len(certificateStrategies), len(family.GetMetric())) + } + + assertMetricFamilyHasLabels(t, family, "tcp_namespace", "tcp_name", "status", "strategy") + assertMetricFamilyDoesNotHaveLabels(t, family, "namespace", "name") + + if got := gaugeValueByLabels(t, family, map[string]string{"tcp_namespace": "default", "tcp_name": "k8s-133", "status": CertificateStatusValid, "strategy": CertificateStrategyX509}); got != 4 { + t.Fatalf("expected valid/x509 certificates gauge to be 4, got %v", got) + } + + if got := gaugeValueByLabels(t, family, map[string]string{"tcp_namespace": "default", "tcp_name": "k8s-133", "status": CertificateStatusInvalid, "strategy": CertificateStrategyKubeconfig}); got != 3 { + t.Fatalf("expected invalid/kubeconfig certificates gauge to be 3, got %v", got) + } +} + +func assertMetricFamilyHasLabels(t *testing.T, family *io_prometheus_client.MetricFamily, labels ...string) { + t.Helper() + + if len(family.GetMetric()) == 0 { + t.Fatalf("metric family %s has no samples", family.GetName()) + } + + present := map[string]struct{}{} + for _, label := range family.GetMetric()[0].GetLabel() { + present[label.GetName()] = struct{}{} + } + + for _, expected := range labels { + if _, ok := present[expected]; !ok { + t.Fatalf("metric family %s missing label %s", family.GetName(), expected) + } + } +} + +func assertMetricFamilyDoesNotHaveLabels(t *testing.T, family *io_prometheus_client.MetricFamily, labels ...string) { + t.Helper() + + if len(family.GetMetric()) == 0 { + t.Fatalf("metric family %s has no samples", family.GetName()) + } + + present := map[string]struct{}{} + for _, label := range family.GetMetric()[0].GetLabel() { + present[label.GetName()] = struct{}{} + } + + for _, forbidden := range labels { + if _, ok := present[forbidden]; ok { + t.Fatalf("metric family %s unexpectedly contains label %s", family.GetName(), forbidden) + } + } +} + +func mustMetricFamily(t *testing.T, name string) *io_prometheus_client.MetricFamily { + t.Helper() + + return mustMetricFamilyFromGatherer(t, metrics.Registry, name) +} + +func mustMetricFamilyFromGatherer(t *testing.T, gatherer prometheus.Gatherer, name string) *io_prometheus_client.MetricFamily { + t.Helper() + + families, err := gatherer.Gather() + if err != nil { + t.Fatalf("failed to gather metrics: %v", err) + } + + for _, family := range families { + if family.GetName() == name { + return family + } + } + + t.Fatalf("metric family %s not found", name) + + return nil +} + +func gaugeValueByLabels(t *testing.T, family *io_prometheus_client.MetricFamily, labels map[string]string) float64 { + t.Helper() + + for _, metric := range family.GetMetric() { + allMatch := true + for name, value := range labels { + matched := false + for _, label := range metric.GetLabel() { + if label.GetName() == name && label.GetValue() == value { + matched = true + + break + } + } + + if !matched { + allMatch = false + + break + } + } + + if allMatch { + return metric.GetGauge().GetValue() + } + } + + t.Fatalf("metric with labels %v not found", labels) + + return 0 +} diff --git a/internal/resources/metrics.go b/internal/resources/metrics.go index 6076ca5..f571306 100644 --- a/internal/resources/metrics.go +++ b/internal/resources/metrics.go @@ -4,6 +4,8 @@ package resources import ( + "sync" + "github.com/prometheus/client_golang/prometheus" "sigs.k8s.io/controller-runtime/pkg/metrics" ) @@ -27,26 +29,28 @@ var ( kubeadmphaseUploadConfigKubeletCollector prometheus.Histogram kubeadmphaseBootstrapTokenCollector prometheus.Histogram kubeadmphaseClusterAdminRBACCollector prometheus.Histogram + + handlerCollectorOnce sync.Once + handlerCollector = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "kamaji", + Subsystem: "handler", + Name: "time_seconds", + Help: "Bucket time requested for the given handler to complete its handling.", + Buckets: []float64{ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, + 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60, + }, + }, []string{"handler"}) ) func LazyLoadHistogramFromResource(collector prometheus.Histogram, resource Resource) prometheus.Histogram { - n := resource.GetName() + handlerCollectorOnce.Do(func() { + metrics.Registry.MustRegister(handlerCollector) + }) if collector == nil { - c := prometheus.NewHistogram(prometheus.HistogramOpts{ - Namespace: "kamaji", - Subsystem: "handler", - Name: n + "_time_seconds", - Help: "Bucket time requested for the given handler to complete its handling.", - Buckets: []float64{ - 0.005, 0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, - 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60, - }, - }) - - metrics.Registry.MustRegister(c) - - return c + //nolint:forcetypeassert // handlerCollector is a HistogramVec and returns a histogram observer for label values. + return handlerCollector.WithLabelValues(resource.GetName()).(prometheus.Histogram) } return collector diff --git a/internal/resources/metrics_test.go b/internal/resources/metrics_test.go new file mode 100644 index 0000000..edecc07 --- /dev/null +++ b/internal/resources/metrics_test.go @@ -0,0 +1,111 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "testing" + + "github.com/prometheus/client_golang/prometheus" + io_prometheus_client "github.com/prometheus/client_model/go" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/metrics" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" +) + +func TestHandlerHistogramUsesSingleMetricWithHandlerLabel(t *testing.T) { + handlerCollector.Reset() + + first := fakeResource{name: "test_handler_a"} + second := fakeResource{name: "test_handler_b"} + + LazyLoadHistogramFromResource(nil, first).Observe(0.10) + LazyLoadHistogramFromResource(nil, second).Observe(0.20) + + families, err := metrics.Registry.Gather() + if err != nil { + t.Fatalf("failed to gather metric families: %v", err) + } + + var ( + foundHandlerFamily bool + foundLegacyName bool + ) + + for _, family := range families { + name := family.GetName() + if name == "kamaji_handler_time_seconds" { + foundHandlerFamily = true + + if !hasLabelValueForFamily(family, "handler", "test_handler_a") { + t.Fatalf("expected handler label value test_handler_a in kamaji_handler_time_seconds") + } + + if !hasLabelValueForFamily(family, "handler", "test_handler_b") { + t.Fatalf("expected handler label value test_handler_b in kamaji_handler_time_seconds") + } + } + + if name == "kamaji_handler_test_handler_a_time_seconds" || name == "kamaji_handler_test_handler_b_time_seconds" { + foundLegacyName = true + } + } + + if !foundHandlerFamily { + t.Fatalf("metric family kamaji_handler_time_seconds not found") + } + + if foundLegacyName { + t.Fatalf("unexpected legacy per-handler metric family found") + } +} + +func hasLabelValueForFamily(family *io_prometheus_client.MetricFamily, labelName, labelValue string) bool { + for _, metric := range family.GetMetric() { + for _, label := range metric.GetLabel() { + if label.GetName() == labelName && label.GetValue() == labelValue { + return true + } + } + } + + return false +} + +type fakeResource struct { + name string +} + +func (f fakeResource) Define(context.Context, *kamajiv1alpha1.TenantControlPlane) error { + return nil +} + +func (f fakeResource) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool { + return false +} + +func (f fakeResource) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) { + return false, nil +} + +func (f fakeResource) CreateOrUpdate(context.Context, *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + return controllerutil.OperationResultNone, nil +} + +func (f fakeResource) GetName() string { + return f.name +} + +func (f fakeResource) ShouldStatusBeUpdated(context.Context, *kamajiv1alpha1.TenantControlPlane) bool { + return false +} + +func (f fakeResource) UpdateTenantControlPlaneStatus(context.Context, *kamajiv1alpha1.TenantControlPlane) error { + return nil +} + +func (f fakeResource) GetHistogram() prometheus.Histogram { + return LazyLoadHistogramFromResource(nil, f) +}