From 2b45c2013ce5c3c83184297933246cbbe67e0c6b Mon Sep 17 00:00:00 2001
From: Nelson Johnstone <nelson.johnstone@neds.com>
Date: Wed, 8 Feb 2023 10:07:41 +1000
Subject: [PATCH] metric variables documentation and e2e tests

Signed-off-by: Nelson Johnstone <93178586+njohnstone2@users.noreply.github.com>
---
 docs/gitbook/usage/metrics.md | 45 +++++++++++++++++++++++++++++++++++
 test/istio/test-canary.sh     | 12 +++++++++-
 test/linkerd/test-canary.sh   |  4 +++-
 test/nginx/test-canary.sh     |  6 ++++-
 test/osm/test-canary.sh       |  6 ++++-
 5 files changed, 69 insertions(+), 4 deletions(-)

diff --git a/docs/gitbook/usage/metrics.md b/docs/gitbook/usage/metrics.md
index 18f899b7..c1e5bb67 100644
--- a/docs/gitbook/usage/metrics.md
+++ b/docs/gitbook/usage/metrics.md
@@ -62,6 +62,7 @@ The following variables are available in query templates:
 * `service` (canary.spec.service.name)
 * `ingress` (canary.spec.ingresRef.name)
 * `interval` (canary.spec.analysis.metrics[].interval)
+* `variables` (canary.spec.analysis.metrics[].templateVariables)
 
 A canary analysis metric can reference a template with `templateRef`:
 
@@ -82,6 +83,50 @@ A canary analysis metric can reference a template with `templateRef`:
         interval: 1m
 ```
 
+A canary analysis metric can reference a set of custom variables with `templateVariables`. These variables will be then injected into the query defined in the referred `MetricTemplate` object during canary analysis:
+
+```yaml
+  analysis:
+    metrics:
+      - name: "my metric"
+        templateRef:
+          name: my-metric
+          namespace: flagger
+        # accepted values
+        thresholdRange:
+          min: 10
+          max: 1000
+        # metric query time window
+        interval: 1m
+        # custom variables used within the referenced metric template
+        templateVariables:
+          direction: inbound
+```
+
+```yaml
+apiVersion: flagger.app/v1beta1
+kind: MetricTemplate
+metadata:
+  name: my-metric
+spec:
+  provider:
+    type: prometheus
+    address: http://prometheus.linkerd-viz:9090
+  query: |
+    histogram_quantile(
+      0.99,
+      sum(
+        rate(
+          response_latency_ms_bucket{
+            namespace="{{ namespace }}",
+            deployment=~"{{ target }}",
+            direction="{{ variables.direction }}"
+          }[{{ interval }}]
+        )
+      ) by (le)
+    )
+```
+
 ## Prometheus
 
 You can create custom metric checks targeting a Prometheus server by
diff --git a/test/istio/test-canary.sh b/test/istio/test-canary.sh
index 4199a0fc..3165bdaf 100755
--- a/test/istio/test-canary.sh
+++ b/test/istio/test-canary.sh
@@ -22,7 +22,7 @@ spec:
         sum(
             rate(
                 istio_request_duration_milliseconds_bucket{
-                    reporter="destination",
+                    reporter="{{ variables.reporter }}",
                     destination_workload_namespace="{{ namespace }}",
                     destination_workload=~"{{ target }}"
                 }[{{ interval }}]
@@ -75,6 +75,8 @@ spec:
       thresholdRange:
         max: 500
       interval: 1m
+      templateVariables:
+        reporter: destination
     webhooks:
       - name: load-test
         url: http://flagger-loadtester.test/
@@ -195,6 +197,8 @@ spec:
       thresholdRange:
         max: 500
       interval: 30s
+      templateVariables:
+        reporter: destination
     webhooks:
       - name: http-acceptance-test
         type: pre-rollout
@@ -293,6 +297,8 @@ spec:
       thresholdRange:
         max: 500
       interval: 30s
+      templateVariables:
+        reporter: destination
     webhooks:
       - name: pre
         type: pre-rollout
@@ -376,6 +382,8 @@ spec:
       thresholdRange:
         max: 500
       interval: 30s
+      templateVariables:
+        reporter: destination
     webhooks:
       - name: pre
         type: pre-rollout
@@ -509,6 +517,8 @@ spec:
       thresholdRange:
         max: 500
       interval: 30s
+      templateVariables:
+        reporter: destination
     webhooks:
       - name: pre
         type: pre-rollout
diff --git a/test/linkerd/test-canary.sh b/test/linkerd/test-canary.sh
index 6677878b..d79c8529 100755
--- a/test/linkerd/test-canary.sh
+++ b/test/linkerd/test-canary.sh
@@ -24,7 +24,7 @@ spec:
                 response_latency_ms_bucket{
                     namespace="{{ namespace }}",
                     deployment=~"{{ target }}",
-                    direction="inbound"
+                    direction="{{ variables.direction }}"
                     }[{{ interval }}]
                 )
             ) by (le)
@@ -65,6 +65,8 @@ spec:
         namespace: linkerd
       threshold: 300
       interval: 1m
+      templateVariables:
+        direction: inbound
     webhooks:
       - name: http-acceptance-test
         type: pre-rollout
diff --git a/test/nginx/test-canary.sh b/test/nginx/test-canary.sh
index b1e79d3d..2199f72f 100755
--- a/test/nginx/test-canary.sh
+++ b/test/nginx/test-canary.sh
@@ -49,7 +49,7 @@ spec:
           http_request_duration_seconds_bucket{
             kubernetes_namespace="{{ namespace }}",
             kubernetes_pod_name=~"{{ target }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
-            path="root"
+            path="{{ variables.path }}"
           }[{{ interval }}]
         )
       ) by (le)
@@ -92,6 +92,8 @@ spec:
       thresholdRange:
         max: 0.5
       interval: 30s
+      templateVariables:
+        path: root
     webhooks:
       - name: load-test
         url: http://flagger-loadtester.test/
@@ -229,6 +231,8 @@ spec:
       thresholdRange:
         max: 0.5
       interval: 30s
+      templateVariables:
+        path: root
     webhooks:
       - name: test-header-routing
         type: rollout
diff --git a/test/osm/test-canary.sh b/test/osm/test-canary.sh
index 1f076602..89de1a29 100755
--- a/test/osm/test-canary.sh
+++ b/test/osm/test-canary.sh
@@ -22,7 +22,7 @@ spec:
             rate(
                 osm_request_duration_ms_bucket{
                     destination_namespace="{{ namespace }}",
-                    destination_kind="Deployment",
+                    destination_kind="{{ variables.destination_kind }}",
                     destination_name=~"{{ target }}"
                 }[{{ interval }}]
             )
@@ -67,6 +67,8 @@ spec:
         namespace: osm-system
       threshold: 300
       interval: 1m
+      templateVariables:
+        destination_kind: Deployment
     webhooks:
       - name: acceptance-test
         type: pre-rollout
@@ -184,6 +186,8 @@ spec:
         namespace: osm-system
       threshold: 300
       interval: 1m
+      templateVariables:
+        destination_kind: Deployment
     webhooks:
       - name: acceptance-test
         type: pre-rollout