Merge pull request #1093 from aryan9600/aryan9600/fix-kuma-e2e

Fix failing kuma e2e tests

kustomize/base/prometheus/deployment.yaml

@@ -19,7 +19,7 @@ spec:
       serviceAccountName: flagger-prometheus
       containers:
           - name: prometheus
-            image: prom/prometheus:v2.23.0
+            image: prom/prometheus:v2.32.1
             imagePullPolicy: IfNotPresent
             args:
               - '--storage.tsdb.retention=2h'
@@ -49,4 +49,4 @@ spec:
           configMap:
             name: flagger-prometheus
         - name: data-volume
-          emptyDir: {}
+          emptyDir: {}

test/kuma/install.sh

@@ -4,6 +4,7 @@ set -o errexit
 
 KUMA_VER="1.4.1"
 REPO_ROOT=$(git rev-parse --show-toplevel)
+KUSTOMIZE_VERSION=3.8.2
 mkdir -p ${REPO_ROOT}/bin
 
 echo ">>> Downloading Kuma ${KUMA_VER}"
@@ -15,11 +16,9 @@ chmod +x ${REPO_ROOT}/bin/kumactl
 echo ">>> Installing Kuma ${KUMA_VER}"
 ${REPO_ROOT}/bin/kumactl install control-plane | kubectl apply -f -
 
-echo ">>> Installing Kuma Metrics"
-${REPO_ROOT}/bin/kumactl install metrics | kubectl apply -f -
-
 echo ">>> Waiting for Kuma Control Plane to be ready"
-kubectl wait --for=condition=ready pod -n kuma-system -l app=kuma-control-plane
+kubectl wait --for condition=established crd/meshes.kuma.io
+kubectl -n kuma-system rollout status deployment/kuma-control-plane
 
 echo ">>> Configuring Default Kuma Mesh"
 cat <<EOF | kubectl apply -f -
@@ -54,8 +53,13 @@ spec:
             expiration: 10y
 EOF
 
+echo '>>> Installing Kustomize'
+cd ${REPO_ROOT}/bin && kustomize_url=https://github.com/kubernetes-sigs/kustomize/releases/download && \
+curl -sL ${kustomize_url}/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz | \
+tar xz
+
 echo '>>> Installing Flagger'
-kubectl apply -k ${REPO_ROOT}/kustomize/kuma
+${REPO_ROOT}/bin/kustomize build ${REPO_ROOT}/test/kuma | kubectl apply -f -
 
 kubectl -n kuma-system set image deployment/flagger flagger=test/flagger:latest
 kubectl -n kuma-system rollout status deployment/flagger

test/kuma/kustomization.yaml

@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kuma-system
+resources:
+  - ../../kustomize/base/prometheus/
+  - ../../kustomize/base/flagger/
+patchesStrategicMerge:
+  - patch.yml
+configMapGenerator:
+  - name: flagger-prometheus
+    behavior: replace
+    files:
+      - prometheus.yml

test/kuma/patch.yml

@@ -0,0 +1,14 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flagger
+spec:
+  template:
+    spec:
+      containers:
+        - name: flagger
+          args:
+            - -log-level=info
+            - -include-label-prefix=app.kubernetes.io
+            - -mesh-provider=kuma
+            - -metrics-server=http://flagger-prometheus:9090

test/kuma/prometheus.yml

@@ -0,0 +1,163 @@
+global:
+  scrape_interval: 5s
+scrape_configs:
+
+# scrape config for AppMesh Envoy sidecar
+- job_name: 'appmesh-envoy'
+  metrics_path: /stats/prometheus
+  kubernetes_sd_configs:
+  - role: pod
+  relabel_configs:
+  - source_labels: [__meta_kubernetes_pod_container_name]
+    action: keep
+    regex: '^envoy$'
+  - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+    action: replace
+    regex: ([^:]+)(?::\d+)?;(\d+)
+    replacement: ${1}:9901
+    target_label: __address__
+  - action: labelmap
+    regex: __meta_kubernetes_pod_label_(.+)
+  - source_labels: [__meta_kubernetes_namespace]
+    action: replace
+    target_label: kubernetes_namespace
+  - source_labels: [__meta_kubernetes_pod_name]
+    action: replace
+    target_label: kubernetes_pod_name
+  # exclude high cardinality metrics
+  metric_relabel_configs:
+  - source_labels: [ cluster_name ]
+    regex: '(outbound|inbound|prometheus_stats).*'
+    action: drop
+  - source_labels: [ tcp_prefix ]
+    regex: '(outbound|inbound|prometheus_stats).*'
+    action: drop
+  - source_labels: [ listener_address ]
+    regex: '(.+)'
+    action: drop
+  - source_labels: [ http_conn_manager_listener_prefix ]
+    regex: '(.+)'
+    action: drop
+  - source_labels: [ http_conn_manager_prefix ]
+    regex: '(.+)'
+    action: drop
+  - source_labels: [ __name__ ]
+    regex: 'envoy_tls.*'
+    action: drop
+  - source_labels: [ __name__ ]
+    regex: 'envoy_tcp_downstream.*'
+    action: drop
+  - source_labels: [ __name__ ]
+    regex: 'envoy_http_(stats|admin).*'
+    action: drop
+  - source_labels: [ __name__ ]
+    regex: 'envoy_cluster_(lb|retry|bind|internal|max|original).*'
+    action: drop
+
+# scrape config for API servers
+- job_name: 'kubernetes-apiservers'
+  kubernetes_sd_configs:
+  - role: endpoints
+    namespaces:
+      names:
+      - default
+  scheme: https
+  tls_config:
+    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    insecure_skip_verify: true
+  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+  relabel_configs:
+  - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
+    action: keep
+    regex: kubernetes;https
+
+# scrape config for cAdvisor
+- job_name: 'kubernetes-cadvisor'
+  scheme: https
+  tls_config:
+    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    insecure_skip_verify: true
+  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+  kubernetes_sd_configs:
+  - role: node
+  relabel_configs:
+  - action: labelmap
+    regex: __meta_kubernetes_node_label_(.+)
+  - target_label: __address__
+    replacement: kubernetes.default.svc:443
+  - source_labels: [__meta_kubernetes_node_name]
+    regex: (.+)
+    target_label: __metrics_path__
+    replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
+  # exclude high cardinality metrics
+  metric_relabel_configs:
+  - source_labels: [__name__]
+    regex: (container|machine)_(cpu|memory|network|fs)_(.+)
+    action: keep
+  - source_labels: [__name__]
+    regex: container_memory_failures_total
+    action: drop
+
+# scrape config for pods
+- job_name: kubernetes-pods
+  kubernetes_sd_configs:
+  - role: pod
+  relabel_configs:
+  - action: keep
+    regex: true
+    source_labels:
+    - __meta_kubernetes_pod_annotation_prometheus_io_scrape
+  - source_labels: [ __address__ ]
+    regex: '.*9901.*'
+    action: drop
+  - action: replace
+    regex: (.+)
+    source_labels:
+    - __meta_kubernetes_pod_annotation_prometheus_io_path
+    target_label: __metrics_path__
+  - action: replace
+    regex: ([^:]+)(?::\d+)?;(\d+)
+    replacement: $1:$2
+    source_labels:
+    - __address__
+    - __meta_kubernetes_pod_annotation_prometheus_io_port
+    target_label: __address__
+  - action: labelmap
+    regex: __meta_kubernetes_pod_label_(.+)
+  - action: replace
+    source_labels:
+    - __meta_kubernetes_namespace
+    target_label: kubernetes_namespace
+  - action: replace
+    source_labels:
+    - __meta_kubernetes_pod_name
+    target_label: kubernetes_pod_name
+
+# scrape config for Kuma dataplanes
+- job_name: 'kuma-dataplanes'
+  scrape_interval: "5s"
+  relabel_configs:
+    - source_labels:
+        - k8s_kuma_io_name
+      regex: "(.*)"
+      target_label: pod
+    - source_labels:
+        - k8s_kuma_io_namespace
+      regex: "(.*)"
+      target_label: namespace
+    - source_labels:
+        - __meta_kuma_mesh
+      regex: "(.*)"
+      target_label: mesh
+    - source_labels:
+        - __meta_kuma_dataplane
+      regex: "(.*)"
+      target_label: dataplane
+    - source_labels:
+        - __meta_kuma_service
+      regex: "(.*)"
+      target_label: service
+    - action: labelmap
+      regex: __meta_kuma_label_(.+)
+  kuma_sd_configs:
+    - server: http://kuma-control-plane.kuma-system:5676