diff --git a/kustomize/base/prometheus/deployment.yaml b/kustomize/base/prometheus/deployment.yaml index 3afb95f8..0008fd85 100644 --- a/kustomize/base/prometheus/deployment.yaml +++ b/kustomize/base/prometheus/deployment.yaml @@ -19,7 +19,7 @@ spec: serviceAccountName: flagger-prometheus containers: - name: prometheus - image: prom/prometheus:v2.23.0 + image: prom/prometheus:v2.32.1 imagePullPolicy: IfNotPresent args: - '--storage.tsdb.retention=2h' @@ -49,4 +49,4 @@ spec: configMap: name: flagger-prometheus - name: data-volume - emptyDir: {} \ No newline at end of file + emptyDir: {} diff --git a/test/kuma/install.sh b/test/kuma/install.sh index fe121e62..0604e344 100755 --- a/test/kuma/install.sh +++ b/test/kuma/install.sh @@ -4,6 +4,7 @@ set -o errexit KUMA_VER="1.4.1" REPO_ROOT=$(git rev-parse --show-toplevel) +KUSTOMIZE_VERSION=3.8.2 mkdir -p ${REPO_ROOT}/bin echo ">>> Downloading Kuma ${KUMA_VER}" @@ -15,11 +16,9 @@ chmod +x ${REPO_ROOT}/bin/kumactl echo ">>> Installing Kuma ${KUMA_VER}" ${REPO_ROOT}/bin/kumactl install control-plane | kubectl apply -f - -echo ">>> Installing Kuma Metrics" -${REPO_ROOT}/bin/kumactl install metrics | kubectl apply -f - - echo ">>> Waiting for Kuma Control Plane to be ready" -kubectl wait --for=condition=ready pod -n kuma-system -l app=kuma-control-plane +kubectl wait --for condition=established crd/meshes.kuma.io +kubectl -n kuma-system rollout status deployment/kuma-control-plane echo ">>> Configuring Default Kuma Mesh" cat <>> Installing Kustomize' +cd ${REPO_ROOT}/bin && kustomize_url=https://github.com/kubernetes-sigs/kustomize/releases/download && \ +curl -sL ${kustomize_url}/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz | \ +tar xz + echo '>>> Installing Flagger' -kubectl apply -k ${REPO_ROOT}/kustomize/kuma +${REPO_ROOT}/bin/kustomize build ${REPO_ROOT}/test/kuma | kubectl apply -f - kubectl -n kuma-system set image deployment/flagger flagger=test/flagger:latest kubectl -n kuma-system rollout status deployment/flagger diff --git a/test/kuma/kustomization.yaml b/test/kuma/kustomization.yaml new file mode 100644 index 00000000..4dbf623f --- /dev/null +++ b/test/kuma/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kuma-system +resources: + - ../../kustomize/base/prometheus/ + - ../../kustomize/base/flagger/ +patchesStrategicMerge: + - patch.yml +configMapGenerator: + - name: flagger-prometheus + behavior: replace + files: + - prometheus.yml diff --git a/test/kuma/patch.yml b/test/kuma/patch.yml new file mode 100644 index 00000000..9b69cbb8 --- /dev/null +++ b/test/kuma/patch.yml @@ -0,0 +1,14 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flagger +spec: + template: + spec: + containers: + - name: flagger + args: + - -log-level=info + - -include-label-prefix=app.kubernetes.io + - -mesh-provider=kuma + - -metrics-server=http://flagger-prometheus:9090 diff --git a/test/kuma/prometheus.yml b/test/kuma/prometheus.yml new file mode 100644 index 00000000..31756863 --- /dev/null +++ b/test/kuma/prometheus.yml @@ -0,0 +1,163 @@ +global: + scrape_interval: 5s +scrape_configs: + +# scrape config for AppMesh Envoy sidecar +- job_name: 'appmesh-envoy' + metrics_path: /stats/prometheus + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_container_name] + action: keep + regex: '^envoy$' + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: ${1}:9901 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name + # exclude high cardinality metrics + metric_relabel_configs: + - source_labels: [ cluster_name ] + regex: '(outbound|inbound|prometheus_stats).*' + action: drop + - source_labels: [ tcp_prefix ] + regex: '(outbound|inbound|prometheus_stats).*' + action: drop + - source_labels: [ listener_address ] + regex: '(.+)' + action: drop + - source_labels: [ http_conn_manager_listener_prefix ] + regex: '(.+)' + action: drop + - source_labels: [ http_conn_manager_prefix ] + regex: '(.+)' + action: drop + - source_labels: [ __name__ ] + regex: 'envoy_tls.*' + action: drop + - source_labels: [ __name__ ] + regex: 'envoy_tcp_downstream.*' + action: drop + - source_labels: [ __name__ ] + regex: 'envoy_http_(stats|admin).*' + action: drop + - source_labels: [ __name__ ] + regex: 'envoy_cluster_(lb|retry|bind|internal|max|original).*' + action: drop + +# scrape config for API servers +- job_name: 'kubernetes-apiservers' + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - default + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: kubernetes;https + +# scrape config for cAdvisor +- job_name: 'kubernetes-cadvisor' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + # exclude high cardinality metrics + metric_relabel_configs: + - source_labels: [__name__] + regex: (container|machine)_(cpu|memory|network|fs)_(.+) + action: keep + - source_labels: [__name__] + regex: container_memory_failures_total + action: drop + +# scrape config for pods +- job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - source_labels: [ __address__ ] + regex: '.*9901.*' + action: drop + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_pod_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: kubernetes_namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: kubernetes_pod_name + +# scrape config for Kuma dataplanes +- job_name: 'kuma-dataplanes' + scrape_interval: "5s" + relabel_configs: + - source_labels: + - k8s_kuma_io_name + regex: "(.*)" + target_label: pod + - source_labels: + - k8s_kuma_io_namespace + regex: "(.*)" + target_label: namespace + - source_labels: + - __meta_kuma_mesh + regex: "(.*)" + target_label: mesh + - source_labels: + - __meta_kuma_dataplane + regex: "(.*)" + target_label: dataplane + - source_labels: + - __meta_kuma_service + regex: "(.*)" + target_label: service + - action: labelmap + regex: __meta_kuma_label_(.+) + kuma_sd_configs: + - server: http://kuma-control-plane.kuma-system:5676