diff --git a/.gitignore b/.gitignore index b5089ae..9d39a03 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +x86_64-darwin.zip +x86_64-linux.zip pack.sh x86_64-darwin x86_64-linux diff --git a/README.md b/README.md index 22d1a7b..5beeb11 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ 1. [Description](#Description) 2. [Special Input Keys and features](#Special-Input-Keys-and-features) -3. [Installation](#Installation) -4. [Prometheus Metrics](#Metrics) +3. [Prometheus metrics and Grafana](#Metrics) +4. [Installation](#Installation) 5. [Notes for large clusters](#Notes-for-large-clusters) 6. [Configuration](#Configuration) @@ -45,9 +45,33 @@ Move the spaceship over a white alien. ## Hands-on Tutorial To experience KubeInvaders in action, try it out in this free O'Reilly Katacoda scenario, [KubeInvaders](https://www.katacoda.com/kuber-ru/courses/kubernetes-chaos). +## Metrics +KubeInvaders exposes metrics for Prometheus through the standard endpoint /metrics + +Example of metrics + +```bash +curl localhost:8080/metrics + +# Total number of chaos jobs executed on the node workernode01 +chaos_jobs_node_count{node=workernode01} 100 + +# Total number of chaos jobs executed against all worker nodes +chaos_node_jobs_total 200 + +# Total number of deleted pods +deleted_pods_total 16 + +# Total number of deleted pods +deleted_namespace_pods_count{namespace=myawesomenamespace} 8 +``` +![Download Grafana dashboard](https://github.com/lucky-sideburn/KubeInvaders/blob/master/grafana/KubeInvadersDashboard.json) + +![Alt Text](https://github.com/lucky-sideburn/KubeInvaders/blob/master/grafana2.png) + +![Alt Text](https://github.com/lucky-sideburn/KubeInvaders/blob/master/grafana.png) ## Installation - ### Install to Kubernetes with Helm (v3+) ```bash @@ -83,9 +107,8 @@ to Kubernetes: Download the game from these locations: -* [MacOS](https://github.com/lucky-sideburn/KubeInvaders/releases/download/chaos_node-2.0/x86_64-darwin.zip) -* [Linux](https://github.com/lucky-sideburn/KubeInvaders/releases/download/chaos_node-2.0/x86_64-linux.zip) - +* [MacOS](https://github.com/lucky-sideburn/KubeInvaders/releases/download/1.0-metrics/x86_64-darwin.zip) +* [Linux](https://github.com/lucky-sideburn/KubeInvaders/releases/download/1.0-metrics/x86_64-linux.zip) ### Run directly from Docker This method can be used for developing KubeInvaders and testing the HTML5 bundle. @@ -134,28 +157,6 @@ For clusters with many workers-nodes, KubeInvaders selects a subset of random it | Item | Max Number | |-----------|--------------| | Nodes | 15 | -## Metrics - -KubeInvaders exposes metrics for Prometheus through the standard endpoint /metrics - -Example of metrics - -```bash -curl localhost:8080/metrics - -# Total number of chaos jobs executed on the node workernode01 -chaos_jobs_node_count{node=workernode01} 100 - -# Total number of chaos jobs executed against all worker nodes -chaos_node_jobs_total 200 - -# Total number of deleted pods -deleted_pods_total 16 - -# Total number of deleted pods -deleted_namespace_pods_count{namespace=myawesomenamespace} 8 -``` - ## Configuration ### Environment Variables - Make the game more difficult to win! diff --git a/grafana.png b/grafana.png new file mode 100644 index 0000000..cb3786b Binary files /dev/null and b/grafana.png differ diff --git a/grafana/KubeInvadersDashboard.json b/grafana/KubeInvadersDashboard.json new file mode 100644 index 0000000..a1fd703 --- /dev/null +++ b/grafana/KubeInvadersDashboard.json @@ -0,0 +1,477 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 10, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 18, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 11, + "interval": "", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(increase(chaos_jobs_node_count[5m]))", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 3, + "legendFormat": "{{node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Chaos jobs", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:91", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:92", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": { + "align": null + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "values": false + } + }, + "pluginVersion": "7.0.1", + "targets": [ + { + "expr": "deleted_pods_total", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Killed pods", + "type": "stat" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 5 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "values": false + } + }, + "pluginVersion": "7.0.1", + "targets": [ + { + "expr": "chaos_node_jobs_total", + "format": "table", + "instant": true, + "interval": "", + "intervalFactor": 3, + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Chaos jobs against nodes", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 18, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 12, + "interval": "", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(increase(deleted_namespace_pods_count[5m]))", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 3, + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Killed pods", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:91", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:92", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 20, + "x": 0, + "y": 14 + }, + "id": 5, + "options": { + "displayMode": "gradient", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "min" + ], + "values": false + }, + "showUnfilled": false + }, + "pluginVersion": "7.0.1", + "targets": [ + { + "expr": "chaos_jobs_node_count", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{node}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Chaos jobs count", + "type": "bargauge" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 20, + "x": 0, + "y": 21 + }, + "id": 7, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "values": false + }, + "showUnfilled": false + }, + "pluginVersion": "7.0.1", + "targets": [ + { + "expr": "deleted_namespace_pods_count", + "instant": true, + "interval": "", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Killed pods count", + "type": "bargauge" + } + ], + "schemaVersion": 25, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "KubeInvaders Dashboard", + "uid": "_IC3ogYMz", + "version": 8 +} \ No newline at end of file diff --git a/grafana2.png b/grafana2.png new file mode 100644 index 0000000..2ce6b44 Binary files /dev/null and b/grafana2.png differ