Update mock data to use multiple receivers

Update config + regenerate all mock data
This commit is contained in:
Łukasz Mierzwa
2017-06-03 13:22:41 -07:00
parent 8a176d5c36
commit a15cbf4e86
26 changed files with 3124 additions and 659 deletions
+326 -57
View File
@@ -10,7 +10,7 @@
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
@@ -18,7 +18,276 @@
"instance": "server5",
"job": "node_exporter"
},
"startsAt": "2017-05-01T20:44:55.763656584Z"
"startsAt": "2017-06-03T01:41:07.542266559Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"labels": {
"alertname": "Free_Disk_Space_Too_Low"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"help": "Example help annotation",
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web1",
"job": "node_exporter"
},
"silenced": 1,
"startsAt": "2017-06-03T01:41:07.542266559Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web2",
"job": "node_exporter"
},
"startsAt": "2017-06-03T01:41:07.542266559Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"labels": {
"alertname": "HTTP_Probe_Failed"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server4",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:41:07.542266559Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server5",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:41:07.542266559Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server6",
"job": "node_ping"
},
"silenced": 2,
"startsAt": "2017-06-03T01:41:07.542266559Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server7",
"job": "node_ping"
},
"silenced": 2,
"startsAt": "2017-06-03T01:41:07.542266559Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server8",
"job": "node_ping"
},
"silenced": 2,
"startsAt": "2017-06-03T01:41:07.542266559Z"
},
{
"annotations": {
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server1",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:41:07.542266559Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server2",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:41:07.542266559Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server3",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:41:07.542266559Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"labels": {
"alertname": "Host_Down"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"alert": "Memory usage exceeding threshold",
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Memory_Usage_Too_High",
"cluster": "prod",
"instance": "server2",
"job": "node_exporter"
},
"startsAt": "2017-06-03T01:41:07.542266559Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"labels": {
"alertname": "Memory_Usage_Too_High"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"alert": "Less than 10% disk space is free",
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
"cluster": "staging",
"instance": "server5",
"job": "node_exporter"
},
"startsAt": "2017-06-03T01:41:07.542266559Z"
}
],
"routeOpts": {
@@ -29,7 +298,7 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -50,7 +319,7 @@
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
@@ -59,14 +328,14 @@
"job": "node_exporter"
},
"silenced": 1,
"startsAt": "2017-05-01T20:44:55.763656584Z"
"startsAt": "2017-06-03T01:41:07.542266559Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
@@ -74,7 +343,7 @@
"instance": "web2",
"job": "node_exporter"
},
"startsAt": "2017-05-01T20:44:55.763656584Z"
"startsAt": "2017-06-03T01:41:07.542266559Z"
}
],
"routeOpts": {
@@ -85,7 +354,7 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -104,7 +373,7 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -113,14 +382,14 @@
"job": "node_ping"
},
"silenced": 2,
"startsAt": "2017-05-01T20:44:55.763656584Z"
"startsAt": "2017-06-03T01:41:07.542266559Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -129,14 +398,14 @@
"job": "node_ping"
},
"silenced": 2,
"startsAt": "2017-05-01T20:44:55.763656584Z"
"startsAt": "2017-06-03T01:41:07.542266559Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -145,7 +414,7 @@
"job": "node_ping"
},
"silenced": 2,
"startsAt": "2017-05-01T20:44:55.763656584Z"
"startsAt": "2017-06-03T01:41:07.542266559Z"
}
],
"routeOpts": {
@@ -156,7 +425,7 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -170,28 +439,12 @@
"blocks": [
{
"alerts": [
{
"annotations": {
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server1",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:44:55.763656584Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -199,7 +452,23 @@
"instance": "server2",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:44:55.763656584Z"
"startsAt": "2017-06-03T01:41:07.542266559Z"
},
{
"annotations": {
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server1",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:41:07.542266559Z"
}
],
"routeOpts": {
@@ -210,7 +479,7 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -229,22 +498,7 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server5",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:44:55.763656584Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -252,14 +506,14 @@
"instance": "server3",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:44:55.763656584Z"
"startsAt": "2017-06-03T01:41:07.542266559Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -267,7 +521,22 @@
"instance": "server4",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:44:55.763656584Z"
"startsAt": "2017-06-03T01:41:07.542266559Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server5",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:41:07.542266559Z"
}
],
"routeOpts": {
@@ -278,7 +547,7 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -298,7 +567,7 @@
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Memory_Usage_Too_High",
@@ -306,7 +575,7 @@
"instance": "server2",
"job": "node_exporter"
},
"startsAt": "2017-05-01T20:44:55.763656584Z"
"startsAt": "2017-06-03T01:41:07.542266559Z"
}
],
"routeOpts": {
@@ -317,7 +586,7 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
+2 -2
View File
@@ -3,7 +3,7 @@
"silences": [
{
"comment": "Silenced instance",
"createdAt": "2017-05-01T20:44:55.717083559Z",
"createdAt": "2017-06-03T01:41:07.511285232Z",
"createdBy": "john@example.com",
"endsAt": "2063-01-01T00:00:00Z",
"id": 1,
@@ -18,7 +18,7 @@
},
{
"comment": "Silenced Host_Down alerts in the dev cluster",
"createdAt": "2017-05-01T20:44:55.739048756Z",
"createdAt": "2017-06-03T01:41:07.526830484Z",
"createdBy": "john@example.com",
"endsAt": "2063-01-01T00:00:00Z",
"id": 2,
+34 -6
View File
@@ -1,6 +1,6 @@
{
"data": {
"config": "route:\n group_by: ['alertname', 'cluster', 'service']\n group_wait: 15s\n group_interval: 35s\n repeat_interval: 999h\n receiver: default\n\ninhibit_rules:\n- source_match:\n severity: 'critical'\n target_match:\n severity: 'warning'\n # Apply inhibition if the alertname is the same.\n equal: ['alertname', 'cluster', 'service']\n\nreceivers:\n- name: 'default'\n",
"config": "route:\n group_by: ['alertname']\n group_wait: 15s\n group_interval: 35s\n repeat_interval: 999h\n receiver: 'default'\n routes:\n - receiver: 'by-cluster-service'\n group_by: ['alertname', 'cluster', 'service']\n match_re:\n alertname: .*\n continue: true\n - receiver: 'by-name'\n group_by: [alertname]\n match_re:\n alertname: .*\n continue: true\n\ninhibit_rules:\n - source_match:\n severity: 'critical'\n target_match:\n severity: 'warning'\n # Apply inhibition if the alertname is the same.\n equal: ['alertname', 'cluster', 'service']\n\nreceivers:\n - name: 'default'\n - name: 'by-cluster-service'\n - name: 'by-name'\n",
"configJSON": {
"global": {
"hipchat_auth_token": "",
@@ -36,22 +36,50 @@
"receivers": [
{
"name": "default"
},
{
"name": "by-cluster-service"
},
{
"name": "by-name"
}
],
"route": {
"group_by": [
"alertname",
"cluster",
"service"
"alertname"
],
"group_interval": 35000000000,
"group_wait": 15000000000,
"receiver": "default",
"repeat_interval": 3596400000000000
"repeat_interval": 3596400000000000,
"routes": [
{
"continue": true,
"group_by": [
"alertname",
"cluster",
"service"
],
"match_re": {
"alertname": {}
},
"receiver": "by-cluster-service"
},
{
"continue": true,
"group_by": [
"alertname"
],
"match_re": {
"alertname": {}
},
"receiver": "by-name"
}
]
},
"templates": null
},
"uptime": "2017-05-01T20:44:45.763122163Z",
"uptime": "2017-06-03T01:40:52.406481362Z",
"versionInfo": {
"branch": "master",
"buildDate": "20160823-12:57:02",
+318 -49
View File
@@ -10,7 +10,7 @@
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
@@ -18,18 +18,287 @@
"instance": "server5",
"job": "node_exporter"
},
"startsAt": "2017-05-01T20:45:58.865330456Z"
"startsAt": "2017-06-03T01:42:15.892691108Z"
}
],
"routeOpts": {
"groupBy": [
"alertname",
"cluster",
"service"
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"labels": {
"alertname": "Free_Disk_Space_Too_Low"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"help": "Example help annotation",
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web1",
"job": "node_exporter"
},
"silenced": 1,
"startsAt": "2017-06-03T01:42:15.892691108Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web2",
"job": "node_exporter"
},
"startsAt": "2017-06-03T01:42:15.892691108Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"labels": {
"alertname": "HTTP_Probe_Failed"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server1",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:42:15.892691108Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server2",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:42:15.892691108Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server3",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:42:15.892691108Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server4",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:42:15.892691108Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server5",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:42:15.892691108Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server6",
"job": "node_ping"
},
"silenced": 2,
"startsAt": "2017-06-03T01:42:15.892691108Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server7",
"job": "node_ping"
},
"silenced": 2,
"startsAt": "2017-06-03T01:42:15.892691108Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server8",
"job": "node_ping"
},
"silenced": 2,
"startsAt": "2017-06-03T01:42:15.892691108Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"labels": {
"alertname": "Host_Down"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"alert": "Memory usage exceeding threshold",
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Memory_Usage_Too_High",
"cluster": "prod",
"instance": "server2",
"job": "node_exporter"
},
"startsAt": "2017-06-03T01:42:15.892691108Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"labels": {
"alertname": "Memory_Usage_Too_High"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"alert": "Less than 10% disk space is free",
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
"cluster": "staging",
"instance": "server5",
"job": "node_exporter"
},
"startsAt": "2017-06-03T01:42:15.892691108Z"
}
],
"routeOpts": {
"groupBy": [
"cluster",
"service",
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -50,7 +319,7 @@
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
@@ -59,14 +328,14 @@
"job": "node_exporter"
},
"silenced": 1,
"startsAt": "2017-05-01T20:45:58.865330456Z"
"startsAt": "2017-06-03T01:42:15.892691108Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
@@ -74,7 +343,7 @@
"instance": "web2",
"job": "node_exporter"
},
"startsAt": "2017-05-01T20:45:58.865330456Z"
"startsAt": "2017-06-03T01:42:15.892691108Z"
}
],
"routeOpts": {
@@ -85,7 +354,7 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -104,23 +373,7 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server8",
"job": "node_ping"
},
"silenced": 2,
"startsAt": "2017-05-01T20:45:58.865330456Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -129,14 +382,14 @@
"job": "node_ping"
},
"silenced": 2,
"startsAt": "2017-05-01T20:45:58.865330456Z"
"startsAt": "2017-06-03T01:42:15.892691108Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -145,7 +398,23 @@
"job": "node_ping"
},
"silenced": 2,
"startsAt": "2017-05-01T20:45:58.865330456Z"
"startsAt": "2017-06-03T01:42:15.892691108Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server8",
"job": "node_ping"
},
"silenced": 2,
"startsAt": "2017-06-03T01:42:15.892691108Z"
}
],
"routeOpts": {
@@ -156,7 +425,7 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -176,7 +445,7 @@
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -184,14 +453,14 @@
"instance": "server1",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:45:58.865330456Z"
"startsAt": "2017-06-03T01:42:15.892691108Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -199,7 +468,7 @@
"instance": "server2",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:45:58.865330456Z"
"startsAt": "2017-06-03T01:42:15.892691108Z"
}
],
"routeOpts": {
@@ -210,7 +479,7 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -229,7 +498,7 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -237,14 +506,14 @@
"instance": "server3",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:45:58.865330456Z"
"startsAt": "2017-06-03T01:42:15.892691108Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -252,14 +521,14 @@
"instance": "server4",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:45:58.865330456Z"
"startsAt": "2017-06-03T01:42:15.892691108Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -267,18 +536,18 @@
"instance": "server5",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:45:58.865330456Z"
"startsAt": "2017-06-03T01:42:15.892691108Z"
}
],
"routeOpts": {
"groupBy": [
"cluster",
"service",
"alertname"
"alertname",
"cluster"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -298,7 +567,7 @@
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Memory_Usage_Too_High",
@@ -306,7 +575,7 @@
"instance": "server2",
"job": "node_exporter"
},
"startsAt": "2017-05-01T20:45:58.865330456Z"
"startsAt": "2017-06-03T01:42:15.892691108Z"
}
],
"routeOpts": {
@@ -317,7 +586,7 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
+2 -2
View File
@@ -3,7 +3,7 @@
"silences": [
{
"comment": "Silenced instance",
"createdAt": "2017-05-01T20:45:58.802450694Z",
"createdAt": "2017-06-03T01:42:15.862959571Z",
"createdBy": "john@example.com",
"endsAt": "2063-01-01T00:00:00Z",
"id": 1,
@@ -18,7 +18,7 @@
},
{
"comment": "Silenced Host_Down alerts in the dev cluster",
"createdAt": "2017-05-01T20:45:58.82453438Z",
"createdAt": "2017-06-03T01:42:15.877866843Z",
"createdBy": "john@example.com",
"endsAt": "2063-01-01T00:00:00Z",
"id": 2,
+34 -6
View File
@@ -1,6 +1,6 @@
{
"data": {
"config": "route:\n group_by: ['alertname', 'cluster', 'service']\n group_wait: 15s\n group_interval: 35s\n repeat_interval: 999h\n receiver: default\n\ninhibit_rules:\n- source_match:\n severity: 'critical'\n target_match:\n severity: 'warning'\n # Apply inhibition if the alertname is the same.\n equal: ['alertname', 'cluster', 'service']\n\nreceivers:\n- name: 'default'\n",
"config": "route:\n group_by: ['alertname']\n group_wait: 15s\n group_interval: 35s\n repeat_interval: 999h\n receiver: 'default'\n routes:\n - receiver: 'by-cluster-service'\n group_by: ['alertname', 'cluster', 'service']\n match_re:\n alertname: .*\n continue: true\n - receiver: 'by-name'\n group_by: [alertname]\n match_re:\n alertname: .*\n continue: true\n\ninhibit_rules:\n - source_match:\n severity: 'critical'\n target_match:\n severity: 'warning'\n # Apply inhibition if the alertname is the same.\n equal: ['alertname', 'cluster', 'service']\n\nreceivers:\n - name: 'default'\n - name: 'by-cluster-service'\n - name: 'by-name'\n",
"configJSON": {
"global": {
"hipchat_auth_token": "",
@@ -36,22 +36,50 @@
"receivers": [
{
"name": "default"
},
{
"name": "by-cluster-service"
},
{
"name": "by-name"
}
],
"route": {
"group_by": [
"alertname",
"cluster",
"service"
"alertname"
],
"group_interval": 35000000000,
"group_wait": 15000000000,
"receiver": "default",
"repeat_interval": 3596400000000000
"repeat_interval": 3596400000000000,
"routes": [
{
"continue": true,
"group_by": [
"alertname",
"cluster",
"service"
],
"match_re": {
"alertname": {}
},
"receiver": "by-cluster-service"
},
{
"continue": true,
"group_by": [
"alertname"
],
"match_re": {
"alertname": {}
},
"receiver": "by-name"
}
]
},
"templates": null
},
"uptime": "2017-05-01T20:45:48.854261256Z",
"uptime": "2017-06-03T01:42:00.764247824Z",
"versionInfo": {
"branch": "master",
"buildDate": "20160831-22:29:03",
+332 -63
View File
@@ -10,7 +10,7 @@
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
@@ -18,18 +18,287 @@
"instance": "server5",
"job": "node_exporter"
},
"startsAt": "2017-05-01T20:47:02.216312801Z"
"startsAt": "2017-06-03T01:43:27.576074554Z"
}
],
"routeOpts": {
"groupBy": [
"service",
"alertname",
"cluster"
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"labels": {
"alertname": "Free_Disk_Space_Too_Low"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"help": "Example help annotation",
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web1",
"job": "node_exporter"
},
"silenced": 1,
"startsAt": "2017-06-03T01:43:27.576074554Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web2",
"job": "node_exporter"
},
"startsAt": "2017-06-03T01:43:27.576074554Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"labels": {
"alertname": "HTTP_Probe_Failed"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server1",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:43:27.576074554Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server2",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:43:27.576074554Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server3",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:43:27.576074554Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server4",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:43:27.576074554Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server5",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:43:27.576074554Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server6",
"job": "node_ping"
},
"silenced": 2,
"startsAt": "2017-06-03T01:43:27.576074554Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server7",
"job": "node_ping"
},
"silenced": 2,
"startsAt": "2017-06-03T01:43:27.576074554Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server8",
"job": "node_ping"
},
"silenced": 2,
"startsAt": "2017-06-03T01:43:27.576074554Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"labels": {
"alertname": "Host_Down"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"alert": "Memory usage exceeding threshold",
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Memory_Usage_Too_High",
"cluster": "prod",
"instance": "server2",
"job": "node_exporter"
},
"startsAt": "2017-06-03T01:43:27.576074554Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"labels": {
"alertname": "Memory_Usage_Too_High"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"alert": "Less than 10% disk space is free",
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
"cluster": "staging",
"instance": "server5",
"job": "node_exporter"
},
"startsAt": "2017-06-03T01:43:27.576074554Z"
}
],
"routeOpts": {
"groupBy": [
"alertname",
"cluster",
"service"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -50,7 +319,7 @@
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
@@ -59,14 +328,14 @@
"job": "node_exporter"
},
"silenced": 1,
"startsAt": "2017-05-01T20:47:02.216312801Z"
"startsAt": "2017-06-03T01:43:27.576074554Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
@@ -74,18 +343,18 @@
"instance": "web2",
"job": "node_exporter"
},
"startsAt": "2017-05-01T20:47:02.216312801Z"
"startsAt": "2017-06-03T01:43:27.576074554Z"
}
],
"routeOpts": {
"groupBy": [
"service",
"alertname",
"cluster"
"cluster",
"service"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -104,39 +373,7 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server7",
"job": "node_ping"
},
"silenced": 2,
"startsAt": "2017-05-01T20:47:02.216312801Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server8",
"job": "node_ping"
},
"silenced": 2,
"startsAt": "2017-05-01T20:47:02.216312801Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -145,7 +382,39 @@
"job": "node_ping"
},
"silenced": 2,
"startsAt": "2017-05-01T20:47:02.216312801Z"
"startsAt": "2017-06-03T01:43:27.576074554Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server7",
"job": "node_ping"
},
"silenced": 2,
"startsAt": "2017-06-03T01:43:27.576074554Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server8",
"job": "node_ping"
},
"silenced": 2,
"startsAt": "2017-06-03T01:43:27.576074554Z"
}
],
"routeOpts": {
@@ -156,7 +425,7 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -176,7 +445,7 @@
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -184,14 +453,14 @@
"instance": "server1",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:47:02.216312801Z"
"startsAt": "2017-06-03T01:43:27.576074554Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -199,7 +468,7 @@
"instance": "server2",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:47:02.216312801Z"
"startsAt": "2017-06-03T01:43:27.576074554Z"
}
],
"routeOpts": {
@@ -210,7 +479,7 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -229,7 +498,7 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -237,14 +506,14 @@
"instance": "server3",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:47:02.216312801Z"
"startsAt": "2017-06-03T01:43:27.576074554Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -252,14 +521,14 @@
"instance": "server4",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:47:02.216312801Z"
"startsAt": "2017-06-03T01:43:27.576074554Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -267,7 +536,7 @@
"instance": "server5",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:47:02.216312801Z"
"startsAt": "2017-06-03T01:43:27.576074554Z"
}
],
"routeOpts": {
@@ -278,7 +547,7 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -298,7 +567,7 @@
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Memory_Usage_Too_High",
@@ -306,7 +575,7 @@
"instance": "server2",
"job": "node_exporter"
},
"startsAt": "2017-05-01T20:47:02.216312801Z"
"startsAt": "2017-06-03T01:43:27.576074554Z"
}
],
"routeOpts": {
@@ -317,7 +586,7 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
+2 -2
View File
@@ -3,7 +3,7 @@
"silences": [
{
"comment": "Silenced instance",
"createdAt": "2017-05-01T20:47:01.957358662Z",
"createdAt": "2017-06-03T01:43:27.549256427Z",
"createdBy": "john@example.com",
"endsAt": "2063-01-01T00:00:00Z",
"id": 1,
@@ -18,7 +18,7 @@
},
{
"comment": "Silenced Host_Down alerts in the dev cluster",
"createdAt": "2017-05-01T20:47:02.193094888Z",
"createdAt": "2017-06-03T01:43:27.562645513Z",
"createdBy": "john@example.com",
"endsAt": "2063-01-01T00:00:00Z",
"id": 2,
+34 -6
View File
@@ -1,6 +1,6 @@
{
"data": {
"config": "route:\n group_by: ['alertname', 'cluster', 'service']\n group_wait: 15s\n group_interval: 35s\n repeat_interval: 999h\n receiver: default\n\ninhibit_rules:\n- source_match:\n severity: 'critical'\n target_match:\n severity: 'warning'\n # Apply inhibition if the alertname is the same.\n equal: ['alertname', 'cluster', 'service']\n\nreceivers:\n- name: 'default'\n",
"config": "route:\n group_by: ['alertname']\n group_wait: 15s\n group_interval: 35s\n repeat_interval: 999h\n receiver: 'default'\n routes:\n - receiver: 'by-cluster-service'\n group_by: ['alertname', 'cluster', 'service']\n match_re:\n alertname: .*\n continue: true\n - receiver: 'by-name'\n group_by: [alertname]\n match_re:\n alertname: .*\n continue: true\n\ninhibit_rules:\n - source_match:\n severity: 'critical'\n target_match:\n severity: 'warning'\n # Apply inhibition if the alertname is the same.\n equal: ['alertname', 'cluster', 'service']\n\nreceivers:\n - name: 'default'\n - name: 'by-cluster-service'\n - name: 'by-name'\n",
"configJSON": {
"global": {
"hipchat_auth_token": "",
@@ -36,22 +36,50 @@
"receivers": [
{
"name": "default"
},
{
"name": "by-cluster-service"
},
{
"name": "by-name"
}
],
"route": {
"group_by": [
"alertname",
"cluster",
"service"
"alertname"
],
"group_interval": 35000000000,
"group_wait": 15000000000,
"receiver": "default",
"repeat_interval": 3596400000000000
"repeat_interval": 3596400000000000,
"routes": [
{
"continue": true,
"group_by": [
"alertname",
"cluster",
"service"
],
"match_re": {
"alertname": {}
},
"receiver": "by-cluster-service"
},
{
"continue": true,
"group_by": [
"alertname"
],
"match_re": {
"alertname": {}
},
"receiver": "by-name"
}
]
},
"templates": null
},
"uptime": "2017-05-01T20:46:51.9830008Z",
"uptime": "2017-06-03T01:43:12.484328743Z",
"versionInfo": {
"branch": "master",
"buildDate": "20160902-15:33:13",
+335 -66
View File
@@ -10,7 +10,7 @@
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
@@ -18,7 +18,276 @@
"instance": "server5",
"job": "node_exporter"
},
"startsAt": "2017-05-01T20:48:07.504986978Z"
"startsAt": "2017-06-03T01:44:38.68282523Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"labels": {
"alertname": "Free_Disk_Space_Too_Low"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"help": "Example help annotation",
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web1",
"job": "node_exporter"
},
"silenced": "8f2b7daa-a051-4466-954b-5520e033c288",
"startsAt": "2017-06-03T01:44:38.68282523Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web2",
"job": "node_exporter"
},
"startsAt": "2017-06-03T01:44:38.68282523Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"labels": {
"alertname": "HTTP_Probe_Failed"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server1",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:44:38.68282523Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server2",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:44:38.68282523Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server3",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:44:38.68282523Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server4",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:44:38.68282523Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server5",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:44:38.68282523Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server6",
"job": "node_ping"
},
"silenced": "bf61a814-9668-4936-883b-29d15eb90dbc",
"startsAt": "2017-06-03T01:44:38.68282523Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server7",
"job": "node_ping"
},
"silenced": "bf61a814-9668-4936-883b-29d15eb90dbc",
"startsAt": "2017-06-03T01:44:38.68282523Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server8",
"job": "node_ping"
},
"silenced": "bf61a814-9668-4936-883b-29d15eb90dbc",
"startsAt": "2017-06-03T01:44:38.68282523Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"labels": {
"alertname": "Host_Down"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"alert": "Memory usage exceeding threshold",
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Memory_Usage_Too_High",
"cluster": "prod",
"instance": "server2",
"job": "node_exporter"
},
"startsAt": "2017-06-03T01:44:38.68282523Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"labels": {
"alertname": "Memory_Usage_Too_High"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"alert": "Less than 10% disk space is free",
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
"cluster": "staging",
"instance": "server5",
"job": "node_exporter"
},
"startsAt": "2017-06-03T01:44:38.68282523Z"
}
],
"routeOpts": {
@@ -29,7 +298,7 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -43,30 +312,12 @@
"blocks": [
{
"alerts": [
{
"annotations": {
"help": "Example help annotation",
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web1",
"job": "node_exporter"
},
"silenced": "a202a84a-7fc7-4340-9358-bcfdea507b38",
"startsAt": "2017-05-01T20:48:07.504986978Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
@@ -74,18 +325,36 @@
"instance": "web2",
"job": "node_exporter"
},
"startsAt": "2017-05-01T20:48:07.504986978Z"
"startsAt": "2017-06-03T01:44:38.68282523Z"
},
{
"annotations": {
"help": "Example help annotation",
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web1",
"job": "node_exporter"
},
"silenced": "8f2b7daa-a051-4466-954b-5520e033c288",
"startsAt": "2017-06-03T01:44:38.68282523Z"
}
],
"routeOpts": {
"groupBy": [
"alertname",
"cluster",
"service"
"service",
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -104,7 +373,7 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -112,15 +381,15 @@
"instance": "server6",
"job": "node_ping"
},
"silenced": "f5471ebc-43d8-402f-9c3f-ddd9e7f6d7c0",
"startsAt": "2017-05-01T20:48:07.504986978Z"
"silenced": "bf61a814-9668-4936-883b-29d15eb90dbc",
"startsAt": "2017-06-03T01:44:38.68282523Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -128,15 +397,15 @@
"instance": "server7",
"job": "node_ping"
},
"silenced": "f5471ebc-43d8-402f-9c3f-ddd9e7f6d7c0",
"startsAt": "2017-05-01T20:48:07.504986978Z"
"silenced": "bf61a814-9668-4936-883b-29d15eb90dbc",
"startsAt": "2017-06-03T01:44:38.68282523Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -144,8 +413,8 @@
"instance": "server8",
"job": "node_ping"
},
"silenced": "f5471ebc-43d8-402f-9c3f-ddd9e7f6d7c0",
"startsAt": "2017-05-01T20:48:07.504986978Z"
"silenced": "bf61a814-9668-4936-883b-29d15eb90dbc",
"startsAt": "2017-06-03T01:44:38.68282523Z"
}
],
"routeOpts": {
@@ -156,7 +425,7 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -176,7 +445,7 @@
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -184,14 +453,14 @@
"instance": "server1",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:48:07.504986978Z"
"startsAt": "2017-06-03T01:44:38.68282523Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -199,7 +468,7 @@
"instance": "server2",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:48:07.504986978Z"
"startsAt": "2017-06-03T01:44:38.68282523Z"
}
],
"routeOpts": {
@@ -210,7 +479,7 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -229,22 +498,7 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server4",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:48:07.504986978Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -252,14 +506,14 @@
"instance": "server5",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:48:07.504986978Z"
"startsAt": "2017-06-03T01:44:38.68282523Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -267,18 +521,33 @@
"instance": "server3",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:48:07.504986978Z"
"startsAt": "2017-06-03T01:44:38.68282523Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server4",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:44:38.68282523Z"
}
],
"routeOpts": {
"groupBy": [
"alertname",
"cluster",
"service",
"alertname"
"service"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -298,7 +567,7 @@
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Memory_Usage_Too_High",
@@ -306,7 +575,7 @@
"instance": "server2",
"job": "node_exporter"
},
"startsAt": "2017-05-01T20:48:07.504986978Z"
"startsAt": "2017-06-03T01:44:38.68282523Z"
}
],
"routeOpts": {
@@ -317,7 +586,7 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
+6 -6
View File
@@ -4,7 +4,7 @@
"comment": "Silenced instance",
"createdBy": "john@example.com",
"endsAt": "2063-01-01T00:00:00Z",
"id": "a202a84a-7fc7-4340-9358-bcfdea507b38",
"id": "8f2b7daa-a051-4466-954b-5520e033c288",
"matchers": [
{
"isRegex": false,
@@ -12,14 +12,14 @@
"value": "web1"
}
],
"startsAt": "2017-05-01T20:48:07.497691065Z",
"updatedAt": "2017-05-01T20:48:07.497691065Z"
"startsAt": "2017-06-03T01:44:38.673642596Z",
"updatedAt": "2017-06-03T01:44:38.673642596Z"
},
{
"comment": "Silenced Host_Down alerts in the dev cluster",
"createdBy": "john@example.com",
"endsAt": "2063-01-01T00:00:00Z",
"id": "f5471ebc-43d8-402f-9c3f-ddd9e7f6d7c0",
"id": "bf61a814-9668-4936-883b-29d15eb90dbc",
"matchers": [
{
"isRegex": false,
@@ -32,8 +32,8 @@
"value": "dev"
}
],
"startsAt": "2017-05-01T20:48:07.501342745Z",
"updatedAt": "2017-05-01T20:48:07.501342745Z"
"startsAt": "2017-06-03T01:44:38.677815485Z",
"updatedAt": "2017-06-03T01:44:38.677815485Z"
}
],
"status": "success"
+2 -2
View File
@@ -1,7 +1,7 @@
{
"data": {
"config": "route:\n group_by: ['alertname', 'cluster', 'service']\n group_wait: 15s\n group_interval: 35s\n repeat_interval: 999h\n receiver: default\n\ninhibit_rules:\n- source_match:\n severity: 'critical'\n target_match:\n severity: 'warning'\n # Apply inhibition if the alertname is the same.\n equal: ['alertname', 'cluster', 'service']\n\nreceivers:\n- name: 'default'\n",
"uptime": "2017-05-01T20:47:57.36073197Z",
"config": "route:\n group_by: ['alertname']\n group_wait: 15s\n group_interval: 35s\n repeat_interval: 999h\n receiver: 'default'\n routes:\n - receiver: 'by-cluster-service'\n group_by: ['alertname', 'cluster', 'service']\n match_re:\n alertname: .*\n continue: true\n - receiver: 'by-name'\n group_by: [alertname]\n match_re:\n alertname: .*\n continue: true\n\ninhibit_rules:\n - source_match:\n severity: 'critical'\n target_match:\n severity: 'warning'\n # Apply inhibition if the alertname is the same.\n equal: ['alertname', 'cluster', 'service']\n\nreceivers:\n - name: 'default'\n - name: 'by-cluster-service'\n - name: 'by-name'\n",
"uptime": "2017-06-03T01:44:23.52304472Z",
"versionInfo": {
"branch": "master",
"buildDate": "20161101-18:10:50",
+361 -92
View File
@@ -10,7 +10,7 @@
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
@@ -18,7 +18,276 @@
"instance": "server5",
"job": "node_exporter"
},
"startsAt": "2017-05-01T20:49:10.737246096Z"
"startsAt": "2017-06-03T01:45:47.493161609Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"labels": {
"alertname": "Free_Disk_Space_Too_Low"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"help": "Example help annotation",
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web1",
"job": "node_exporter"
},
"silenced": "aee02f72-4f11-492c-bc86-67a7d6d0c7a7",
"startsAt": "2017-06-03T01:45:47.493161609Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web2",
"job": "node_exporter"
},
"startsAt": "2017-06-03T01:45:47.493161609Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"labels": {
"alertname": "HTTP_Probe_Failed"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server6",
"job": "node_ping"
},
"silenced": "a7e42132-3843-4541-993a-7f383157c189",
"startsAt": "2017-06-03T01:45:47.493161609Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server7",
"job": "node_ping"
},
"silenced": "a7e42132-3843-4541-993a-7f383157c189",
"startsAt": "2017-06-03T01:45:47.493161609Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server8",
"job": "node_ping"
},
"silenced": "a7e42132-3843-4541-993a-7f383157c189",
"startsAt": "2017-06-03T01:45:47.493161609Z"
},
{
"annotations": {
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server1",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:45:47.493161609Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server2",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:45:47.493161609Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server3",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:45:47.493161609Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server4",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:45:47.493161609Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server5",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:45:47.493161609Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"labels": {
"alertname": "Host_Down"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"alert": "Memory usage exceeding threshold",
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Memory_Usage_Too_High",
"cluster": "prod",
"instance": "server2",
"job": "node_exporter"
},
"startsAt": "2017-06-03T01:45:47.493161609Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"labels": {
"alertname": "Memory_Usage_Too_High"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"alert": "Less than 10% disk space is free",
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
"cluster": "staging",
"instance": "server5",
"job": "node_exporter"
},
"startsAt": "2017-06-03T01:45:47.493161609Z"
}
],
"routeOpts": {
@@ -29,7 +298,7 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -50,7 +319,7 @@
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
@@ -58,15 +327,15 @@
"instance": "web1",
"job": "node_exporter"
},
"silenced": "a2e60dae-4166-4b14-be6d-62e9ca32153f",
"startsAt": "2017-05-01T20:49:10.737246096Z"
"silenced": "aee02f72-4f11-492c-bc86-67a7d6d0c7a7",
"startsAt": "2017-06-03T01:45:47.493161609Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
@@ -74,7 +343,7 @@
"instance": "web2",
"job": "node_exporter"
},
"startsAt": "2017-05-01T20:49:10.737246096Z"
"startsAt": "2017-06-03T01:45:47.493161609Z"
}
],
"routeOpts": {
@@ -85,7 +354,7 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -104,39 +373,7 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server6",
"job": "node_ping"
},
"silenced": "e64b0232-fd8c-427c-9174-f77390bd0bc4",
"startsAt": "2017-05-01T20:49:10.737246096Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server7",
"job": "node_ping"
},
"silenced": "e64b0232-fd8c-427c-9174-f77390bd0bc4",
"startsAt": "2017-05-01T20:49:10.737246096Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -144,8 +381,40 @@
"instance": "server8",
"job": "node_ping"
},
"silenced": "e64b0232-fd8c-427c-9174-f77390bd0bc4",
"startsAt": "2017-05-01T20:49:10.737246096Z"
"silenced": "a7e42132-3843-4541-993a-7f383157c189",
"startsAt": "2017-06-03T01:45:47.493161609Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server6",
"job": "node_ping"
},
"silenced": "a7e42132-3843-4541-993a-7f383157c189",
"startsAt": "2017-06-03T01:45:47.493161609Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server7",
"job": "node_ping"
},
"silenced": "a7e42132-3843-4541-993a-7f383157c189",
"startsAt": "2017-06-03T01:45:47.493161609Z"
}
],
"routeOpts": {
@@ -156,7 +425,7 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -176,7 +445,7 @@
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -184,14 +453,14 @@
"instance": "server1",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:49:10.737246096Z"
"startsAt": "2017-06-03T01:45:47.493161609Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -199,18 +468,18 @@
"instance": "server2",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:49:10.737246096Z"
"startsAt": "2017-06-03T01:45:47.493161609Z"
}
],
"routeOpts": {
"groupBy": [
"service",
"alertname",
"cluster",
"service"
"cluster"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -229,37 +498,7 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server3",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:49:10.737246096Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server4",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:49:10.737246096Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -267,18 +506,48 @@
"instance": "server5",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:49:10.737246096Z"
"startsAt": "2017-06-03T01:45:47.493161609Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server3",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:45:47.493161609Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server4",
"job": "node_ping"
},
"startsAt": "2017-06-03T01:45:47.493161609Z"
}
],
"routeOpts": {
"groupBy": [
"alertname",
"cluster",
"service",
"alertname"
"service"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
@@ -298,7 +567,7 @@
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Memory_Usage_Too_High",
@@ -306,18 +575,18 @@
"instance": "server2",
"job": "node_exporter"
},
"startsAt": "2017-05-01T20:49:10.737246096Z"
"startsAt": "2017-06-03T01:45:47.493161609Z"
}
],
"routeOpts": {
"groupBy": [
"service",
"alertname",
"cluster",
"service"
"cluster"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
+6 -6
View File
@@ -4,7 +4,7 @@
"comment": "Silenced instance",
"createdBy": "john@example.com",
"endsAt": "2063-01-01T00:00:00Z",
"id": "a2e60dae-4166-4b14-be6d-62e9ca32153f",
"id": "aee02f72-4f11-492c-bc86-67a7d6d0c7a7",
"matchers": [
{
"isRegex": false,
@@ -12,14 +12,14 @@
"value": "web1"
}
],
"startsAt": "2017-05-01T20:49:10.732910442Z",
"updatedAt": "2017-05-01T20:49:10.732910442Z"
"startsAt": "2017-06-03T01:45:47.483774606Z",
"updatedAt": "2017-06-03T01:45:47.483774606Z"
},
{
"comment": "Silenced Host_Down alerts in the dev cluster",
"createdBy": "john@example.com",
"endsAt": "2063-01-01T00:00:00Z",
"id": "e64b0232-fd8c-427c-9174-f77390bd0bc4",
"id": "a7e42132-3843-4541-993a-7f383157c189",
"matchers": [
{
"isRegex": false,
@@ -32,8 +32,8 @@
"value": "dev"
}
],
"startsAt": "2017-05-01T20:49:10.735083335Z",
"updatedAt": "2017-05-01T20:49:10.735083335Z"
"startsAt": "2017-06-03T01:45:47.48832377Z",
"updatedAt": "2017-06-03T01:45:47.48832377Z"
}
],
"status": "success"
+34 -6
View File
@@ -1,6 +1,6 @@
{
"data": {
"config": "route:\n group_by: ['alertname', 'cluster', 'service']\n group_wait: 15s\n group_interval: 35s\n repeat_interval: 999h\n receiver: default\n\ninhibit_rules:\n- source_match:\n severity: 'critical'\n target_match:\n severity: 'warning'\n # Apply inhibition if the alertname is the same.\n equal: ['alertname', 'cluster', 'service']\n\nreceivers:\n- name: 'default'\n",
"config": "route:\n group_by: ['alertname']\n group_wait: 15s\n group_interval: 35s\n repeat_interval: 999h\n receiver: 'default'\n routes:\n - receiver: 'by-cluster-service'\n group_by: ['alertname', 'cluster', 'service']\n match_re:\n alertname: .*\n continue: true\n - receiver: 'by-name'\n group_by: [alertname]\n match_re:\n alertname: .*\n continue: true\n\ninhibit_rules:\n - source_match:\n severity: 'critical'\n target_match:\n severity: 'warning'\n # Apply inhibition if the alertname is the same.\n equal: ['alertname', 'cluster', 'service']\n\nreceivers:\n - name: 'default'\n - name: 'by-cluster-service'\n - name: 'by-name'\n",
"configJSON": {
"global": {
"hipchat_auth_token": "",
@@ -38,22 +38,50 @@
"receivers": [
{
"name": "default"
},
{
"name": "by-cluster-service"
},
{
"name": "by-name"
}
],
"route": {
"group_by": [
"alertname",
"cluster",
"service"
"alertname"
],
"group_interval": 35000000000,
"group_wait": 15000000000,
"receiver": "default",
"repeat_interval": 3596400000000000
"repeat_interval": 3596400000000000,
"routes": [
{
"continue": true,
"group_by": [
"alertname",
"cluster",
"service"
],
"match_re": {
"alertname": {}
},
"receiver": "by-cluster-service"
},
{
"continue": true,
"group_by": [
"alertname"
],
"match_re": {
"alertname": {}
},
"receiver": "by-name"
}
]
},
"templates": null
},
"uptime": "2017-05-01T20:49:00.61677729Z",
"uptime": "2017-06-03T01:45:32.431568061Z",
"versionInfo": {
"branch": "master",
"buildDate": "20161125-08:14:40",
+344 -71
View File
@@ -10,7 +10,7 @@
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
@@ -18,23 +18,296 @@
"instance": "server5",
"job": "node_exporter"
},
"startsAt": "2017-05-01T20:50:13.961979914Z"
"startsAt": "2017-06-03T02:35:06.958786061Z"
}
],
"routeOpts": {
"groupBy": [
"service",
"alertname",
"cluster"
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}:{alertname=\"Free_Disk_Space_Too_Low\", cluster=\"staging\"}",
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Free_Disk_Space_Too_Low\"}",
"labels": {
"alertname": "Free_Disk_Space_Too_Low"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"help": "Example help annotation",
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web1",
"job": "node_exporter"
},
"silenced": "64479e61-817d-4e57-92bb-23be8ba07a93",
"startsAt": "2017-06-03T02:35:06.958786061Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web2",
"job": "node_exporter"
},
"startsAt": "2017-06-03T02:35:06.958786061Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"HTTP_Probe_Failed\"}",
"labels": {
"alertname": "HTTP_Probe_Failed"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server7",
"job": "node_ping"
},
"silenced": "50a7c149-a44b-4e97-ac98-c072c4475fe4",
"startsAt": "2017-06-03T02:35:06.958786061Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server8",
"job": "node_ping"
},
"silenced": "50a7c149-a44b-4e97-ac98-c072c4475fe4",
"startsAt": "2017-06-03T02:35:06.958786061Z"
},
{
"annotations": {
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server1",
"job": "node_ping"
},
"startsAt": "2017-06-03T02:35:06.958786061Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server2",
"job": "node_ping"
},
"startsAt": "2017-06-03T02:35:06.958786061Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server3",
"job": "node_ping"
},
"startsAt": "2017-06-03T02:35:06.958786061Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server4",
"job": "node_ping"
},
"startsAt": "2017-06-03T02:35:06.958786061Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server5",
"job": "node_ping"
},
"startsAt": "2017-06-03T02:35:06.958786061Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server6",
"job": "node_ping"
},
"silenced": "50a7c149-a44b-4e97-ac98-c072c4475fe4",
"startsAt": "2017-06-03T02:35:06.958786061Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Host_Down\"}",
"labels": {
"alertname": "Host_Down"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"alert": "Memory usage exceeding threshold",
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Memory_Usage_Too_High",
"cluster": "prod",
"instance": "server2",
"job": "node_exporter"
},
"startsAt": "2017-06-03T02:35:06.958786061Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Memory_Usage_Too_High\"}",
"labels": {
"alertname": "Memory_Usage_Too_High"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"alert": "Less than 10% disk space is free",
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
"cluster": "staging",
"instance": "server5",
"job": "node_exporter"
},
"startsAt": "2017-06-03T02:35:06.958786061Z"
}
],
"routeOpts": {
"groupBy": [
"alertname",
"cluster",
"service"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Free_Disk_Space_Too_Low\", cluster=\"staging\"}",
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
"cluster": "staging"
@@ -51,7 +324,7 @@
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
@@ -59,15 +332,15 @@
"instance": "web1",
"job": "node_exporter"
},
"silenced": "4b1c588a-c40f-4ec8-9602-754637ad315a",
"startsAt": "2017-05-01T20:50:13.961979914Z"
"silenced": "64479e61-817d-4e57-92bb-23be8ba07a93",
"startsAt": "2017-06-03T02:35:06.958786061Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "HTTP_Probe_Failed",
@@ -75,7 +348,7 @@
"instance": "web2",
"job": "node_exporter"
},
"startsAt": "2017-05-01T20:50:13.961979914Z"
"startsAt": "2017-06-03T02:35:06.958786061Z"
}
],
"routeOpts": {
@@ -86,12 +359,12 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}:{alertname=\"HTTP_Probe_Failed\", cluster=\"dev\"}",
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"HTTP_Probe_Failed\", cluster=\"dev\"}",
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev"
@@ -106,23 +379,7 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server6",
"job": "node_ping"
},
"silenced": "c4ffbd39-92fa-40b4-9ba1-5d19df3a410d",
"startsAt": "2017-05-01T20:50:13.961979914Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -130,15 +387,15 @@
"instance": "server7",
"job": "node_ping"
},
"silenced": "c4ffbd39-92fa-40b4-9ba1-5d19df3a410d",
"startsAt": "2017-05-01T20:50:13.961979914Z"
"silenced": "50a7c149-a44b-4e97-ac98-c072c4475fe4",
"startsAt": "2017-06-03T02:35:06.958786061Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -146,8 +403,24 @@
"instance": "server8",
"job": "node_ping"
},
"silenced": "c4ffbd39-92fa-40b4-9ba1-5d19df3a410d",
"startsAt": "2017-05-01T20:50:13.961979914Z"
"silenced": "50a7c149-a44b-4e97-ac98-c072c4475fe4",
"startsAt": "2017-06-03T02:35:06.958786061Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server6",
"job": "node_ping"
},
"silenced": "50a7c149-a44b-4e97-ac98-c072c4475fe4",
"startsAt": "2017-06-03T02:35:06.958786061Z"
}
],
"routeOpts": {
@@ -158,12 +431,12 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}:{alertname=\"Host_Down\", cluster=\"dev\"}",
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Host_Down\", cluster=\"dev\"}",
"labels": {
"alertname": "Host_Down",
"cluster": "dev"
@@ -179,7 +452,7 @@
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -187,14 +460,14 @@
"instance": "server1",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:50:13.961979914Z"
"startsAt": "2017-06-03T02:35:06.958786061Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -202,7 +475,7 @@
"instance": "server2",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:50:13.961979914Z"
"startsAt": "2017-06-03T02:35:06.958786061Z"
}
],
"routeOpts": {
@@ -213,12 +486,12 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}:{alertname=\"Host_Down\", cluster=\"prod\"}",
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Host_Down\", cluster=\"prod\"}",
"labels": {
"alertname": "Host_Down",
"cluster": "prod"
@@ -233,22 +506,7 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server5",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:50:13.961979914Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -256,14 +514,14 @@
"instance": "server3",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:50:13.961979914Z"
"startsAt": "2017-06-03T02:35:06.958786061Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
@@ -271,23 +529,38 @@
"instance": "server4",
"job": "node_ping"
},
"startsAt": "2017-05-01T20:50:13.961979914Z"
"startsAt": "2017-06-03T02:35:06.958786061Z"
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server5",
"job": "node_ping"
},
"startsAt": "2017-06-03T02:35:06.958786061Z"
}
],
"routeOpts": {
"groupBy": [
"alertname",
"cluster",
"service"
"service",
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}:{alertname=\"Host_Down\", cluster=\"staging\"}",
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Host_Down\", cluster=\"staging\"}",
"labels": {
"alertname": "Host_Down",
"cluster": "staging"
@@ -303,7 +576,7 @@
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibited": false,
"labels": {
"alertname": "Memory_Usage_Too_High",
@@ -311,7 +584,7 @@
"instance": "server2",
"job": "node_exporter"
},
"startsAt": "2017-05-01T20:50:13.961979914Z"
"startsAt": "2017-06-03T02:35:06.958786061Z"
}
],
"routeOpts": {
@@ -322,12 +595,12 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}:{alertname=\"Memory_Usage_Too_High\", cluster=\"prod\"}",
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Memory_Usage_Too_High\", cluster=\"prod\"}",
"labels": {
"alertname": "Memory_Usage_Too_High",
"cluster": "prod"
+6 -6
View File
@@ -4,7 +4,7 @@
"comment": "Silenced instance",
"createdBy": "john@example.com",
"endsAt": "2063-01-01T00:00:00Z",
"id": "4b1c588a-c40f-4ec8-9602-754637ad315a",
"id": "64479e61-817d-4e57-92bb-23be8ba07a93",
"matchers": [
{
"isRegex": false,
@@ -12,14 +12,14 @@
"value": "web1"
}
],
"startsAt": "2017-05-01T20:50:13.957724279Z",
"updatedAt": "2017-05-01T20:50:13.957724279Z"
"startsAt": "2017-06-03T01:47:01.941185995Z",
"updatedAt": "2017-06-03T01:47:01.941185995Z"
},
{
"comment": "Silenced Host_Down alerts in the dev cluster",
"createdBy": "john@example.com",
"endsAt": "2063-01-01T00:00:00Z",
"id": "c4ffbd39-92fa-40b4-9ba1-5d19df3a410d",
"id": "50a7c149-a44b-4e97-ac98-c072c4475fe4",
"matchers": [
{
"isRegex": false,
@@ -32,8 +32,8 @@
"value": "dev"
}
],
"startsAt": "2017-05-01T20:50:13.959964356Z",
"updatedAt": "2017-05-01T20:50:13.959964356Z"
"startsAt": "2017-06-03T01:47:01.945603009Z",
"updatedAt": "2017-06-03T01:47:01.945603009Z"
}
],
"status": "success"
+37 -9
View File
@@ -1,6 +1,6 @@
{
"data": {
"config": "route:\n group_by: ['alertname', 'cluster', 'service']\n group_wait: 15s\n group_interval: 35s\n repeat_interval: 999h\n receiver: default\n\ninhibit_rules:\n- source_match:\n severity: 'critical'\n target_match:\n severity: 'warning'\n # Apply inhibition if the alertname is the same.\n equal: ['alertname', 'cluster', 'service']\n\nreceivers:\n- name: 'default'\n",
"config": "route:\n group_by: ['alertname']\n group_wait: 15s\n group_interval: 35s\n repeat_interval: 999h\n receiver: 'default'\n routes:\n - receiver: 'by-cluster-service'\n group_by: ['alertname', 'cluster', 'service']\n match_re:\n alertname: .*\n continue: true\n - receiver: 'by-name'\n group_by: [alertname]\n match_re:\n alertname: .*\n continue: true\n\ninhibit_rules:\n - source_match:\n severity: 'critical'\n target_match:\n severity: 'warning'\n # Apply inhibition if the alertname is the same.\n equal: ['alertname', 'cluster', 'service']\n\nreceivers:\n - name: 'default'\n - name: 'by-cluster-service'\n - name: 'by-name'\n",
"configJSON": {
"global": {
"hipchat_auth_token": "",
@@ -38,33 +38,61 @@
"receivers": [
{
"name": "default"
},
{
"name": "by-cluster-service"
},
{
"name": "by-name"
}
],
"route": {
"group_by": [
"alertname",
"cluster",
"service"
"alertname"
],
"group_interval": 35000000000,
"group_wait": 15000000000,
"receiver": "default",
"repeat_interval": 3596400000000000
"repeat_interval": 3596400000000000,
"routes": [
{
"continue": true,
"group_by": [
"alertname",
"cluster",
"service"
],
"match_re": {
"alertname": {}
},
"receiver": "by-cluster-service"
},
{
"continue": true,
"group_by": [
"alertname"
],
"match_re": {
"alertname": {}
},
"receiver": "by-name"
}
]
},
"templates": null
},
"meshStatus": {
"name": "02:42:ac:11:00:02",
"nickName": "056ea9373951",
"nickName": "0598877b77eb",
"peers": [
{
"name": "02:42:ac:11:00:02",
"nickName": "056ea9373951",
"uid": 15872478154679362182
"nickName": "0598877b77eb",
"uid": 9276870122370880903
}
]
},
"uptime": "2017-05-01T20:50:03.840143699Z",
"uptime": "2017-06-03T01:46:46.697462943Z",
"versionInfo": {
"branch": "master",
"buildDate": "20170425-19:48:54",
+373 -72
View File
@@ -11,7 +11,7 @@
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibitedBy": [],
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
@@ -20,7 +20,308 @@
"job": "node_exporter"
},
"silencedBy": [],
"startsAt": "2017-05-01T20:59:51.364108486Z"
"startsAt": "2017-06-03T02:35:45.471451088Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Free_Disk_Space_Too_Low\"}",
"labels": {
"alertname": "Free_Disk_Space_Too_Low"
}
},
{
"blocks": [
{
"alerts": [
{
"Status": "suppressed",
"annotations": {
"help": "Example help annotation",
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibitedBy": null,
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web1",
"job": "node_exporter"
},
"silencedBy": [
"a34ec9ba-669f-4140-b066-655099c48c66"
],
"startsAt": "2017-06-03T02:35:45.471451088Z"
},
{
"Status": "active",
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibitedBy": [],
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web2",
"job": "node_exporter"
},
"silencedBy": [],
"startsAt": "2017-06-03T02:35:45.471451088Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"HTTP_Probe_Failed\"}",
"labels": {
"alertname": "HTTP_Probe_Failed"
}
},
{
"blocks": [
{
"alerts": [
{
"Status": "active",
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibitedBy": [],
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server5",
"job": "node_ping"
},
"silencedBy": [],
"startsAt": "2017-06-03T02:35:45.471451088Z"
},
{
"Status": "suppressed",
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibitedBy": null,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server6",
"job": "node_ping"
},
"silencedBy": [
"993d637b-92eb-43fa-ae1b-9a05856fd6dd"
],
"startsAt": "2017-06-03T02:35:45.471451088Z"
},
{
"Status": "suppressed",
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibitedBy": null,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server7",
"job": "node_ping"
},
"silencedBy": [
"993d637b-92eb-43fa-ae1b-9a05856fd6dd"
],
"startsAt": "2017-06-03T02:35:45.471451088Z"
},
{
"Status": "suppressed",
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibitedBy": null,
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server8",
"job": "node_ping"
},
"silencedBy": [
"993d637b-92eb-43fa-ae1b-9a05856fd6dd"
],
"startsAt": "2017-06-03T02:35:45.471451088Z"
},
{
"Status": "active",
"annotations": {
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibitedBy": [],
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server1",
"job": "node_ping"
},
"silencedBy": [],
"startsAt": "2017-06-03T02:35:45.471451088Z"
},
{
"Status": "active",
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibitedBy": [],
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server2",
"job": "node_ping"
},
"silencedBy": [],
"startsAt": "2017-06-03T02:35:45.471451088Z"
},
{
"Status": "active",
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibitedBy": [],
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server3",
"job": "node_ping"
},
"silencedBy": [],
"startsAt": "2017-06-03T02:35:45.471451088Z"
},
{
"Status": "active",
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibitedBy": [],
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server4",
"job": "node_ping"
},
"silencedBy": [],
"startsAt": "2017-06-03T02:35:45.471451088Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Host_Down\"}",
"labels": {
"alertname": "Host_Down"
}
},
{
"blocks": [
{
"alerts": [
{
"Status": "active",
"annotations": {
"alert": "Memory usage exceeding threshold",
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibitedBy": [],
"labels": {
"alertname": "Memory_Usage_Too_High",
"cluster": "prod",
"instance": "server2",
"job": "node_exporter"
},
"silencedBy": [],
"startsAt": "2017-06-03T02:35:45.471451088Z"
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Memory_Usage_Too_High\"}",
"labels": {
"alertname": "Memory_Usage_Too_High"
}
},
{
"blocks": [
{
"alerts": [
{
"Status": "active",
"annotations": {
"alert": "Less than 10% disk space is free",
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibitedBy": [],
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
"cluster": "staging",
"instance": "server5",
"job": "node_exporter"
},
"silencedBy": [],
"startsAt": "2017-06-03T02:35:45.471451088Z"
}
],
"routeOpts": {
@@ -31,12 +332,12 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}:{alertname=\"Free_Disk_Space_Too_Low\", cluster=\"staging\"}",
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Free_Disk_Space_Too_Low\", cluster=\"staging\"}",
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
"cluster": "staging"
@@ -54,7 +355,7 @@
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibitedBy": null,
"labels": {
"alertname": "HTTP_Probe_Failed",
@@ -63,9 +364,9 @@
"job": "node_exporter"
},
"silencedBy": [
"d68b4fd6-f18a-474a-8953-255a86250742"
"a34ec9ba-669f-4140-b066-655099c48c66"
],
"startsAt": "2017-05-01T20:59:51.364108486Z"
"startsAt": "2017-06-03T02:35:45.471451088Z"
},
{
"Status": "active",
@@ -73,7 +374,7 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibitedBy": [],
"labels": {
"alertname": "HTTP_Probe_Failed",
@@ -82,7 +383,7 @@
"job": "node_exporter"
},
"silencedBy": [],
"startsAt": "2017-05-01T20:59:51.364108486Z"
"startsAt": "2017-06-03T02:35:45.471451088Z"
}
],
"routeOpts": {
@@ -93,12 +394,12 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}:{alertname=\"HTTP_Probe_Failed\", cluster=\"dev\"}",
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"HTTP_Probe_Failed\", cluster=\"dev\"}",
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev"
@@ -114,7 +415,7 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibitedBy": null,
"labels": {
"alertname": "Host_Down",
@@ -123,9 +424,9 @@
"job": "node_ping"
},
"silencedBy": [
"89142af0-b17f-4ffd-b2c2-2398de624466"
"993d637b-92eb-43fa-ae1b-9a05856fd6dd"
],
"startsAt": "2017-05-01T20:59:51.364108486Z"
"startsAt": "2017-06-03T02:35:45.471451088Z"
},
{
"Status": "suppressed",
@@ -133,7 +434,7 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibitedBy": null,
"labels": {
"alertname": "Host_Down",
@@ -142,9 +443,9 @@
"job": "node_ping"
},
"silencedBy": [
"89142af0-b17f-4ffd-b2c2-2398de624466"
"993d637b-92eb-43fa-ae1b-9a05856fd6dd"
],
"startsAt": "2017-05-01T20:59:51.364108486Z"
"startsAt": "2017-06-03T02:35:45.471451088Z"
},
{
"Status": "suppressed",
@@ -152,7 +453,7 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibitedBy": null,
"labels": {
"alertname": "Host_Down",
@@ -161,9 +462,9 @@
"job": "node_ping"
},
"silencedBy": [
"89142af0-b17f-4ffd-b2c2-2398de624466"
"993d637b-92eb-43fa-ae1b-9a05856fd6dd"
],
"startsAt": "2017-05-01T20:59:51.364108486Z"
"startsAt": "2017-06-03T02:35:45.471451088Z"
}
],
"routeOpts": {
@@ -174,12 +475,12 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}:{alertname=\"Host_Down\", cluster=\"dev\"}",
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Host_Down\", cluster=\"dev\"}",
"labels": {
"alertname": "Host_Down",
"cluster": "dev"
@@ -196,7 +497,7 @@
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibitedBy": [],
"labels": {
"alertname": "Host_Down",
@@ -205,7 +506,7 @@
"job": "node_ping"
},
"silencedBy": [],
"startsAt": "2017-05-01T20:59:51.364108486Z"
"startsAt": "2017-06-03T02:35:45.471451088Z"
},
{
"Status": "active",
@@ -213,7 +514,7 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibitedBy": [],
"labels": {
"alertname": "Host_Down",
@@ -222,7 +523,7 @@
"job": "node_ping"
},
"silencedBy": [],
"startsAt": "2017-05-01T20:59:51.364108486Z"
"startsAt": "2017-06-03T02:35:45.471451088Z"
}
],
"routeOpts": {
@@ -233,12 +534,12 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}:{alertname=\"Host_Down\", cluster=\"prod\"}",
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Host_Down\", cluster=\"prod\"}",
"labels": {
"alertname": "Host_Down",
"cluster": "prod"
@@ -254,41 +555,7 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"inhibitedBy": [],
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server3",
"job": "node_ping"
},
"silencedBy": [],
"startsAt": "2017-05-01T20:59:51.364108486Z"
},
{
"Status": "active",
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"inhibitedBy": [],
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server4",
"job": "node_ping"
},
"silencedBy": [],
"startsAt": "2017-05-01T20:59:51.364108486Z"
},
{
"Status": "active",
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibitedBy": [],
"labels": {
"alertname": "Host_Down",
@@ -297,7 +564,41 @@
"job": "node_ping"
},
"silencedBy": [],
"startsAt": "2017-05-01T20:59:51.364108486Z"
"startsAt": "2017-06-03T02:35:45.471451088Z"
},
{
"Status": "active",
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibitedBy": [],
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server3",
"job": "node_ping"
},
"silencedBy": [],
"startsAt": "2017-06-03T02:35:45.471451088Z"
},
{
"Status": "active",
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"inhibitedBy": [],
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server4",
"job": "node_ping"
},
"silencedBy": [],
"startsAt": "2017-06-03T02:35:45.471451088Z"
}
],
"routeOpts": {
@@ -308,12 +609,12 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}:{alertname=\"Host_Down\", cluster=\"staging\"}",
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Host_Down\", cluster=\"staging\"}",
"labels": {
"alertname": "Host_Down",
"cluster": "staging"
@@ -330,7 +631,7 @@
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"inhibitedBy": [],
"labels": {
"alertname": "Memory_Usage_Too_High",
@@ -339,23 +640,23 @@
"job": "node_exporter"
},
"silencedBy": [],
"startsAt": "2017-05-01T20:59:51.364108486Z"
"startsAt": "2017-06-03T02:35:45.471451088Z"
}
],
"routeOpts": {
"groupBy": [
"service",
"alertname",
"cluster"
"cluster",
"service"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}:{alertname=\"Memory_Usage_Too_High\", cluster=\"prod\"}",
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Memory_Usage_Too_High\", cluster=\"prod\"}",
"labels": {
"alertname": "Memory_Usage_Too_High",
"cluster": "prod"
+18 -18
View File
@@ -1,25 +1,10 @@
{
"data": [
{
"comment": "Silenced instance",
"createdBy": "john@example.com",
"endsAt": "2063-01-01T00:00:00Z",
"id": "d68b4fd6-f18a-474a-8953-255a86250742",
"matchers": [
{
"isRegex": false,
"name": "instance",
"value": "web1"
}
],
"startsAt": "2017-05-01T20:59:51.359703208Z",
"updatedAt": "2017-05-01T20:59:51.359703208Z"
},
{
"comment": "Silenced Host_Down alerts in the dev cluster",
"createdBy": "john@example.com",
"endsAt": "2063-01-01T00:00:00Z",
"id": "89142af0-b17f-4ffd-b2c2-2398de624466",
"id": "993d637b-92eb-43fa-ae1b-9a05856fd6dd",
"matchers": [
{
"isRegex": false,
@@ -32,8 +17,23 @@
"value": "dev"
}
],
"startsAt": "2017-05-01T20:59:51.361896025Z",
"updatedAt": "2017-05-01T20:59:51.361896025Z"
"startsAt": "2017-06-03T02:35:45.466011616Z",
"updatedAt": "2017-06-03T02:35:45.466011616Z"
},
{
"comment": "Silenced instance",
"createdBy": "john@example.com",
"endsAt": "2063-01-01T00:00:00Z",
"id": "a34ec9ba-669f-4140-b066-655099c48c66",
"matchers": [
{
"isRegex": false,
"name": "instance",
"value": "web1"
}
],
"startsAt": "2017-06-03T02:35:45.461523072Z",
"updatedAt": "2017-06-03T02:35:45.461523072Z"
}
],
"status": "success"
+37 -9
View File
@@ -1,6 +1,6 @@
{
"data": {
"config": "route:\n group_by: ['alertname', 'cluster', 'service']\n group_wait: 15s\n group_interval: 35s\n repeat_interval: 999h\n receiver: default\n\ninhibit_rules:\n- source_match:\n severity: 'critical'\n target_match:\n severity: 'warning'\n # Apply inhibition if the alertname is the same.\n equal: ['alertname', 'cluster', 'service']\n\nreceivers:\n- name: 'default'\n",
"config": "route:\n group_by: ['alertname']\n group_wait: 15s\n group_interval: 35s\n repeat_interval: 999h\n receiver: 'default'\n routes:\n - receiver: 'by-cluster-service'\n group_by: ['alertname', 'cluster', 'service']\n match_re:\n alertname: .*\n continue: true\n - receiver: 'by-name'\n group_by: [alertname]\n match_re:\n alertname: .*\n continue: true\n\ninhibit_rules:\n - source_match:\n severity: 'critical'\n target_match:\n severity: 'warning'\n # Apply inhibition if the alertname is the same.\n equal: ['alertname', 'cluster', 'service']\n\nreceivers:\n - name: 'default'\n - name: 'by-cluster-service'\n - name: 'by-name'\n",
"configJSON": {
"global": {
"hipchat_auth_token": "",
@@ -38,33 +38,61 @@
"receivers": [
{
"name": "default"
},
{
"name": "by-cluster-service"
},
{
"name": "by-name"
}
],
"route": {
"group_by": [
"alertname",
"cluster",
"service"
"alertname"
],
"group_interval": 35000000000,
"group_wait": 15000000000,
"receiver": "default",
"repeat_interval": 3596400000000000
"repeat_interval": 3596400000000000,
"routes": [
{
"continue": true,
"group_by": [
"alertname",
"cluster",
"service"
],
"match_re": {
"alertname": {}
},
"receiver": "by-cluster-service"
},
{
"continue": true,
"group_by": [
"alertname"
],
"match_re": {
"alertname": {}
},
"receiver": "by-name"
}
]
},
"templates": null
},
"meshStatus": {
"name": "02:42:ac:11:00:02",
"nickName": "c8a6f2549aee",
"nickName": "4b7e0113875f",
"peers": [
{
"name": "02:42:ac:11:00:02",
"nickName": "c8a6f2549aee",
"uid": 5746289647142050162
"nickName": "4b7e0113875f",
"uid": 5843517098830853602
}
]
},
"uptime": "2017-05-01T20:59:36.222911307Z",
"uptime": "2017-06-03T02:35:30.289277165Z",
"versionInfo": {
"branch": "master",
"buildDate": "20170428-10:15:22",
+392 -67
View File
@@ -10,14 +10,339 @@
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
"cluster": "staging",
"instance": "server5",
"job": "node_exporter"
},
"startsAt": "2017-05-09T20:45:54.84503502Z",
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": [],
"silencedBy": [],
"state": "active"
}
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Free_Disk_Space_Too_Low\"}",
"labels": {
"alertname": "Free_Disk_Space_Too_Low"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"help": "Example help annotation",
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web1",
"job": "node_exporter"
},
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": null,
"silencedBy": [
"72abd13d-ce8f-4969-824e-4bf0be564fd9"
],
"state": "suppressed"
}
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web2",
"job": "node_exporter"
},
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": [],
"silencedBy": [],
"state": "active"
}
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"HTTP_Probe_Failed\"}",
"labels": {
"alertname": "HTTP_Probe_Failed"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server1",
"job": "node_ping"
},
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": [],
"silencedBy": [],
"state": "active"
}
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server2",
"job": "node_ping"
},
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": [],
"silencedBy": [],
"state": "active"
}
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server3",
"job": "node_ping"
},
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": [],
"silencedBy": [],
"state": "active"
}
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server4",
"job": "node_ping"
},
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": [],
"silencedBy": [],
"state": "active"
}
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server5",
"job": "node_ping"
},
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": [],
"silencedBy": [],
"state": "active"
}
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server6",
"job": "node_ping"
},
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": null,
"silencedBy": [
"366a415b-5dae-468b-ba5c-6ae3593d2339"
],
"state": "suppressed"
}
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server7",
"job": "node_ping"
},
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": null,
"silencedBy": [
"366a415b-5dae-468b-ba5c-6ae3593d2339"
],
"state": "suppressed"
}
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server8",
"job": "node_ping"
},
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": null,
"silencedBy": [
"366a415b-5dae-468b-ba5c-6ae3593d2339"
],
"state": "suppressed"
}
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Host_Down\"}",
"labels": {
"alertname": "Host_Down"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"alert": "Memory usage exceeding threshold",
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "Memory_Usage_Too_High",
"cluster": "prod",
"instance": "server2",
"job": "node_exporter"
},
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": [],
"silencedBy": [],
"state": "active"
}
}
],
"routeOpts": {
"groupBy": [
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "by-name",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Memory_Usage_Too_High\"}",
"labels": {
"alertname": "Memory_Usage_Too_High"
}
},
{
"blocks": [
{
"alerts": [
{
"annotations": {
"alert": "Less than 10% disk space is free",
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
"cluster": "staging",
"instance": "server5",
"job": "node_exporter"
},
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": [],
"silencedBy": [],
@@ -33,12 +358,12 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}:{alertname=\"Free_Disk_Space_Too_Low\", cluster=\"staging\"}",
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Free_Disk_Space_Too_Low\", cluster=\"staging\"}",
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
"cluster": "staging"
@@ -55,18 +380,18 @@
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web1",
"job": "node_exporter"
},
"startsAt": "2017-05-09T20:45:54.84503502Z",
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": null,
"silencedBy": [
"6e1d1c04-8bd1-4aee-8c48-258d3b886f49"
"72abd13d-ce8f-4969-824e-4bf0be564fd9"
],
"state": "suppressed"
}
@@ -76,14 +401,14 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev",
"instance": "web2",
"job": "node_exporter"
},
"startsAt": "2017-05-09T20:45:54.84503502Z",
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": [],
"silencedBy": [],
@@ -99,12 +424,12 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}:{alertname=\"HTTP_Probe_Failed\", cluster=\"dev\"}",
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"HTTP_Probe_Failed\", cluster=\"dev\"}",
"labels": {
"alertname": "HTTP_Probe_Failed",
"cluster": "dev"
@@ -119,39 +444,18 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server6",
"job": "node_ping"
},
"startsAt": "2017-05-09T20:45:54.84503502Z",
"status": {
"inhibitedBy": null,
"silencedBy": [
"b7b316a1-9ca8-4b66-8369-83ba78984c46"
],
"state": "suppressed"
}
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server7",
"job": "node_ping"
},
"startsAt": "2017-05-09T20:45:54.84503502Z",
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": null,
"silencedBy": [
"b7b316a1-9ca8-4b66-8369-83ba78984c46"
"366a415b-5dae-468b-ba5c-6ae3593d2339"
],
"state": "suppressed"
}
@@ -161,18 +465,39 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server8",
"job": "node_ping"
},
"startsAt": "2017-05-09T20:45:54.84503502Z",
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": null,
"silencedBy": [
"b7b316a1-9ca8-4b66-8369-83ba78984c46"
"366a415b-5dae-468b-ba5c-6ae3593d2339"
],
"state": "suppressed"
}
},
{
"annotations": {
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "Host_Down",
"cluster": "dev",
"instance": "server6",
"job": "node_ping"
},
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": null,
"silencedBy": [
"366a415b-5dae-468b-ba5c-6ae3593d2339"
],
"state": "suppressed"
}
@@ -180,18 +505,18 @@
],
"routeOpts": {
"groupBy": [
"service",
"alertname",
"cluster"
"cluster",
"service"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}:{alertname=\"Host_Down\", cluster=\"dev\"}",
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Host_Down\", cluster=\"dev\"}",
"labels": {
"alertname": "Host_Down",
"cluster": "dev"
@@ -203,18 +528,17 @@
"alerts": [
{
"annotations": {
"summary": "Example summary",
"url": "http://localhost/example.html"
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server1",
"instance": "server2",
"job": "node_ping"
},
"startsAt": "2017-05-09T20:45:54.84503502Z",
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": [],
"silencedBy": [],
@@ -223,17 +547,18 @@
},
{
"annotations": {
"summary": "Example summary"
"summary": "Example summary",
"url": "http://localhost/example.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "Host_Down",
"cluster": "prod",
"instance": "server2",
"instance": "server1",
"job": "node_ping"
},
"startsAt": "2017-05-09T20:45:54.84503502Z",
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": [],
"silencedBy": [],
@@ -249,12 +574,12 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}:{alertname=\"Host_Down\", cluster=\"prod\"}",
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Host_Down\", cluster=\"prod\"}",
"labels": {
"alertname": "Host_Down",
"cluster": "prod"
@@ -269,14 +594,14 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server3",
"job": "node_ping"
},
"startsAt": "2017-05-09T20:45:54.84503502Z",
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": [],
"silencedBy": [],
@@ -288,14 +613,14 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server4",
"job": "node_ping"
},
"startsAt": "2017-05-09T20:45:54.84503502Z",
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": [],
"silencedBy": [],
@@ -307,14 +632,14 @@
"summary": "Example summary"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "Host_Down",
"cluster": "staging",
"instance": "server5",
"job": "node_ping"
},
"startsAt": "2017-05-09T20:45:54.84503502Z",
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": [],
"silencedBy": [],
@@ -330,12 +655,12 @@
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}:{alertname=\"Host_Down\", cluster=\"staging\"}",
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Host_Down\", cluster=\"staging\"}",
"labels": {
"alertname": "Host_Down",
"cluster": "staging"
@@ -351,14 +676,14 @@
"dashboard": "http://localhost/dashboard.html"
},
"endsAt": "0001-01-01T00:00:00Z",
"generatorURL": "",
"generatorURL": "localhost/prometheus",
"labels": {
"alertname": "Memory_Usage_Too_High",
"cluster": "prod",
"instance": "server2",
"job": "node_exporter"
},
"startsAt": "2017-05-09T20:45:54.84503502Z",
"startsAt": "2017-06-03T02:36:54.162719582Z",
"status": {
"inhibitedBy": [],
"silencedBy": [],
@@ -368,18 +693,18 @@
],
"routeOpts": {
"groupBy": [
"alertname",
"cluster",
"service"
"service",
"alertname"
],
"groupInterval": 35000000000,
"groupWait": 15000000000,
"receiver": "default",
"receiver": "by-cluster-service",
"repeatInterval": 3596400000000000
}
}
],
"groupKey": "{}:{alertname=\"Memory_Usage_Too_High\", cluster=\"prod\"}",
"groupKey": "{}/{alertname=~\"^(?:.*)$\"}:{alertname=\"Memory_Usage_Too_High\", cluster=\"prod\"}",
"labels": {
"alertname": "Memory_Usage_Too_High",
"cluster": "prod"
+6 -6
View File
@@ -4,7 +4,7 @@
"comment": "Silenced instance",
"createdBy": "john@example.com",
"endsAt": "2063-01-01T00:00:00Z",
"id": "6e1d1c04-8bd1-4aee-8c48-258d3b886f49",
"id": "72abd13d-ce8f-4969-824e-4bf0be564fd9",
"matchers": [
{
"isRegex": false,
@@ -12,14 +12,14 @@
"value": "web1"
}
],
"startsAt": "2017-05-09T20:45:54.83551337Z",
"updatedAt": "2017-05-09T20:45:54.83551337Z"
"startsAt": "2017-06-03T02:36:54.153521654Z",
"updatedAt": "2017-06-03T02:36:54.153521654Z"
},
{
"comment": "Silenced Host_Down alerts in the dev cluster",
"createdBy": "john@example.com",
"endsAt": "2063-01-01T00:00:00Z",
"id": "b7b316a1-9ca8-4b66-8369-83ba78984c46",
"id": "366a415b-5dae-468b-ba5c-6ae3593d2339",
"matchers": [
{
"isRegex": false,
@@ -32,8 +32,8 @@
"value": "dev"
}
],
"startsAt": "2017-05-09T20:45:54.838410296Z",
"updatedAt": "2017-05-09T20:45:54.838410296Z"
"startsAt": "2017-06-03T02:36:54.158015986Z",
"updatedAt": "2017-06-03T02:36:54.158015986Z"
}
],
"status": "success"
+37 -9
View File
@@ -1,6 +1,6 @@
{
"data": {
"config": "route:\n group_by: ['alertname', 'cluster', 'service']\n group_wait: 15s\n group_interval: 35s\n repeat_interval: 999h\n receiver: default\n\ninhibit_rules:\n- source_match:\n severity: 'critical'\n target_match:\n severity: 'warning'\n # Apply inhibition if the alertname is the same.\n equal: ['alertname', 'cluster', 'service']\n\nreceivers:\n- name: 'default'\n",
"config": "route:\n group_by: ['alertname']\n group_wait: 15s\n group_interval: 35s\n repeat_interval: 999h\n receiver: 'default'\n routes:\n - receiver: 'by-cluster-service'\n group_by: ['alertname', 'cluster', 'service']\n match_re:\n alertname: .*\n continue: true\n - receiver: 'by-name'\n group_by: [alertname]\n match_re:\n alertname: .*\n continue: true\n\ninhibit_rules:\n - source_match:\n severity: 'critical'\n target_match:\n severity: 'warning'\n # Apply inhibition if the alertname is the same.\n equal: ['alertname', 'cluster', 'service']\n\nreceivers:\n - name: 'default'\n - name: 'by-cluster-service'\n - name: 'by-name'\n",
"configJSON": {
"global": {
"hipchat_auth_token": "",
@@ -38,33 +38,61 @@
"receivers": [
{
"name": "default"
},
{
"name": "by-cluster-service"
},
{
"name": "by-name"
}
],
"route": {
"group_by": [
"alertname",
"cluster",
"service"
"alertname"
],
"group_interval": 35000000000,
"group_wait": 15000000000,
"receiver": "default",
"repeat_interval": 3596400000000000
"repeat_interval": 3596400000000000,
"routes": [
{
"continue": true,
"group_by": [
"alertname",
"cluster",
"service"
],
"match_re": {
"alertname": "^(?:.*)$"
},
"receiver": "by-cluster-service"
},
{
"continue": true,
"group_by": [
"alertname"
],
"match_re": {
"alertname": "^(?:.*)$"
},
"receiver": "by-name"
}
]
},
"templates": null
},
"meshStatus": {
"name": "02:42:ac:11:00:02",
"nickName": "16ffa16ef025",
"nickName": "09df03db50be",
"peers": [
{
"name": "02:42:ac:11:00:02",
"nickName": "16ffa16ef025",
"uid": 9016669185209747900
"nickName": "09df03db50be",
"uid": 7391608572669058056
}
]
},
"uptime": "2017-05-09T20:45:39.702335917Z",
"uptime": "2017-06-03T02:36:38.990772207Z",
"versionInfo": {
"branch": "master",
"buildDate": "20170509-08:56:14",
+22 -9
View File
@@ -1,17 +1,30 @@
route:
group_by: ['alertname', 'cluster', 'service']
group_by: ['alertname']
group_wait: 15s
group_interval: 35s
repeat_interval: 999h
receiver: default
receiver: 'default'
routes:
- receiver: 'by-cluster-service'
group_by: ['alertname', 'cluster', 'service']
match_re:
alertname: .*
continue: true
- receiver: 'by-name'
group_by: [alertname]
match_re:
alertname: .*
continue: true
inhibit_rules:
- source_match:
severity: 'critical'
target_match:
severity: 'warning'
# Apply inhibition if the alertname is the same.
equal: ['alertname', 'cluster', 'service']
- source_match:
severity: 'critical'
target_match:
severity: 'warning'
# Apply inhibition if the alertname is the same.
equal: ['alertname', 'cluster', 'service']
receivers:
- name: 'default'
- name: 'default'
- name: 'by-cluster-service'
- name: 'by-name'
+24 -12
View File
@@ -51,7 +51,8 @@ for i in xrange(0, 5):
"help": "Example help annotation",
"summary": "Example summary",
"url": "http://localhost/example.html"
}
},
"generatorURL": "localhost/prometheus"
}, {
"labels": {
"alertname": "HTTP_Probe_Failed",
@@ -61,7 +62,8 @@ for i in xrange(0, 5):
},
"annotations": {
"summary": "Example summary"
}
},
"generatorURL": "localhost/prometheus"
}, {
"labels": {
"alertname": "Host_Down",
@@ -72,7 +74,8 @@ for i in xrange(0, 5):
"annotations": {
"summary": "Example summary",
"url": "http://localhost/example.html"
}
},
"generatorURL": "localhost/prometheus"
}, {
"labels": {
"alertname": "Host_Down",
@@ -82,7 +85,8 @@ for i in xrange(0, 5):
},
"annotations": {
"summary": "Example summary"
}
},
"generatorURL": "localhost/prometheus"
}, {
"labels": {
"alertname": "Host_Down",
@@ -92,7 +96,8 @@ for i in xrange(0, 5):
},
"annotations": {
"summary": "Example summary"
}
},
"generatorURL": "localhost/prometheus"
}, {
"labels": {
"alertname": "Host_Down",
@@ -102,7 +107,8 @@ for i in xrange(0, 5):
},
"annotations": {
"summary": "Example summary"
}
},
"generatorURL": "localhost/prometheus"
}, {
"labels": {
"alertname": "Host_Down",
@@ -112,7 +118,8 @@ for i in xrange(0, 5):
},
"annotations": {
"summary": "Example summary"
}
},
"generatorURL": "localhost/prometheus"
}, {
"labels": {
"alertname": "Host_Down",
@@ -122,7 +129,8 @@ for i in xrange(0, 5):
},
"annotations": {
"summary": "Example summary"
}
},
"generatorURL": "localhost/prometheus"
}, {
"labels": {
"alertname": "Host_Down",
@@ -132,7 +140,8 @@ for i in xrange(0, 5):
},
"annotations": {
"summary": "Example summary"
}
},
"generatorURL": "localhost/prometheus"
}, {
"labels": {
"alertname": "Host_Down",
@@ -142,7 +151,8 @@ for i in xrange(0, 5):
},
"annotations": {
"summary": "Example summary"
}
},
"generatorURL": "localhost/prometheus"
}, {
"labels": {
"alertname": "Memory_Usage_Too_High",
@@ -153,7 +163,8 @@ for i in xrange(0, 5):
"annotations": {
"alert": "Memory usage exceeding threshold",
"dashboard": "http://localhost/dashboard.html"
}
},
"generatorURL": "localhost/prometheus"
}, {
"labels": {
"alertname": "Free_Disk_Space_Too_Low",
@@ -164,7 +175,8 @@ for i in xrange(0, 5):
"annotations": {
"alert": "Less than 10% disk space is free",
"dashboard": "http://localhost/dashboard.html"
}
},
"generatorURL": "localhost/prometheus"
}]
)
time.sleep(10)