github/karma
1
0
Fork 0
mirror of https://github.com/prymitive/karma synced 2026-05-05 03:16:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(backend): retry failed alertmanager requests

Browse Source
This commit is contained in:
Łukasz Mierzwa
2021-11-01 11:59:39 +00:00
committed by Łukasz Mierzwa
parent d172c58a1a
commit c71c9e6107
14 changed files with 65 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@@ -19,6 +19,7 @@
- Refactored internal APIs.
- Overview modal won't show label name for every value to save screen space.
- Retry failed requests when collecting alerts and silences from alertmanager.
## v0.92

5
cmd/karma/tests/testscript/068_sentry.txt
View File

@@ -143,7 +143,10 @@ level=info msg="Pulling latest alerts and silences from Alertmanager"
level=info msg="Collecting alerts and silences" alertmanager=default
level=info msg="GET request" timeout=40 uri=http://127.0.0.1:9093/metrics
level=error msg="Request failed" error="Get \"http://127.0.0.1:9093/metrics\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=default uri=http://127.0.0.1:9093
level=error msg="Collection failed" error="Get \"http://127.0.0.1:9093/api/v2/status\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=default
level=error msg="Collection failed" error="Get \"http://127.0.0.1:9093/api/v2/status\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=default try=1/2
level=info msg="GET request" timeout=40 uri=http://127.0.0.1:9093/metrics
level=error msg="Request failed" error="Get \"http://127.0.0.1:9093/metrics\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=default uri=http://127.0.0.1:9093
level=error msg="Collection failed" error="Get \"http://127.0.0.1:9093/api/v2/status\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=default try=2/2
level=info msg="Collection completed"
level=info msg="Done, starting HTTP server"
level=info msg="Starting HTTP server" address=127.0.0.1:8068

5
cmd/karma/tests/testscript/069_simple_config.txt
View File

@@ -14,7 +14,10 @@ level=info msg="Pulling latest alerts and silences from Alertmanager"
level=info msg="Collecting alerts and silences" alertmanager=default
level=info msg="GET request" timeout=40 uri=http://127.0.0.1/metrics
level=error msg="Request failed" error="Get \"http://127.0.0.1/metrics\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default uri=http://127.0.0.1
level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default
level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=1/2
level=info msg="GET request" timeout=40 uri=http://127.0.0.1/metrics
level=error msg="Request failed" error="Get \"http://127.0.0.1/metrics\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default uri=http://127.0.0.1
level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=2/2
level=info msg="Collection completed"
level=info msg="Done, starting HTTP server"
level=info msg="Starting HTTP server" address=127.0.0.1:8069

5
cmd/karma/tests/testscript/073_pid_file_remove_error.txt
View File

@@ -14,7 +14,10 @@ level=info msg="Pulling latest alerts and silences from Alertmanager"
level=info msg="Collecting alerts and silences" alertmanager=default
level=info msg="GET request" timeout=40 uri=http://127.0.0.1/metrics
level=error msg="Request failed" error="Get \"http://127.0.0.1/metrics\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default uri=http://127.0.0.1
level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default
level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=1/2
level=info msg="GET request" timeout=40 uri=http://127.0.0.1/metrics
level=error msg="Request failed" error="Get \"http://127.0.0.1/metrics\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default uri=http://127.0.0.1
level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=2/2
level=info msg="Collection completed"
level=info msg="Done, starting HTTP server"
level=info msg="Starting HTTP server" address=127.0.0.1:8073

5
cmd/karma/tests/testscript/077_listen_invalid_port.txt
View File

@@ -13,7 +13,10 @@ level=info msg="Pulling latest alerts and silences from Alertmanager"
level=info msg="Collecting alerts and silences" alertmanager=default
level=info msg="GET request" timeout=40 uri=http://foo:xxx@127.0.0.1/metrics
level=error msg="Request failed" error="Get \"http://foo:***@127.0.0.1/metrics\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default uri=http://foo:xxx@127.0.0.1
level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default
level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=1/2
level=info msg="GET request" timeout=40 uri=http://foo:xxx@127.0.0.1/metrics
level=error msg="Request failed" error="Get \"http://foo:***@127.0.0.1/metrics\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default uri=http://foo:xxx@127.0.0.1
level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=2/2
level=info msg="Collection completed"
level=info msg="Done, starting HTTP server"
level=error msg="Execution failed" error="listen tcp: address 9999999: invalid port"

5
cmd/karma/tests/testscript/083_metrics.txt
View File

@@ -124,7 +124,10 @@ level=info msg="Pulling latest alerts and silences from Alertmanager"
level=info msg="Collecting alerts and silences" alertmanager=default
level=info msg="GET request" timeout=40 uri=http://127.0.0.1/metrics
level=error msg="Request failed" error="Get \"http://127.0.0.1/metrics\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default uri=http://127.0.0.1
level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default
level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=1/2
level=info msg="GET request" timeout=40 uri=http://127.0.0.1/metrics
level=error msg="Request failed" error="Get \"http://127.0.0.1/metrics\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default uri=http://127.0.0.1
level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=2/2
level=info msg="Collection completed"
level=info msg="Done, starting HTTP server"
level=info msg="Starting HTTP server" address=127.0.0.1:8083

5
cmd/karma/tests/testscript/085_debug.txt
View File

@@ -14,7 +14,10 @@ level=info msg="Pulling latest alerts and silences from Alertmanager"
level=info msg="Collecting alerts and silences" alertmanager=default
level=info msg="GET request" timeout=40 uri=http://127.0.0.1/metrics
level=error msg="Request failed" error="Get \"http://127.0.0.1/metrics\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default uri=http://127.0.0.1
level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default
level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=1/2
level=info msg="GET request" timeout=40 uri=http://127.0.0.1/metrics
level=error msg="Request failed" error="Get \"http://127.0.0.1/metrics\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default uri=http://127.0.0.1
level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=2/2
level=info msg="Collection completed"
level=info msg="Done, starting HTTP server"
level=info msg="Starting HTTP server" address=127.0.0.1:8085

5
cmd/karma/tests/testscript/088_listen_tls_key_invalid.txt
View File

@@ -17,7 +17,10 @@ level=info msg="Pulling latest alerts and silences from Alertmanager"
level=info msg="Collecting alerts and silences" alertmanager=local
level=info msg="GET request" timeout=10 uri=https://127.0.0.1:9093/metrics
level=error msg="Request failed" error="Get \"https://127.0.0.1:9093/metrics\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=local uri=https://127.0.0.1:9093
level=error msg="Collection failed" error="Get \"https://127.0.0.1:9093/api/v2/status\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=local
level=error msg="Collection failed" error="Get \"https://127.0.0.1:9093/api/v2/status\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=local try=1/2
level=info msg="GET request" timeout=10 uri=https://127.0.0.1:9093/metrics
level=error msg="Request failed" error="Get \"https://127.0.0.1:9093/metrics\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=local uri=https://127.0.0.1:9093
level=error msg="Collection failed" error="Get \"https://127.0.0.1:9093/api/v2/status\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=local try=2/2
level=info msg="Collection completed"
level=info msg="Done, starting HTTP server"
level=info msg="Starting HTTPS server" address=127.0.0.1:8088

5
cmd/karma/tests/testscript/089_listen_tls_cert_invalid.txt
View File

@@ -17,7 +17,10 @@ level=info msg="Pulling latest alerts and silences from Alertmanager"
level=info msg="Collecting alerts and silences" alertmanager=local
level=info msg="GET request" timeout=10 uri=https://127.0.0.1:9093/metrics
level=error msg="Request failed" error="Get \"https://127.0.0.1:9093/metrics\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=local uri=https://127.0.0.1:9093
level=error msg="Collection failed" error="Get \"https://127.0.0.1:9093/api/v2/status\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=local
level=error msg="Collection failed" error="Get \"https://127.0.0.1:9093/api/v2/status\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=local try=1/2
level=info msg="GET request" timeout=10 uri=https://127.0.0.1:9093/metrics
level=error msg="Request failed" error="Get \"https://127.0.0.1:9093/metrics\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=local uri=https://127.0.0.1:9093
level=error msg="Collection failed" error="Get \"https://127.0.0.1:9093/api/v2/status\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=local try=2/2
level=info msg="Collection completed"
level=info msg="Done, starting HTTP server"
level=info msg="Starting HTTPS server" address=127.0.0.1:8089

5
cmd/karma/tests/testscript/090_listen_tls_key_missing.txt
View File

@@ -17,7 +17,10 @@ level=info msg="Pulling latest alerts and silences from Alertmanager"
level=info msg="Collecting alerts and silences" alertmanager=local
level=info msg="GET request" timeout=10 uri=https://127.0.0.1:9093/metrics
level=error msg="Request failed" error="Get \"https://127.0.0.1:9093/metrics\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=local uri=https://127.0.0.1:9093
level=error msg="Collection failed" error="Get \"https://127.0.0.1:9093/api/v2/status\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=local
level=error msg="Collection failed" error="Get \"https://127.0.0.1:9093/api/v2/status\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=local try=1/2
level=info msg="GET request" timeout=10 uri=https://127.0.0.1:9093/metrics
level=error msg="Request failed" error="Get \"https://127.0.0.1:9093/metrics\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=local uri=https://127.0.0.1:9093
level=error msg="Collection failed" error="Get \"https://127.0.0.1:9093/api/v2/status\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=local try=2/2
level=info msg="Collection completed"
level=info msg="Done, starting HTTP server"
level=info msg="Starting HTTPS server" address=127.0.0.1:8090

5
cmd/karma/tests/testscript/091_listen_tls_cert_missing.txt
View File

@@ -17,7 +17,10 @@ level=info msg="Pulling latest alerts and silences from Alertmanager"
level=info msg="Collecting alerts and silences" alertmanager=local
level=info msg="GET request" timeout=10 uri=https://127.0.0.1:9093/metrics
level=error msg="Request failed" error="Get \"https://127.0.0.1:9093/metrics\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=local uri=https://127.0.0.1:9093
level=error msg="Collection failed" error="Get \"https://127.0.0.1:9093/api/v2/status\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=local
level=error msg="Collection failed" error="Get \"https://127.0.0.1:9093/api/v2/status\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=local try=1/2
level=info msg="GET request" timeout=10 uri=https://127.0.0.1:9093/metrics
level=error msg="Request failed" error="Get \"https://127.0.0.1:9093/metrics\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=local uri=https://127.0.0.1:9093
level=error msg="Collection failed" error="Get \"https://127.0.0.1:9093/api/v2/status\": dial tcp 127.0.0.1:9093: connect: connection refused" alertmanager=local try=2/2
level=info msg="Collection completed"
level=info msg="Done, starting HTTP server"
level=info msg="Starting HTTPS server" address=127.0.0.1:8091

5
cmd/karma/tests/testscript/094_shutdown_slow_client.txt
View File

@@ -16,7 +16,10 @@ level=info msg="Pulling latest alerts and silences from Alertmanager"
level=info msg="Collecting alerts and silences" alertmanager=proxied
level=info msg="GET request" timeout=40 uri=http://127.0.0.1:9094/metrics
level=error msg="Request failed" error="Get \"http://127.0.0.1:9094/metrics\": dial tcp 127.0.0.1:9094: connect: connection refused" alertmanager=proxied uri=http://127.0.0.1:9094
level=error msg="Collection failed" error="Get \"http://127.0.0.1:9094/api/v2/status\": dial tcp 127.0.0.1:9094: connect: connection refused" alertmanager=proxied
level=error msg="Collection failed" error="Get \"http://127.0.0.1:9094/api/v2/status\": dial tcp 127.0.0.1:9094: connect: connection refused" alertmanager=proxied try=1/2
level=info msg="GET request" timeout=40 uri=http://127.0.0.1:9094/metrics
level=error msg="Request failed" error="Get \"http://127.0.0.1:9094/metrics\": dial tcp 127.0.0.1:9094: connect: connection refused" alertmanager=proxied uri=http://127.0.0.1:9094
level=error msg="Collection failed" error="Get \"http://127.0.0.1:9094/api/v2/status\": dial tcp 127.0.0.1:9094: connect: connection refused" alertmanager=proxied try=2/2
level=info msg="Collection completed"
level=info msg="Done, starting HTTP server"
level=info msg="Starting HTTP server" address=127.0.0.1:8094

5
cmd/karma/tests/testscript/099_alert_history.txt
View File

@@ -14,7 +14,10 @@ level=info msg="Pulling latest alerts and silences from Alertmanager"
level=info msg="Collecting alerts and silences" alertmanager=default
level=info msg="GET request" timeout=40 uri=http://127.0.0.1/metrics
level=error msg="Request failed" error="Get \"http://127.0.0.1/metrics\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default uri=http://127.0.0.1
level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default
level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=1/2
level=info msg="GET request" timeout=40 uri=http://127.0.0.1/metrics
level=error msg="Request failed" error="Get \"http://127.0.0.1/metrics\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default uri=http://127.0.0.1
level=error msg="Collection failed" error="Get \"http://127.0.0.1/api/v2/status\": dial tcp 127.0.0.1:80: connect: connection refused" alertmanager=default try=2/2
level=info msg="Collection completed"
level=info msg="Done, starting HTTP server"
level=info msg="Starting HTTP server" address=127.0.0.1:8099

19
cmd/karma/timer.go
View File

@@ -1,6 +1,7 @@
package main
import (
"fmt"
"runtime"
"sync"
@@ -9,6 +10,10 @@ import (
"github.com/rs/zerolog/log"
)
const (
maxTries = 2
)
func pullFromAlertmanager() {
// always flush cache once we're done
defer apiCache.Purge()
@@ -22,9 +27,17 @@ func pullFromAlertmanager() {
for _, upstream := range upstreams {
go func(am *alertmanager.Alertmanager) {
log.Info().Str("alertmanager", am.Name).Msg("Collecting alerts and silences")
err := am.Pull()
if err != nil {
log.Error().Err(err).Str("alertmanager", am.Name).Msg("Collection failed")
for i := 1; i <= maxTries; i++ {
err := am.Pull()
if err != nil {
log.Error().
Err(err).
Str("alertmanager", am.Name).
Str("try", fmt.Sprintf("%d/%d", i, maxTries)).
Msg("Collection failed")
} else {
break
}
}
wg.Done()
}(upstream)