diff --git a/pkg/goldpinger/client.go b/pkg/goldpinger/client.go index e3b0529..54bb71c 100644 --- a/pkg/goldpinger/client.go +++ b/pkg/goldpinger/client.go @@ -15,6 +15,7 @@ package goldpinger import ( + "context" "errors" "fmt" "net" @@ -22,6 +23,7 @@ import ( "time" apiclient "github.com/bloomberg/goldpinger/v3/pkg/client" + "github.com/bloomberg/goldpinger/v3/pkg/client/operations" "github.com/bloomberg/goldpinger/v3/pkg/models" httptransport "github.com/go-openapi/runtime/client" "github.com/go-openapi/strfmt" @@ -29,14 +31,14 @@ import ( // CheckNeighbours queries the kubernetes API server for all other goldpinger pods // then calls Ping() on each one -func CheckNeighbours() *models.CheckResults { - return PingAllPods(SelectPods()) +func CheckNeighbours(ctx context.Context) *models.CheckResults { + return PingAllPods(ctx, SelectPods()) } // CheckNeighboursNeighbours queries the kubernetes API server for all other goldpinger // pods then calls Check() on each one -func CheckNeighboursNeighbours() *models.CheckAllResults { - return CheckAllPods(SelectPods()) +func CheckNeighboursNeighbours(ctx context.Context) *models.CheckAllResults { + return CheckAllPods(ctx, SelectPods()) } type PingAllPodsResult struct { @@ -71,7 +73,7 @@ func checkDNS() *models.DNSResults { return &results } -func PingAllPods(pods map[string]*GoldpingerPod) *models.CheckResults { +func PingAllPods(pingAllCtx context.Context, pods map[string]*GoldpingerPod) *models.CheckResults { result := models.CheckResults{} @@ -109,7 +111,14 @@ func PingAllPods(pods map[string]*GoldpingerPod) *models.CheckResults { } CountError("ping") } else { - resp, err := client.Operations.Ping(nil) + pingCtx, cancel := context.WithTimeout( + pingAllCtx, + time.Duration(GoldpingerConfig.PingTimeoutMs)*time.Millisecond, + ) + defer cancel() + + params := operations.NewPingParamsWithContext(pingCtx) + resp, err := client.Operations.Ping(params) responseTime = time.Since(start).Nanoseconds() / int64(time.Millisecond) OK = (err == nil) if OK { @@ -167,7 +176,7 @@ type CheckServicePodsResult struct { podIPv4 strfmt.IPv4 } -func CheckAllPods(pods map[string]*GoldpingerPod) *models.CheckAllResults { +func CheckAllPods(checkAllCtx context.Context, pods map[string]*GoldpingerPod) *models.CheckAllResults { result := models.CheckAllResults{Responses: make(map[string]models.CheckAllPodResult)} @@ -200,7 +209,14 @@ func CheckAllPods(pods map[string]*GoldpingerPod) *models.CheckAllResults { } CountError("checkAll") } else { - resp, err := client.Operations.CheckServicePods(nil) + checkCtx, cancel := context.WithTimeout( + checkAllCtx, + time.Duration(GoldpingerConfig.CheckTimeoutMs)*time.Millisecond, + ) + defer cancel() + + params := operations.NewCheckServicePodsParamsWithContext(checkCtx) + resp, err := client.Operations.CheckServicePods(params) OK = (err == nil) if OK { channelResult.checkAllPodResult = models.CheckAllPodResult{ diff --git a/pkg/goldpinger/config.go b/pkg/goldpinger/config.go index d06ead4..db92f17 100644 --- a/pkg/goldpinger/config.go +++ b/pkg/goldpinger/config.go @@ -33,4 +33,9 @@ var GoldpingerConfig = struct { KubernetesClient *kubernetes.Clientset DnsHosts []string `long:"host-to-resolve" description:"A host to attempt dns resolve on (space delimited)" env:"HOSTS_TO_RESOLVE" env-delim:" "` + + // Timeouts + PingTimeoutMs int64 `long:"ping-timeout-ms" description:"The timeout in milliseconds for a ping call to other goldpinger pods" env:"PING_TIMEOUT_MS" default:"300"` + CheckTimeoutMs int64 `long:"check-timeout-ms" description:"The timeout in milliseconds for a check call to other goldpinger pods" env:"CHECK_TIMEOUT_MS" default:"1000"` + CheckAllTimeoutMs int64 `long:"check-all-timeout-ms" description:"The timeout in milliseconds for a check-all call to other goldpinger pods" env:"CHECK_ALL_TIMEOUT_MS" default:"5000"` }{} diff --git a/pkg/goldpinger/heatmap.go b/pkg/goldpinger/heatmap.go index 5b5c1da..8eca47b 100644 --- a/pkg/goldpinger/heatmap.go +++ b/pkg/goldpinger/heatmap.go @@ -18,6 +18,7 @@ package goldpinger import ( "bytes" + "context" "fmt" "image" "image/color" @@ -26,6 +27,7 @@ import ( "net/http" "sort" "strconv" + "time" "golang.org/x/image/font" "golang.org/x/image/font/basicfont" @@ -79,8 +81,14 @@ func HeatmapHandler(w http.ResponseWriter, r *http.Request) { // parse the query to set the parameters query := r.URL.Query() + ctx, cancel := context.WithTimeout( + r.Context(), + time.Duration(GoldpingerConfig.CheckAllTimeoutMs)*time.Millisecond, + ) + defer cancel() + // get the results - checkResults := CheckAllPods(GetAllPods()) + checkResults := CheckAllPods(ctx, GetAllPods()) // set some sizes numberOfPods := len(checkResults.Responses) diff --git a/pkg/goldpinger/stats.go b/pkg/goldpinger/stats.go index e923d17..d20d29e 100644 --- a/pkg/goldpinger/stats.go +++ b/pkg/goldpinger/stats.go @@ -15,6 +15,7 @@ package goldpinger import ( + "context" "log" "time" @@ -106,7 +107,7 @@ func init() { log.Println("Metrics setup - see /metrics") } -func GetStats() *models.PingResults { +func GetStats(ctx context.Context) *models.PingResults { // GetStats no longer populates the received and made calls - use metrics for that instead return &models.PingResults{ BootTime: strfmt.DateTime(bootTime), diff --git a/pkg/goldpinger/updater.go b/pkg/goldpinger/updater.go index 3c0e172..90236ac 100644 --- a/pkg/goldpinger/updater.go +++ b/pkg/goldpinger/updater.go @@ -15,6 +15,7 @@ package goldpinger import ( + "context" "fmt" "log" "time" @@ -26,10 +27,14 @@ func StartUpdater() { return } + updateInterval := time.Duration(GoldpingerConfig.RefreshInterval) * time.Second + // start the updater go func() { for { - results := PingAllPods(SelectPods()) + ctx, cancel := context.WithTimeout(context.Background(), updateInterval) + + results := PingAllPods(ctx, SelectPods()) var troublemakers []string for podIP, value := range results.PodResults { if *value.OK != true { @@ -39,7 +44,9 @@ func StartUpdater() { if len(troublemakers) > 0 { log.Println("Updater ran into trouble with these peers: ", troublemakers) } - time.Sleep(time.Duration(GoldpingerConfig.RefreshInterval) * time.Second) + + cancel() + time.Sleep(updateInterval) } }() } diff --git a/pkg/restapi/configure_goldpinger.go b/pkg/restapi/configure_goldpinger.go index 8eb9ca7..e1411b1 100644 --- a/pkg/restapi/configure_goldpinger.go +++ b/pkg/restapi/configure_goldpinger.go @@ -17,9 +17,11 @@ package restapi import ( + "context" "crypto/tls" "log" "net/http" + "time" "strings" @@ -55,19 +57,40 @@ func configureAPI(api *operations.GoldpingerAPI) http.Handler { api.PingHandler = operations.PingHandlerFunc( func(params operations.PingParams) middleware.Responder { goldpinger.CountCall("received", "ping") - return operations.NewPingOK().WithPayload(goldpinger.GetStats()) + + ctx, cancel := context.WithTimeout( + params.HTTPRequest.Context(), + time.Duration(goldpinger.GoldpingerConfig.PingTimeoutMs)*time.Millisecond, + ) + defer cancel() + + return operations.NewPingOK().WithPayload(goldpinger.GetStats(ctx)) }) api.CheckServicePodsHandler = operations.CheckServicePodsHandlerFunc( func(params operations.CheckServicePodsParams) middleware.Responder { goldpinger.CountCall("received", "check") - return operations.NewCheckServicePodsOK().WithPayload(goldpinger.CheckNeighbours()) + + ctx, cancel := context.WithTimeout( + params.HTTPRequest.Context(), + time.Duration(goldpinger.GoldpingerConfig.CheckTimeoutMs)*time.Millisecond, + ) + defer cancel() + + return operations.NewCheckServicePodsOK().WithPayload(goldpinger.CheckNeighbours(ctx)) }) api.CheckAllPodsHandler = operations.CheckAllPodsHandlerFunc( func(params operations.CheckAllPodsParams) middleware.Responder { goldpinger.CountCall("received", "check_all") - return operations.NewCheckAllPodsOK().WithPayload(goldpinger.CheckNeighboursNeighbours()) + + ctx, cancel := context.WithTimeout( + params.HTTPRequest.Context(), + time.Duration(goldpinger.GoldpingerConfig.CheckAllTimeoutMs)*time.Millisecond, + ) + defer cancel() + + return operations.NewCheckAllPodsOK().WithPayload(goldpinger.CheckNeighboursNeighbours(ctx)) }) api.HealthzHandler = operations.HealthzHandlerFunc( diff --git a/static/index.html b/static/index.html index 46c9cd0..b5985cc 100644 --- a/static/index.html +++ b/static/index.html @@ -248,15 +248,19 @@ var main = function(timeout){ var edges = []; var resp = data.responses; for (let callId in resp) { - var call = resp[callId].response['podResults']; - if (typeof call !== 'string'){ - for (let target in call) { - edges.push({ - source: callId, - target: target, - _data: call[target] - }); - }; + // If there was an error, the response can be undefined, + // especially if there was a timeout/context exceeded + if (resp[callId].response !== undefined) { + var call = resp[callId].response['podResults']; + if (typeof call !== 'string'){ + for (let target in call) { + edges.push({ + source: callId, + target: target, + _data: call[target] + }); + }; + } } }; @@ -480,6 +484,3 @@ $("#update-heatmap").click(function (e) { - - -