Merge pull request #81 from skamboj/customize-timeouts

Customize timeouts

pkg/goldpinger/client.go

@@ -15,6 +15,7 @@
 package goldpinger
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net"
@@ -22,6 +23,7 @@ import (
 	"time"
 
 	apiclient "github.com/bloomberg/goldpinger/v3/pkg/client"
+	"github.com/bloomberg/goldpinger/v3/pkg/client/operations"
 	"github.com/bloomberg/goldpinger/v3/pkg/models"
 	httptransport "github.com/go-openapi/runtime/client"
 	"github.com/go-openapi/strfmt"
@@ -29,14 +31,14 @@ import (
 
 // CheckNeighbours queries the kubernetes API server for all other goldpinger pods
 // then calls Ping() on each one
-func CheckNeighbours() *models.CheckResults {
-	return PingAllPods(SelectPods())
+func CheckNeighbours(ctx context.Context) *models.CheckResults {
+	return PingAllPods(ctx, SelectPods())
 }
 
 // CheckNeighboursNeighbours queries the kubernetes API server for all other goldpinger
 // pods then calls Check() on each one
-func CheckNeighboursNeighbours() *models.CheckAllResults {
-	return CheckAllPods(SelectPods())
+func CheckNeighboursNeighbours(ctx context.Context) *models.CheckAllResults {
+	return CheckAllPods(ctx, SelectPods())
 }
 
 type PingAllPodsResult struct {
@@ -71,7 +73,7 @@ func checkDNS() *models.DNSResults {
 	return &results
 }
 
-func PingAllPods(pods map[string]*GoldpingerPod) *models.CheckResults {
+func PingAllPods(pingAllCtx context.Context, pods map[string]*GoldpingerPod) *models.CheckResults {
 
 	result := models.CheckResults{}
 
@@ -109,7 +111,14 @@ func PingAllPods(pods map[string]*GoldpingerPod) *models.CheckResults {
 				}
 				CountError("ping")
 			} else {
-				resp, err := client.Operations.Ping(nil)
+				pingCtx, cancel := context.WithTimeout(
+					pingAllCtx,
+					time.Duration(GoldpingerConfig.PingTimeoutMs)*time.Millisecond,
+				)
+				defer cancel()
+
+				params := operations.NewPingParamsWithContext(pingCtx)
+				resp, err := client.Operations.Ping(params)
 				responseTime = time.Since(start).Nanoseconds() / int64(time.Millisecond)
 				OK = (err == nil)
 				if OK {
@@ -167,7 +176,7 @@ type CheckServicePodsResult struct {
 	podIPv4           strfmt.IPv4
 }
 
-func CheckAllPods(pods map[string]*GoldpingerPod) *models.CheckAllResults {
+func CheckAllPods(checkAllCtx context.Context, pods map[string]*GoldpingerPod) *models.CheckAllResults {
 
 	result := models.CheckAllResults{Responses: make(map[string]models.CheckAllPodResult)}
 
@@ -200,7 +209,14 @@ func CheckAllPods(pods map[string]*GoldpingerPod) *models.CheckAllResults {
 				}
 				CountError("checkAll")
 			} else {
-				resp, err := client.Operations.CheckServicePods(nil)
+				checkCtx, cancel := context.WithTimeout(
+					checkAllCtx,
+					time.Duration(GoldpingerConfig.CheckTimeoutMs)*time.Millisecond,
+				)
+				defer cancel()
+
+				params := operations.NewCheckServicePodsParamsWithContext(checkCtx)
+				resp, err := client.Operations.CheckServicePods(params)
 				OK = (err == nil)
 				if OK {
 					channelResult.checkAllPodResult = models.CheckAllPodResult{

pkg/goldpinger/config.go

@@ -33,4 +33,9 @@ var GoldpingerConfig = struct {
 	KubernetesClient *kubernetes.Clientset
 
 	DnsHosts []string `long:"host-to-resolve" description:"A host to attempt dns resolve on (space delimited)" env:"HOSTS_TO_RESOLVE" env-delim:" "`
+
+	// Timeouts
+	PingTimeoutMs     int64 `long:"ping-timeout-ms" description:"The timeout in milliseconds for a ping call to other goldpinger pods" env:"PING_TIMEOUT_MS" default:"300"`
+	CheckTimeoutMs    int64 `long:"check-timeout-ms" description:"The timeout in milliseconds for a check call to other goldpinger pods" env:"CHECK_TIMEOUT_MS" default:"1000"`
+	CheckAllTimeoutMs int64 `long:"check-all-timeout-ms" description:"The timeout in milliseconds for a check-all call to other goldpinger pods" env:"CHECK_ALL_TIMEOUT_MS" default:"5000"`
 }{}

pkg/goldpinger/heatmap.go

@@ -18,6 +18,7 @@ package goldpinger
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"image"
 	"image/color"
@@ -26,6 +27,7 @@ import (
 	"net/http"
 	"sort"
 	"strconv"
+	"time"
 
 	"golang.org/x/image/font"
 	"golang.org/x/image/font/basicfont"
@@ -79,8 +81,14 @@ func HeatmapHandler(w http.ResponseWriter, r *http.Request) {
 	// parse the query to set the parameters
 	query := r.URL.Query()
 
+	ctx, cancel := context.WithTimeout(
+		r.Context(),
+		time.Duration(GoldpingerConfig.CheckAllTimeoutMs)*time.Millisecond,
+	)
+	defer cancel()
+
 	// get the results
-	checkResults := CheckAllPods(GetAllPods())
+	checkResults := CheckAllPods(ctx, GetAllPods())
 
 	// set some sizes
 	numberOfPods := len(checkResults.Responses)

pkg/goldpinger/stats.go

@@ -15,6 +15,7 @@
 package goldpinger
 
 import (
+	"context"
 	"log"
 	"time"
 
@@ -106,7 +107,7 @@ func init() {
 	log.Println("Metrics setup - see /metrics")
 }
 
-func GetStats() *models.PingResults {
+func GetStats(ctx context.Context) *models.PingResults {
 	// GetStats no longer populates the received and made calls - use metrics for that instead
 	return &models.PingResults{
 		BootTime: strfmt.DateTime(bootTime),

pkg/goldpinger/updater.go

@@ -15,6 +15,7 @@
 package goldpinger
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"time"
@@ -26,10 +27,14 @@ func StartUpdater() {
 		return
 	}
 
+	updateInterval := time.Duration(GoldpingerConfig.RefreshInterval) * time.Second
+
 	// start the updater
 	go func() {
 		for {
-			results := PingAllPods(SelectPods())
+			ctx, cancel := context.WithTimeout(context.Background(), updateInterval)
+
+			results := PingAllPods(ctx, SelectPods())
 			var troublemakers []string
 			for podIP, value := range results.PodResults {
 				if *value.OK != true {
@@ -39,7 +44,9 @@ func StartUpdater() {
 			if len(troublemakers) > 0 {
 				log.Println("Updater ran into trouble with these peers: ", troublemakers)
 			}
-			time.Sleep(time.Duration(GoldpingerConfig.RefreshInterval) * time.Second)
+
+			cancel()
+			time.Sleep(updateInterval)
 		}
 	}()
 }

pkg/restapi/configure_goldpinger.go

@@ -17,9 +17,11 @@
 package restapi
 
 import (
+	"context"
 	"crypto/tls"
 	"log"
 	"net/http"
+	"time"
 
 	"strings"
 
@@ -55,19 +57,40 @@ func configureAPI(api *operations.GoldpingerAPI) http.Handler {
 	api.PingHandler = operations.PingHandlerFunc(
 		func(params operations.PingParams) middleware.Responder {
 			goldpinger.CountCall("received", "ping")
-			return operations.NewPingOK().WithPayload(goldpinger.GetStats())
+
+			ctx, cancel := context.WithTimeout(
+				params.HTTPRequest.Context(),
+				time.Duration(goldpinger.GoldpingerConfig.PingTimeoutMs)*time.Millisecond,
+			)
+			defer cancel()
+
+			return operations.NewPingOK().WithPayload(goldpinger.GetStats(ctx))
 		})
 
 	api.CheckServicePodsHandler = operations.CheckServicePodsHandlerFunc(
 		func(params operations.CheckServicePodsParams) middleware.Responder {
 			goldpinger.CountCall("received", "check")
-			return operations.NewCheckServicePodsOK().WithPayload(goldpinger.CheckNeighbours())
+
+			ctx, cancel := context.WithTimeout(
+				params.HTTPRequest.Context(),
+				time.Duration(goldpinger.GoldpingerConfig.CheckTimeoutMs)*time.Millisecond,
+			)
+			defer cancel()
+
+			return operations.NewCheckServicePodsOK().WithPayload(goldpinger.CheckNeighbours(ctx))
 		})
 
 	api.CheckAllPodsHandler = operations.CheckAllPodsHandlerFunc(
 		func(params operations.CheckAllPodsParams) middleware.Responder {
 			goldpinger.CountCall("received", "check_all")
-			return operations.NewCheckAllPodsOK().WithPayload(goldpinger.CheckNeighboursNeighbours())
+
+			ctx, cancel := context.WithTimeout(
+				params.HTTPRequest.Context(),
+				time.Duration(goldpinger.GoldpingerConfig.CheckAllTimeoutMs)*time.Millisecond,
+			)
+			defer cancel()
+
+			return operations.NewCheckAllPodsOK().WithPayload(goldpinger.CheckNeighboursNeighbours(ctx))
 		})
 
 	api.HealthzHandler = operations.HealthzHandlerFunc(

static/index.html

@@ -248,15 +248,19 @@ var main = function(timeout){
                 var edges = [];
                 var resp = data.responses;
                 for (let callId in resp) {
-                        var call = resp[callId].response['podResults'];
-                        if (typeof call !== 'string'){
-                                for (let target in call) {
-                                        edges.push({
-                                                source: callId,
-                                                target: target,
-                                                _data: call[target]
-                                        });
-                                };
+                        // If there was an error, the response can be undefined,
+                        // especially if there was a timeout/context exceeded
+                        if (resp[callId].response !== undefined) {
+                                var call = resp[callId].response['podResults'];
+                                if (typeof call !== 'string'){
+                                        for (let target in call) {
+                                                edges.push({
+                                                        source: callId,
+                                                        target: target,
+                                                        _data: call[target]
+                                                });
+                                        };
+                                }
                         }
                 };
 
@@ -480,6 +484,3 @@ $("#update-heatmap").click(function (e) {
 </script>
 </body>
 </html>
-
-
-