From 29133e54ca0f55c8e4046ff8e8c27ce6b9e8e527 Mon Sep 17 00:00:00 2001
From: Tom Wilkie <tomwilkie@users.noreply.github.com>
Date: Fri, 24 Jun 2016 09:04:08 +0100
Subject: [PATCH] Add backoff to the consul client (#1608)

* Add backoff to the consul client

* Review feedback
---
 app/multitenant/consul_client.go | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/app/multitenant/consul_client.go b/app/multitenant/consul_client.go
index 693273f0b..39de05f07 100644
--- a/app/multitenant/consul_client.go
+++ b/app/multitenant/consul_client.go
@@ -133,14 +133,20 @@ func (c *consulClient) CAS(key string, out interface{}, f CASCallback) error {
 }
 
 func (c *consulClient) WatchPrefix(prefix string, out interface{}, done chan struct{}, f func(string, interface{}) bool) {
-	index := uint64(0)
+	const (
+		initialBackoff = 1 * time.Second
+		maxBackoff     = 1 * time.Minute
+	)
+	var (
+		backoff = initialBackoff / 2
+		index   = uint64(0)
+	)
 	for {
 		select {
 		case <-done:
 			return
 		default:
 		}
-
 		kvps, meta, err := c.kv.List(prefix, &consul.QueryOptions{
 			RequireConsistent: true,
 			WaitIndex:         index,
@@ -148,8 +154,18 @@ func (c *consulClient) WatchPrefix(prefix string, out interface{}, done chan str
 		})
 		if err != nil {
 			log.Errorf("Error getting path %s: %v", prefix, err)
-			continue
+			backoff = backoff * 2
+			if backoff > maxBackoff {
+				backoff = maxBackoff
+			}
+			select {
+			case <-done:
+				return
+			case <-time.After(backoff):
+				continue
+			}
 		}
+		backoff = initialBackoff
 		if index == meta.LastIndex {
 			continue
 		}