Merge pull request #251 from weaveworks/gates

Implement confirm rollout gate, hook and API

artifacts/ab-testing/deployment.yaml

@@ -25,7 +25,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.4.0
+        image: quay.io/stefanprodan/podinfo:1.7.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

artifacts/appmesh/deployment.yaml

@@ -25,7 +25,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.4.0
+        image: quay.io/stefanprodan/podinfo:1.7.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

artifacts/canaries/deployment.yaml

@@ -25,7 +25,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.4.0
+        image: quay.io/stefanprodan/podinfo:1.7.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

artifacts/flagger/crd.yaml

@@ -174,6 +174,7 @@ spec:
                 - ""
                 - Initializing
                 - Initialized
+                - Waiting
                 - Progressing
                 - Succeeded
                 - Failed

artifacts/gloo/deployment.yaml

@@ -25,7 +25,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.4.0
+        image: quay.io/stefanprodan/podinfo:1.7.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

artifacts/loadtester/deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: loadtester
-          image: weaveworks/flagger-loadtester:0.5.0
+          image: weaveworks/flagger-loadtester:0.6.1
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

artifacts/nginx/deployment.yaml

@@ -23,7 +23,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.4.0
+        image: quay.io/stefanprodan/podinfo:1.7.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

artifacts/workloads/canary-deployment.yaml

@@ -23,7 +23,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.4.0
+        image: quay.io/stefanprodan/podinfo:1.7.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

artifacts/workloads/primary-deployment.yaml

@@ -23,7 +23,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.4.1
+        image: quay.io/stefanprodan/podinfo:1.7.1
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

charts/flagger/templates/crd.yaml

@@ -175,6 +175,7 @@ spec:
                 - ""
                 - Initializing
                 - Initialized
+                - Waiting
                 - Progressing
                 - Succeeded
                 - Failed

charts/loadtester/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 name: loadtester
-version: 0.5.0
-appVersion: 0.5.0
+version: 0.6.0
+appVersion: 0.6.1
 kubeVersion: ">=1.11.0-0"
 engine: gotpl
 description: Flagger's load testing services based on rakyll/hey and bojand/ghz that generates traffic during canary analysis when configured as a webhook.

charts/loadtester/values.yaml

@@ -2,7 +2,7 @@ replicaCount: 1
 
 image:
   repository: weaveworks/flagger-loadtester
-  tag: 0.4.0
+  tag: 0.6.1
   pullPolicy: IfNotPresent
 
 logLevel: info

charts/podinfo/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
-version: 2.2.0
-appVersion: 1.4.0
+version: 2.3.0
+appVersion: 1.7.0
 name: podinfo
 engine: gotpl
 description: Flagger canary deployment demo chart

charts/podinfo/values.yaml

@@ -1,7 +1,7 @@
 # Default values for podinfo.
 image:
   repository: quay.io/stefanprodan/podinfo
-  tag: 1.4.0
+  tag: 1.7.0
   pullPolicy: IfNotPresent
 
 service:

cmd/loadtester/main.go

@@ -10,7 +10,7 @@ import (
 	"time"
 )
 
-var VERSION = "0.5.0"
+var VERSION = "0.6.1"
 var (
 	logLevel          string
 	port              string
@@ -47,5 +47,7 @@ func main() {
 	go taskRunner.Start(100*time.Millisecond, stopCh)
 
 	logger.Infof("Starting load tester v%s API on port %s", VERSION, port)
-	loadtester.ListenAndServe(port, time.Minute, logger, taskRunner, stopCh)
+
+	gateStorage := loadtester.NewGateStorage("in-memory")
+	loadtester.ListenAndServe(port, time.Minute, logger, taskRunner, gateStorage, stopCh)
 }

docs/gitbook/how-it-works.md

@@ -105,6 +105,53 @@ convention you can specify your label with the `-selector-labels` flag.
 The target deployment should expose a TCP port that will be used by Flagger to create the ClusterIP Service and 
 the Istio Virtual Service. The container port from the target deployment should match the `service.port` value.
 
+### Canary status
+
+Get the current status of canary deployments cluster wide: 
+
+```bash
+kubectl get canaries --all-namespaces
+
+NAMESPACE   NAME      STATUS        WEIGHT   LASTTRANSITIONTIME
+test        podinfo   Progressing   15       2019-06-30T14:05:07Z
+prod        frontend  Succeeded     0        2019-06-30T16:15:07Z
+prod        backend   Failed        0        2019-06-30T17:05:07Z
+```
+
+The status condition reflects the last know state of the canary analysis:
+
+```bash
+kubectl -n test get canary/podinfo -oyaml | awk '/status/,0'
+```
+
+A successful rollout status:
+
+```yaml
+status:
+  canaryWeight: 0
+  failedChecks: 0
+  iterations: 0
+  lastAppliedSpec: "14788816656920327485"
+  lastPromotedSpec: "14788816656920327485"
+  conditions:
+  - lastTransitionTime: "2019-07-10T08:23:18Z"
+    lastUpdateTime: "2019-07-10T08:23:18Z"
+    message: Canary analysis completed successfully, promotion finished.
+    reason: Succeeded
+    status: "True"
+    type: Promoted
+```
+
+The `Promoted` status condition can have one of the following reasons: Initialized, Waiting, Progressing, Succeeded or Failed.
+A failed canary will have the promoted status set to `false`,
+the reason to `failed` and the last applied spec will be different to the last promoted one.
+
+Wait for a successful rollout:
+
+```bash
+kubectl wait canary/podinfo --for=condition=promoted
+```
+
 ### Istio routing
 
 Flagger creates an Istio Virtual Service and Destination Rules based on the Canary service spec. 
@@ -608,6 +655,8 @@ The canary analysis can be extended with webhooks. Flagger will call each webhoo
 determine from the response status code (HTTP 2xx) if the canary is failing or not.
 
 There are three types of hooks:
+* Confirm-rollout hooks are executed before scaling up the canary deployment and ca be used for manual approval.
+The rollout is paused until the hook returns a successful HTTP status code.
 * Pre-rollout hooks are executed before routing traffic to canary. 
 The canary advancement is paused if a pre-rollout hook fails and if the number of failures reach the 
 threshold the canary will be rollback.
@@ -621,6 +670,9 @@ Spec:
 ```yaml
   canaryAnalysis:
     webhooks:
+      - name: "start gate"
+        type: confirm-rollout
+        url: http://flagger-loadtester.test/gate/approve
       - name: "smoke test"
         type: pre-rollout
         url: http://flagger-helmtester.kube-system/
@@ -832,3 +884,66 @@ As an alternative to Helm you can use the [Bash Automated Testing System](https:
 ```
 
 Note that you should create a ConfigMap with your Bats tests and mount it inside the tester container.
+
+### Manual Gating
+
+For manual approval of a canary deployment you can use the `confirm-rollout` webhook. 
+The confirmation hooks are executed before the pre-rollout hooks. 
+Flagger will halt the canary traffic shifting and analysis until the confirm webhook returns HTTP status 200.
+
+Manual gating with Flagger's tester:
+
+```yaml
+  canaryAnalysis:
+    webhooks:
+      - name: "gate"
+        type: confirm-rollout
+        url: http://flagger-loadtester.test/gate/halt
+```
+
+The `/gate/halt` returns HTTP 403 thus blocking the rollout. 
+
+If you have notifications enabled, Flagger will post a message to Slack or MS Teams if a canary rollout is waiting for approval.
+
+Change the URL to `/gate/approve` to start the canary analysis:
+
+```yaml
+  canaryAnalysis:
+    webhooks:
+      - name: "start gate"
+        type: confirm-rollout
+        url: http://flagger-loadtester.test/gate/approve
+```
+
+Manual gating can be driven with Flagger's tester API. Set the confirmation URL to `/gate/check`:
+
+```yaml
+  canaryAnalysis:
+    webhooks:
+      - name: "ask confirmation"
+        type: confirm-rollout
+        url: http://flagger-loadtester.test/gate/check
+```
+
+By default the gate is closed, you can start or resume the canary rollout with:
+
+```bash
+kubectl -n test exec -it flagger-loadtester-xxxx-xxxx sh
+
+curl -d '{"name": "podinfo","namespace":"test"}' http://localhost:8080/gate/open 
+```
+
+You can pause the rollout at any time with:
+
+```bash
+curl -d '{"name": "podinfo","namespace":"test"}' http://localhost:8080/gate/close 
+```
+
+If a canary analysis is paused the status will change to waiting:
+
+```bash
+kubectl get canary/podinfo
+
+NAME      STATUS        WEIGHT
+podinfo   Waiting       0
+```

docs/gitbook/tutorials/canary-helm-gitops.md

@@ -344,7 +344,7 @@ launch the `frontend` and `backend` apps.
 A CI/CD pipeline for the `frontend` release could look like this:
 
 * cut a release from the master branch of the podinfo code repo with the git tag `1.4.1`
-* CI builds the image and pushes the `podinfo:1.4.1` image to the container registry
+* CI builds the image and pushes the `podinfo:1.7.1` image to the container registry
 * Flux scans the registry and updates the Helm release `image.tag` to `1.4.1`
 * Flux commits and push the change to the cluster repo
 * Flux applies the updated Helm release on the cluster

docs/gitbook/tutorials/flagger-smi-istio.md

@@ -236,7 +236,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=quay.io/stefanprodan/podinfo:1.7.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:

docs/gitbook/usage/ab-testing.md

@@ -131,7 +131,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/abtest \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=quay.io/stefanprodan/podinfo:1.7.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:

docs/gitbook/usage/appmesh-progressive-delivery.md

@@ -176,7 +176,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=quay.io/stefanprodan/podinfo:1.7.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:

docs/gitbook/usage/blue-green.md

@@ -59,8 +59,8 @@ Create a deployment and a horizontal pod autoscaler:
 ```bash
 export REPO=https://raw.githubusercontent.com/weaveworks/flagger/master
 
-kubectl apply -f ${REPO}/artifacts/canary/deployment.yaml
-kubectl apply -f ${REPO}/artifacts/canary/hpa.yaml
+kubectl apply -f ${REPO}/artifacts/canaries/deployment.yaml
+kubectl apply -f ${REPO}/artifacts/canaries/hpa.yaml
 ```
 
 Deploy the load testing service to generate traffic during the analysis:
@@ -172,7 +172,7 @@ Trigger a deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=quay.io/stefanprodan/podinfo:1.7.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:
@@ -297,7 +297,7 @@ Trigger a deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.3
+podinfod=quay.io/stefanprodan/podinfo:1.7.3
 ```
 
 Generate 404s:

docs/gitbook/usage/gloo-progressive-delivery.md

@@ -197,7 +197,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=quay.io/stefanprodan/podinfo:1.7.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:

docs/gitbook/usage/linkerd-progressive-delivery.md

@@ -165,7 +165,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=quay.io/stefanprodan/podinfo:1.7.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:

docs/gitbook/usage/nginx-progressive-delivery.md

@@ -189,7 +189,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=quay.io/stefanprodan/podinfo:1.7.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:

docs/gitbook/usage/progressive-delivery.md

@@ -119,7 +119,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=quay.io/stefanprodan/podinfo:1.7.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:

kustomize/base/flagger/crd.yaml

@@ -174,6 +174,7 @@ spec:
                 - ""
                 - Initializing
                 - Initialized
+                - Waiting
                 - Progressing
                 - Succeeded
                 - Failed

kustomize/tester/deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: loadtester
-          image: weaveworks/flagger-loadtester:0.5.0
+          image: weaveworks/flagger-loadtester:0.6.1
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

kustomize/tester/kustomization.yaml

@@ -4,4 +4,4 @@ resources:
   - deployment.yaml
 images:
   - name: weaveworks/flagger-loadtester
-    newTag: 0.5.0
+    newTag: 0.6.1

pkg/apis/flagger/v1alpha3/status.go

@@ -43,6 +43,8 @@ const (
 	// CanaryPhaseInitialized means the primary deployment, hpa and ClusterIP services
 	// have been created along with the service mesh or ingress objects
 	CanaryPhaseInitialized CanaryPhase = "Initialized"
+	// CanaryPhaseWaiting means the canary rollout is paused (waiting for confirmation to proceed)
+	CanaryPhaseWaiting CanaryPhase = "Waiting"
 	// CanaryPhaseProgressing means the canary analysis is underway
 	CanaryPhaseProgressing CanaryPhase = "Progressing"
 	// CanaryPhaseSucceeded means the canary analysis has been successful

pkg/apis/flagger/v1alpha3/types.go

@@ -137,6 +137,8 @@ const (
 	PreRolloutHook HookType = "pre-rollout"
 	// PreRolloutHook execute webhook after the canary analysis
 	PostRolloutHook HookType = "post-rollout"
+	// ConfirmRolloutHook halt canary analysis until webhook returns HTTP 200
+	ConfirmRolloutHook HookType = "confirm-rollout"
 )
 
 // CanaryWebhook holds the reference to external checks used for canary analysis

pkg/canary/status.go

@@ -156,7 +156,7 @@ func (c *Deployer) SetStatusPhase(cd *flaggerv1.Canary, phase flaggerv1.CanaryPh
 		cdCopy.Status.Phase = phase
 		cdCopy.Status.LastTransitionTime = metav1.Now()
 
-		if phase != flaggerv1.CanaryPhaseProgressing {
+		if phase != flaggerv1.CanaryPhaseProgressing && phase != flaggerv1.CanaryPhaseWaiting {
 			cdCopy.Status.CanaryWeight = 0
 			cdCopy.Status.Iterations = 0
 		}
@@ -205,6 +205,9 @@ func (c *Deployer) MakeStatusConditions(canaryStatus flaggerv1.CanaryStatus,
 	case flaggerv1.CanaryPhaseInitialized:
 		status = corev1.ConditionTrue
 		message = "Deployment initialization completed."
+	case flaggerv1.CanaryPhaseWaiting:
+		status = corev1.ConditionUnknown
+		message = "Waiting for approval."
 	case flaggerv1.CanaryPhaseProgressing:
 		status = corev1.ConditionUnknown
 		message = "New revision detected, starting canary analysis."

pkg/controller/scheduler.go

@@ -134,6 +134,11 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 		return
 	}
 
+	// check gates
+	if isApproved := c.runConfirmRolloutHooks(cd); !isApproved {
+		return
+	}
+
 	// set max weight default value to 100%
 	maxWeight := 100
 	if cd.Spec.CanaryAnalysis.MaxWeight > 0 {
@@ -454,7 +459,10 @@ func (c *Controller) shouldSkipAnalysis(cd *flaggerv1.Canary, meshRouter router.
 }
 
 func (c *Controller) shouldAdvance(cd *flaggerv1.Canary) (bool, error) {
-	if cd.Status.LastAppliedSpec == "" || cd.Status.Phase == flaggerv1.CanaryPhaseInitializing || cd.Status.Phase == flaggerv1.CanaryPhaseProgressing {
+	if cd.Status.LastAppliedSpec == "" ||
+		cd.Status.Phase == flaggerv1.CanaryPhaseInitializing ||
+		cd.Status.Phase == flaggerv1.CanaryPhaseProgressing ||
+		cd.Status.Phase == flaggerv1.CanaryPhaseWaiting {
 		return true, nil
 	}
 
@@ -523,6 +531,35 @@ func (c *Controller) hasCanaryRevisionChanged(cd *flaggerv1.Canary) bool {
 	return false
 }
 
+func (c *Controller) runConfirmRolloutHooks(canary *flaggerv1.Canary) bool {
+	for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks {
+		if webhook.Type == flaggerv1.ConfirmRolloutHook {
+			err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryPhaseProgressing, webhook)
+			if err != nil {
+				if canary.Status.Phase != flaggerv1.CanaryPhaseWaiting {
+					if err := c.deployer.SetStatusPhase(canary, flaggerv1.CanaryPhaseWaiting); err != nil {
+						c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).Errorf("%v", err)
+					}
+					c.recordEventWarningf(canary, "Halt %s.%s advancement waiting for approval %s",
+						canary.Name, canary.Namespace, webhook.Name)
+					c.sendNotification(canary, "Canary is waiting for approval.", false, false)
+				}
+				return false
+			} else {
+				if canary.Status.Phase == flaggerv1.CanaryPhaseWaiting {
+					if err := c.deployer.SetStatusPhase(canary, flaggerv1.CanaryPhaseProgressing); err != nil {
+						c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).Errorf("%v", err)
+						return false
+					}
+					c.recordEventInfof(canary, "Confirm-rollout check %s passed", webhook.Name)
+					return false
+				}
+			}
+		}
+	}
+	return true
+}
+
 func (c *Controller) runPreRolloutHooks(canary *flaggerv1.Canary) bool {
 	for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks {
 		if webhook.Type == flaggerv1.PreRolloutHook {

pkg/loadtester/gate.go

@@ -0,0 +1,31 @@
+package loadtester
+
+import "sync"
+
+type GateStorage struct {
+	backend string
+	data    *sync.Map
+}
+
+func NewGateStorage(backend string) *GateStorage {
+	return &GateStorage{
+		backend: backend,
+		data:    new(sync.Map),
+	}
+}
+
+func (gs *GateStorage) open(key string) {
+	gs.data.Store(key, true)
+}
+
+func (gs *GateStorage) close(key string) {
+	gs.data.Store(key, false)
+}
+
+func (gs *GateStorage) isOpen(key string) (locked bool) {
+	val, ok := gs.data.LoadOrStore(key, false)
+	if ok {
+		return val.(bool)
+	}
+	return
+}

pkg/loadtester/server.go

@@ -14,13 +14,101 @@ import (
 )
 
 // ListenAndServe starts a web server and waits for SIGTERM
-func ListenAndServe(port string, timeout time.Duration, logger *zap.SugaredLogger, taskRunner *TaskRunner, stopCh <-chan struct{}) {
+func ListenAndServe(port string, timeout time.Duration, logger *zap.SugaredLogger, taskRunner *TaskRunner, gate *GateStorage, stopCh <-chan struct{}) {
 	mux := http.DefaultServeMux
 	mux.Handle("/metrics", promhttp.Handler())
 	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
 		w.Write([]byte("OK"))
 	})
+	mux.HandleFunc("/gate/approve", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("OK"))
+	})
+	mux.HandleFunc("/gate/halt", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusForbidden)
+		w.Write([]byte("Forbidden"))
+	})
+	mux.HandleFunc("/gate/check", func(w http.ResponseWriter, r *http.Request) {
+		body, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			logger.Error("reading the request body failed", zap.Error(err))
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		defer r.Body.Close()
+
+		canary := &flaggerv1.CanaryWebhookPayload{}
+		err = json.Unmarshal(body, canary)
+		if err != nil {
+			logger.Error("decoding the request body failed", zap.Error(err))
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		canaryName := fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)
+		approved := gate.isOpen(canaryName)
+		if approved {
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte("Approved"))
+		} else {
+			w.WriteHeader(http.StatusForbidden)
+			w.Write([]byte("Forbidden"))
+		}
+
+		logger.Infof("%s gate check: approved %v", canaryName, approved)
+	})
+
+	mux.HandleFunc("/gate/open", func(w http.ResponseWriter, r *http.Request) {
+		body, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			logger.Error("reading the request body failed", zap.Error(err))
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		defer r.Body.Close()
+
+		canary := &flaggerv1.CanaryWebhookPayload{}
+		err = json.Unmarshal(body, canary)
+		if err != nil {
+			logger.Error("decoding the request body failed", zap.Error(err))
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		canaryName := fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)
+		gate.open(canaryName)
+
+		w.WriteHeader(http.StatusAccepted)
+
+		logger.Infof("%s gate opened", canaryName)
+	})
+
+	mux.HandleFunc("/gate/close", func(w http.ResponseWriter, r *http.Request) {
+		body, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			logger.Error("reading the request body failed", zap.Error(err))
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		defer r.Body.Close()
+
+		canary := &flaggerv1.CanaryWebhookPayload{}
+		err = json.Unmarshal(body, canary)
+		if err != nil {
+			logger.Error("decoding the request body failed", zap.Error(err))
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		canaryName := fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)
+		gate.close(canaryName)
+
+		w.WriteHeader(http.StatusAccepted)
+
+		logger.Infof("%s gate closed", canaryName)
+	})
+
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		body, err := ioutil.ReadAll(r.Body)
 		if err != nil {

test/e2e-gloo-tests.sh

@@ -55,7 +55,7 @@ spec:
   canaryAnalysis:
     interval: 15s
     threshold: 15
-    maxWeight: 30
+    maxWeight: 50
     stepWeight: 10
     metrics:
     - name: request-success-rate

test/e2e-gloo.sh

@@ -2,16 +2,14 @@
 
 set -o errexit
 
-GLOO_VER="0.14.2"
+GLOO_VER="0.17.6"
 REPO_ROOT=$(git rev-parse --show-toplevel)
 export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
 
 echo '>>> Installing Gloo'
 helm repo add gloo https://storage.googleapis.com/solo-public-helm
 helm upgrade -i gloo gloo/gloo --version ${GLOO_VER} \
---wait \
---namespace gloo-system \
---set gatewayProxies.gateway-proxy.service.type=NodePort
+--namespace gloo-system
 
 kubectl -n gloo-system rollout status deployment/gloo
 kubectl -n gloo-system rollout status deployment/gateway-proxy