Merge pull request #1276 from prymitive/retry-fetch

feat(ui): retry failed alert fetch() calls

ui/package-lock.json

@@ -8051,6 +8051,11 @@
         "object-is": "^1.0.2"
       }
     },
+    "err-code": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/err-code/-/err-code-1.1.2.tgz",
+      "integrity": "sha1-BuARbTAo9q70gGhJ6w6mp0iuaWA="
+    },
     "errno": {
       "version": "0.1.7",
       "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz",
@@ -15526,6 +15531,22 @@
       "integrity": "sha512-MG5r82wBzh7pSKDRa9y+vllNHz3e3d4CNj1PQE4BQYxLme0gKYYBm9YENq+UkEikyZ0XbiGWxYlVw3Rl9O/U8g==",
       "dev": true
     },
+    "promise-retry": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-1.1.1.tgz",
+      "integrity": "sha1-ZznpaOMFHaIM5kl/srUPaRHfPW0=",
+      "requires": {
+        "err-code": "^1.0.0",
+        "retry": "^0.10.0"
+      },
+      "dependencies": {
+        "retry": {
+          "version": "0.10.1",
+          "resolved": "https://registry.npmjs.org/retry/-/retry-0.10.1.tgz",
+          "integrity": "sha1-52OI0heZLCUnUCQdPTlW/tmNj/Q="
+        }
+      }
+    },
     "promise.allsettled": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/promise.allsettled/-/promise.allsettled-1.0.2.tgz",

ui/package.json

@@ -29,6 +29,7 @@
     "mobx-stored": "1.1.0",
     "moment": "2.24.0",
     "object-hash": "2.0.1",
+    "promise-retry": "1.1.1",
     "prop-types": "15.7.2",
     "qs": "6.9.1",
     "react": "16.12.0",

ui/src/Stores/AlertStore.js

@@ -8,6 +8,8 @@ import qs from "qs";
 
 import moment from "moment";
 
+import promiseRetry from "promise-retry";
+
 import { FetchGet } from "Common/Fetch";
 
 const QueryStringEncodeOptions = {
@@ -258,6 +260,11 @@ class AlertStore {
 
   constructor(initialFilters) {
     this.filters.setFilters(initialFilters);
+    this.retryConfig = {
+      retries: 5,
+      minTimeout: 1000,
+      maxTimeout: 5000
+    };
   }
 
   fetch = action((sortOrder, sortLabel, sortReverse) => {
@@ -268,7 +275,9 @@ class AlertStore {
         `alerts.json?sortOrder=${sortOrder}&sortLabel=${sortLabel}&sortReverse=${sortReverse}&`
       ) + FormatAPIFilterQuery(this.filters.values.map(f => f.raw));
 
-    return FetchGet(alertsURI, {})
+    return promiseRetry((retry, number) => {
+      return FetchGet(alertsURI, {}).catch(retry);
+    }, this.retryConfig)
       .then(result => {
         this.status.setProcessing();
         return result.json();

ui/src/Stores/AlertStore.test.js

@@ -373,20 +373,66 @@ describe("AlertStore.fetch", () => {
     fetch.mockReject("Fetch error");
 
     const store = new AlertStore([]);
+    store.retryConfig = {
+      retries: 5,
+      minTimeout: 10,
+      maxTimeout: 10
+    };
     await expect(store.fetch()).resolves.toHaveProperty("error");
 
-    expect(global.fetch).toHaveBeenCalledTimes(1);
+    expect(global.fetch).toHaveBeenCalledTimes(6);
     expect(store.status.value).toEqual(AlertStoreStatuses.Failure);
     expect(store.info.version).toBe("unknown");
     // there should be a trace of the error
     expect(consoleSpy).toHaveBeenCalledTimes(1);
   });
 
+  it("fetch() retries on failure", async () => {
+    const store = new AlertStore([]);
+    store.retryConfig = {
+      retries: 5,
+      minTimeout: 10,
+      maxTimeout: 10
+    };
+
+    fetch.mockReject("Fetch error");
+    await expect(store.fetch()).rejects.toBeTruthy();
+    expect(global.fetch).toHaveBeenCalledTimes(6);
+  });
+
+  it("fetch() retry counter is reset after successful fetch", async () => {
+    const store = new AlertStore(["label=value"]);
+    store.retryConfig = {
+      retries: 5,
+      minTimeout: 10,
+      maxTimeout: 10
+    };
+
+    fetch.mockReject("Fetch error");
+    await expect(store.fetch()).rejects.toBeTruthy();
+    expect(global.fetch).toHaveBeenCalledTimes(6);
+
+    const response = EmptyAPIResponse();
+    fetch.mockResponse(JSON.stringify(response));
+    await expect(store.fetch()).resolves.toBeUndefined();
+    expect(global.fetch).toHaveBeenCalledTimes(7);
+
+    fetch.mockReject("Fetch error");
+    await expect(store.fetch()).rejects.toBeTruthy();
+    expect(global.fetch).toHaveBeenCalledTimes(13);
+  });
+
   it("unapplied filters are marked as applied on fetch error", async () => {
+    const store = new AlertStore([NewUnappliedFilter("foo")]);
+    store.retryConfig = {
+      retries: 5,
+      minTimeout: 10,
+      maxTimeout: 10
+    };
+    store.filters.values[0].applied = false;
+
     jest.spyOn(console, "trace").mockImplementation(() => {});
     fetch.mockReject("Fetch error");
-    const store = new AlertStore([NewUnappliedFilter("foo")]);
-    store.filters.values[0].applied = false;
     await expect(store.fetch()).resolves.toHaveProperty("error");
     expect(store.filters.values[0].applied).toBe(true);
   });