From b620c3de0d7c849896ca24625437db9e4ea15ac7 Mon Sep 17 00:00:00 2001
From: AJ ONeal <aj@therootcompany.com>
Date: Tue, 12 Dec 2023 03:06:30 -0700
Subject: [PATCH] ref(classify): parallelize release downloads to fetch *much*
 faster

---
 _webi/lint-builds.js |  7 +++++--
 _webi/parallel.js    | 44 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+), 2 deletions(-)
 create mode 100644 _webi/parallel.js

diff --git a/_webi/lint-builds.js b/_webi/lint-builds.js
index 524c054..c6ec7d2 100644
--- a/_webi/lint-builds.js
+++ b/_webi/lint-builds.js
@@ -6,6 +6,7 @@ let Path = require('node:path');
 
 let BuildsCacher = require('./builds-cacher.js');
 let HostTargets = require('./build-classifier/host-targets.js');
+let Parallel = require('./parallel.js');
 
 var INSTALLERS_DIR = Path.join(__dirname, '..');
 var CACHE_DIR = Path.join(__dirname, '../_cache');
@@ -154,15 +155,17 @@ async function main() {
 
   console.info('');
   console.info(`Fetching project release assets`);
+  let parallel = 25;
   let projects = [];
-  for (let name of valids) {
+  await Parallel.run(parallel, valids, getAll);
+  async function getAll(name, i) {
     console.info(`    ${name}`);
     let projInfo = await bc.getPackages({
       //Releases: Releases,
       name: name,
       date: new Date(),
     });
-    projects.push(projInfo);
+    projects[i] = projInfo;
   }
 
   console.info(`Classifying build assets for...`);
diff --git a/_webi/parallel.js b/_webi/parallel.js
new file mode 100644
index 0000000..08d9f89
--- /dev/null
+++ b/_webi/parallel.js
@@ -0,0 +1,44 @@
+'use strict';
+
+var Parallel = module.exports;
+Parallel.run = async function (limit, arr, fn) {
+  let index = 0;
+  let actives = [];
+  let results = [];
+  limit = Math.min(limit, arr.length);
+
+  function launch() {
+    let _index = index;
+    let p = fn(arr[_index], _index, arr);
+
+    // some tasks may be synchronous
+    // so we must push before removing
+    actives.push(p);
+
+    p.then(function _resolve(result) {
+      let i = actives.indexOf(p);
+      actives.splice(i, 1);
+      results[_index] = result;
+    });
+
+    index += 1;
+  }
+
+  // start tasks in parallel, up to limit
+  for (; actives.length < limit; ) {
+    launch();
+  }
+
+  // keep the task queue full
+  for (; index < arr.length; ) {
+    // wait for one task to complete
+    await Promise.race(actives);
+    // add one task again
+    launch();
+  }
+
+  // wait for all remaining tasks
+  await Promise.all(actives);
+
+  return results;
+};