ref(classify): parallelize release downloads to fetch *much* faster

_webi/lint-builds.js

@@ -6,6 +6,7 @@ let Path = require('node:path');
 
 let BuildsCacher = require('./builds-cacher.js');
 let HostTargets = require('./build-classifier/host-targets.js');
+let Parallel = require('./parallel.js');
 
 var INSTALLERS_DIR = Path.join(__dirname, '..');
 var CACHE_DIR = Path.join(__dirname, '../_cache');
@@ -154,15 +155,17 @@ async function main() {
 
   console.info('');
   console.info(`Fetching project release assets`);
+  let parallel = 25;
   let projects = [];
-  for (let name of valids) {
+  await Parallel.run(parallel, valids, getAll);
+  async function getAll(name, i) {
     console.info(`    ${name}`);
     let projInfo = await bc.getPackages({
       //Releases: Releases,
       name: name,
       date: new Date(),
     });
-    projects.push(projInfo);
+    projects[i] = projInfo;
   }
 
   console.info(`Classifying build assets for...`);

_webi/parallel.js

@@ -0,0 +1,44 @@
+'use strict';
+
+var Parallel = module.exports;
+Parallel.run = async function (limit, arr, fn) {
+  let index = 0;
+  let actives = [];
+  let results = [];
+  limit = Math.min(limit, arr.length);
+
+  function launch() {
+    let _index = index;
+    let p = fn(arr[_index], _index, arr);
+
+    // some tasks may be synchronous
+    // so we must push before removing
+    actives.push(p);
+
+    p.then(function _resolve(result) {
+      let i = actives.indexOf(p);
+      actives.splice(i, 1);
+      results[_index] = result;
+    });
+
+    index += 1;
+  }
+
+  // start tasks in parallel, up to limit
+  for (; actives.length < limit; ) {
+    launch();
+  }
+
+  // keep the task queue full
+  for (; index < arr.length; ) {
+    // wait for one task to complete
+    await Promise.race(actives);
+    // add one task again
+    launch();
+  }
+
+  // wait for all remaining tasks
+  await Promise.all(actives);
+
+  return results;
+};