chore: update deps

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "_webi/build-classifier"]
+	path = _webi/build-classifier
+	url = https://github.com/webinstall/webi-build-classifier.git

_webi/builds-cacher.js

@@ -0,0 +1,452 @@
+'use strict';
+
+var BuildsCacher = module.exports;
+
+let Fs = require('node:fs/promises');
+let Path = require('node:path');
+
+let request = require('@root/request');
+let Triplet = require('./build-classifier/triplet.js');
+
+var ALIAS_RE = /^alias: (\w+)$/m;
+
+var LEGACY_ARCH_MAP = {
+  '*': 'ANYARCH',
+  arm64: 'aarch64',
+  armv6l: 'armv6',
+  armv7l: 'armv7',
+  amd64: 'x86_64',
+  mipsle: 'mipsel',
+  mips64le: 'mips64el',
+  mipsr6le: 'mipsr6el',
+  mips64r6le: 'mips64r6el',
+  // yes... el for arm and mips, but le for ppc
+  // (perhaps the joke got old?)
+  ppc64el: 'ppc64le',
+  386: 'x86',
+};
+var LEGACY_OS_MAP = {
+  '*': 'ANYOS',
+  macos: 'darwin',
+  posix: 'posix_2017',
+};
+
+var TERMS_META = [
+  // pattern
+  '{ARCH}',
+  '{EXT}',
+  '{LIBC}',
+  '{NAME}',
+  '{OS}',
+  '{VENDOR}',
+  // // os-/arch-indepedent
+  // 'ANYARCH',
+  // 'ANYOS',
+  // // libc
+  // 'none',
+  // channel
+  'beta',
+  'dev',
+  'preview',
+  'stable',
+];
+
+async function getPartialHeader(path) {
+  let readme = `${path}/README.md`;
+  let head = await readFirstBytes(readme).catch(function (err) {
+    if (err.code !== 'ENOENT') {
+      console.warn(`warn: ${path}: ${err.message}`);
+    }
+    return null;
+  });
+
+  return head;
+}
+
+// let fsOpen = util.promisify(Fs.open);
+// let fsRead = util.promisify(Fs.read);
+async function readFirstBytes(path) {
+  let start = 0;
+  let n = 1024;
+  let fh = await Fs.open(path, 'r');
+  let buf = new Buffer.alloc(n);
+  let result = await fh.read(buf, start, n);
+  let str = result.buffer.toString('utf8');
+  await fh.close();
+
+  return str;
+}
+
+let promises = {};
+async function getLatestBuilds(Releases, cacheDir, name, date) {
+  let id = `${cacheDir}/${name}`;
+  if (!promises[id]) {
+    promises[id] = Promise.resolve();
+  }
+
+  promises[id] = promises[id].then(async function () {
+    return await getLatestBuildsInner(Releases, cacheDir, name, date);
+  });
+
+  return await promises[id];
+}
+
+async function getLatestBuildsInner(Releases, cacheDir, name, date) {
+  let data = await Releases.latest(request);
+
+  if (!date) {
+    date = new Date();
+  }
+  let isoDate = date.toISOString();
+  let yearMonth = isoDate.slice(0, 7);
+
+  // TODO hash file
+  let dataFile = `${cacheDir}/${yearMonth}/${name}.json`;
+  // TODO fsstat releases.js vs require-ing time as well
+  let tsFile = `${cacheDir}/${yearMonth}/${name}.updated.txt`;
+
+  let dirPath = Path.dirname(dataFile);
+  await Fs.mkdir(dirPath, { recursive: true });
+
+  let json = JSON.stringify(data, null, 2);
+  await Fs.writeFile(dataFile, json, 'utf8');
+
+  let seconds = date.valueOf();
+  let ms = seconds / 1000;
+  let msStr = ms.toFixed(3);
+  await Fs.writeFile(tsFile, msStr, 'utf8');
+
+  return data;
+}
+
+BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
+  if (!ALL_TERMS) {
+    ALL_TERMS = Triplet.TERMS_PRIMARY_MAP;
+  }
+
+  let bc = {};
+  bc.usedTerms = {};
+  bc.orphanTerms = Object.assign({}, ALL_TERMS);
+  bc.unknownTerms = {};
+  bc._triplets = {};
+  bc._targetsByBuildIdCache = {};
+  bc._caches = {};
+  bc._staleAge = 15 * 60 * 1000;
+
+  for (let term of TERMS_META) {
+    delete bc.orphanTerms[term];
+  }
+
+  bc.getPackages = async function () {
+    let dirs = {
+      hidden: {},
+      errors: {},
+      alias: {},
+      invalid: {},
+      selfhosted: {},
+      valid: {},
+    };
+
+    let entries = await Fs.readdir(installers, { withFileTypes: true });
+    for (let entry of entries) {
+      // skip non-installer dirs
+      if (entry.isSymbolicLink()) {
+        dirs.alias[entry.name] = 'symlink';
+        continue;
+      }
+      if (!entry.isDirectory()) {
+        dirs.hidden[entry.name] = '!directory';
+        continue;
+      }
+      if (entry.name === 'node_modules') {
+        dirs.hidden[entry.name] = 'node_modules';
+        continue;
+      }
+      if (entry.name.startsWith('_')) {
+        dirs.hidden[entry.name] = '_*';
+        continue;
+      }
+      if (entry.name.startsWith('.')) {
+        dirs.hidden[entry.name] = '.*';
+        continue;
+      }
+      if (entry.name.startsWith('~')) {
+        dirs.hidden[entry.name] = '~*';
+        continue;
+      }
+      if (entry.name.endsWith('~')) {
+        dirs.hidden[entry.name] = '*~';
+        continue;
+      }
+
+      // skip invalid installers
+      let path = Path.join(installers, entry.name);
+      let head = await getPartialHeader(path);
+      if (!head) {
+        dirs.invalid[entry.name] = '!README.md';
+        continue;
+      }
+
+      let alias = head.match(ALIAS_RE);
+      if (alias) {
+        dirs.alias[entry.name] = true;
+        continue;
+      }
+
+      let releasesPath = Path.join(path, 'releases.js');
+      let releases;
+      try {
+        releases = require(releasesPath);
+      } catch (err) {
+        if (err.code !== 'MODULE_NOT_FOUND') {
+          dirs.errors[entry.name] = err;
+          continue;
+        }
+        if (err.requireStack.length === 2) {
+          dirs.selfhosted[entry.name] = true;
+          continue;
+        }
+        // err.requireStack.length > 1
+        console.error('');
+        console.error('PROBLEM');
+        console.error(`    ${err.message}`);
+        console.error('');
+        console.error('SOLUTION');
+        console.error('    npm clean-install');
+        console.error('');
+        throw new Error(
+          '[SANITY FAIL] should never have missing modules in prod',
+        );
+      }
+
+      dirs.valid[entry.name] = true;
+    }
+
+    return dirs;
+  };
+
+  // Typically a package is organized by release (ex: go has 1.20, 1.21, etc),
+  // but we will organize by the build (ex: go1.20-darwin-arm64.tar.gz, etc).
+  bc.getBuilds = async function ({ Releases, name, date }) {
+    let cacheDir = caches;
+    let installerDir = installers;
+
+    if (!Releases) {
+      Releases = require(`${installerDir}/${name}/releases.js`);
+    }
+    // TODO update all releases files with object export
+    if (!Releases.latest) {
+      Releases.latest = Releases;
+    }
+
+    if (!date) {
+      date = new Date();
+    }
+    let isoDate = date.toISOString();
+    let yearMonth = isoDate.slice(0, 7);
+    let dataFile = `${cacheDir}/${yearMonth}/${name}.json`;
+
+    // let secondsStr = await Fs.readFile(tsFile, 'ascii').catch(function (err) {
+    //   if (err.code !== 'ENOENT') {
+    //     throw err;
+    //   }
+    //   return '0';
+    // });
+    // secondsStr = secondsStr.trim();
+    // let seconds = parseFloat(secondsStr) || 0;
+
+    // let age = now - seconds;
+
+    let data = bc._caches[name];
+    let versions = data?.versions || [];
+    let triplets = [];
+    let now = date.valueOf();
+    if (data) {
+      process.nextTick(async function () {
+        let age = now - data.updated;
+        if (age < bc._staleAge) {
+          return;
+        }
+        data = await getLatestBuilds(Releases, cacheDir, name);
+        let updated = date.valueOf();
+        updateVersions(data, versions);
+        Object.assign(data, { name, updated, versions, triplets });
+        bc._caches[name] = data;
+      });
+      return data;
+    }
+
+    let json = await Fs.readFile(dataFile, 'ascii').catch(async function (err) {
+      if (err.code !== 'ENOENT') {
+        throw err;
+      }
+
+      return null;
+    });
+
+    try {
+      data = JSON.parse(json);
+    } catch (e) {
+      console.error(`error: ${dataFile}:\n\t${e.message}`);
+      data = null;
+    }
+
+    if (!data) {
+      data = await getLatestBuilds(Releases, cacheDir, name);
+    }
+    let updated = date.valueOf();
+    updateVersions(data, versions);
+    Object.assign(data, { name, updated, versions, triplets });
+    bc._caches[name] = data;
+
+    for (let build of data.releases) {
+      if (LEGACY_OS_MAP[build.os]) {
+        build.os = LEGACY_OS_MAP[build.os];
+      }
+      if (LEGACY_ARCH_MAP[build.arch]) {
+        build.arch = LEGACY_ARCH_MAP[build.arch];
+      }
+    }
+
+    return data;
+  };
+
+  // TODO
+  //   - sort version
+  //   - tag channels
+  //   - @beta not install older than stable
+  function updateVersions(data, versions) {
+    for (let release of data.releases) {
+      let hasVersion = versions.includes(release.version);
+      if (!hasVersion) {
+        versions.unshift(release.version);
+      }
+    }
+  }
+
+  // Makes sure that packages are updated once an hour, on average
+  bc._staleNames = [];
+  bc._freshenTimeout = null;
+  bc.freshenRandomPackage = async function (minDelay) {
+    if (!minDelay) {
+      minDelay = 15 * 1000;
+    }
+
+    if (bc._staleNames.length === 0) {
+      let dirs = await bc.getPackages();
+      bc._staleNames = Object.keys(dirs.valid);
+      bc._staleNames.sort(function () {
+        return 0.5 - Math.random();
+      });
+    }
+
+    let name = bc._staleNames.pop();
+    let Releases = require(`${installers}/${name}/releases.js`);
+    //data = await getLatestBuilds(Releases, cacheDir, name);
+    void (await bc.getBuilds({
+      Releases: Releases,
+      name: name,
+      date: new Date(),
+    }));
+
+    let hour = 60 * 60 * 1000;
+    let delay = minDelay;
+    let spread = hour / bc._staleNames.length;
+    let seed = Math.random();
+    delay += seed * spread;
+
+    clearTimeout(bc._freshenTimeout);
+    bc._freshenTimeout = setTimeout(bc.freshenRandomPackage, delay);
+    bc._freshenTimeout.unref();
+  };
+
+  bc.classify = function (pkg, build) {
+    let maybeInstallable = Triplet.maybeInstallable(pkg, build);
+    if (!maybeInstallable) {
+      return null;
+    }
+
+    // because some packages are shimmed to match a single download against
+    let preTarget = Object.assign({ os: '', arch: '', libc: '' }, build);
+
+    let targetId = `${preTarget.os}:${preTarget.arch}:${preTarget.libc}`;
+    let buildId = `${pkg.name}:${targetId}@${build.download}`;
+    let target = bc._targetsByBuildIdCache[buildId];
+    if (target) {
+      Object.assign(build, { target: target, triplet: target.triplet });
+      return target;
+    }
+
+    let pattern = Triplet.toPattern(pkg, build);
+    if (!pattern) {
+      let err = new Error(`no pattern generated for ${name}`);
+      err.code = 'E_BUILD_NO_PATTERN';
+      target = { error: err };
+      bc._targetsByBuildIdCache[buildId] = target;
+      return target;
+    }
+
+    let rawTerms = pattern.split(/[_\{\}\/\.\-]+/g);
+    for (let term of rawTerms) {
+      delete bc.orphanTerms[term];
+      bc.usedTerms[term] = true;
+    }
+
+    // {NAME}/{NAME}-{VER}-Windows-x86_64_v2-musl.exe =>
+    //     {NAME}.windows.x86_64v2.musl.exe
+    let terms = Triplet.patternToTerms(pattern);
+    if (!terms.length) {
+      let err = new Error(`'${terms}' was trimmed to ''`);
+      target = { error: err };
+      bc._targetsByBuildIdCache[buildId] = target;
+      return target;
+    }
+
+    for (let term of terms) {
+      if (!term) {
+        continue;
+      }
+
+      if (ALL_TERMS[term]) {
+        delete bc.orphanTerms[term];
+        bc.usedTerms[term] = true;
+        continue;
+      }
+
+      bc.unknownTerms[term] = true;
+    }
+
+    // {NAME}.windows.x86_64v2.musl.exe
+    //     windows-x86_64_v2-musl
+    target = { triplet: '' };
+    void Triplet.termsToTarget(target, pkg, build, terms);
+
+    target.triplet = `${target.arch}-${target.vendor}-${target.os}-${target.libc}`;
+    let hasTriplet = pkg.triplets.includes(target.triplet);
+    if (!hasTriplet) {
+      // TODO I don't love this hidden behavior
+      // perhaps classify should just happen when the package is loaded
+      // (and the sanity error should be removed, or thrown after the loop is complete)
+      pkg.triplets.push(target.triplet);
+    }
+
+    bc._triplets[target.triplet] = true;
+    bc._targetsByBuildIdCache[buildId] = target;
+
+    let triple = [target.arch, target.vendor, target.os, target.libc];
+    for (let term of triple) {
+      if (!ALL_TERMS[term]) {
+        throw new Error(
+          `[SANITY FAIL] '${pkg.name}' '${target.triplet}' generated unknown term '${term}'`,
+        );
+      }
+
+      delete bc.orphanTerms[term];
+      bc.usedTerms[term] = true;
+    }
+
+    return target;
+  };
+
+  return bc;
+};

_webi/lint-builds.js

@@ -0,0 +1,470 @@
+#!/usr/bin/env node
+'use strict';
+
+let Fs = require('node:fs/promises');
+let Path = require('node:path');
+
+let BuildsCacher = require('./builds-cacher.js');
+let HostTargets = require('./build-classifier/host-targets.js');
+let Parallel = require('./parallel.js');
+
+var INSTALLERS_DIR = Path.join(__dirname, '..');
+var CACHE_DIR = Path.join(__dirname, '../_cache');
+
+let UserAgentsMap = require('./build-classifier/uas.json');
+let uas = Object.keys(UserAgentsMap);
+let uaTargetsMap = {};
+for (let ua of uas) {
+  let terms = ua.split(/[\s\/]+/g);
+  let target = {};
+  void HostTargets.termsToTarget(target, terms);
+  if (!target) {
+    continue;
+  }
+  if (target.errors.length) {
+    throw target.errors[0];
+  }
+  if (!target.os) {
+    // TODO make target null, or create error for this
+    console.warn(`no os for terms: ${terms}`);
+    //throw new Error(`terms: ${terms}`);
+    continue;
+  }
+  if (!target.arch) {
+    // TODO make target null, or create error for this
+    console.warn(`no arch for terms: ${terms}`);
+    //throw new Error(`terms: ${terms}`);
+    continue;
+  }
+  if (!target.libc) {
+    // TODO make target null, or create error for this
+    console.warn(`no libc for terms: ${terms}`);
+    //throw new Error(`terms: ${terms}`);
+    continue;
+  }
+  let triplet = `${target.os}-${target.arch}-${target.libc}`;
+  uaTargetsMap[triplet] = target;
+}
+let uaTargets = [];
+let triplets = Object.keys(uaTargetsMap);
+for (let triplet of triplets) {
+  let target = uaTargetsMap[triplet];
+  uaTargets.push(target);
+}
+
+function showDirs(dirs) {
+  {
+    let errors = Object.keys(dirs.errors);
+    console.error('');
+    console.error(`Errors: ${errors.length}`);
+    for (let name of errors) {
+      let err = dirs.errors[name];
+      console.error(`${name}/: ${err.message}`);
+    }
+  }
+
+  {
+    let hidden = Object.keys(dirs.hidden);
+    console.debug('');
+    console.debug(`Hidden: ${hidden.length}`);
+    for (let name of hidden) {
+      let kind = dirs.hidden[name];
+      if (kind === '!directory') {
+        console.debug(`    ${name}`);
+      } else {
+        console.debug(`    ${name}/`);
+      }
+    }
+  }
+
+  {
+    let alias = Object.keys(dirs.alias);
+    console.debug('');
+    console.debug(`Alias: ${alias.length}`);
+    for (let name of alias) {
+      let kind = dirs.alias[name];
+      if (kind === 'symlink') {
+        console.debug(`    ${name} => ...`);
+      } else {
+        console.debug(`    ${name}/`);
+      }
+    }
+  }
+
+  {
+    let invalids = Object.keys(dirs.invalid);
+    console.warn('');
+    console.warn(`Invalid: ${invalids.length}`);
+    for (let name of invalids) {
+      console.warn(`    ${name}/`);
+    }
+  }
+
+  {
+    let selfhosted = Object.keys(dirs.selfhosted);
+    console.info('');
+    console.info(`Self-Hosted: ${selfhosted.length}`);
+    for (let name of selfhosted) {
+      console.info(`    ${name}/`);
+    }
+  }
+
+  {
+    let valids = Object.keys(dirs.valid);
+    console.info('');
+    console.info(`Found: ${valids.length}`);
+    for (let name of valids) {
+      console.info(`    ${name}/`);
+    }
+  }
+}
+
+let bc = BuildsCacher.create({
+  caches: CACHE_DIR,
+  installers: INSTALLERS_DIR,
+});
+
+async function getPackagesWithBuilds(installersDir, pkgNames, parallel = 25) {
+  let packages = [];
+
+  await Parallel.run(parallel, pkgNames, getAll);
+
+  async function getAll(name, i) {
+    let Releases = require(`${installersDir}/${name}/releases.js`);
+    let pkg = await bc.getBuilds({
+      Releases: Releases,
+      name: name,
+      date: new Date(),
+    });
+    packages[i] = pkg;
+  }
+
+  return packages;
+}
+
+function getBuildsByTarget(packages) {
+  let packagesByName = {};
+
+  for (let pkg of packages) {
+    let buildsByOs = getBuildsByOs(pkg);
+    packagesByName[pkg.name] = buildsByOs;
+  }
+
+  return packagesByName;
+}
+
+function getBuildsByOs(pkg) {
+  let buildsByOs = {};
+
+  for (let build of pkg.releases) {
+    // TODO check targets cache
+    let target = bc.classify(pkg, build);
+    if (!target) {
+      // ignore known, non-package extensions
+      continue;
+    }
+
+    if (target.error) {
+      let err = target.error;
+      let code = err.code || '';
+      console.error(`[ERROR]: ${code} ${pkg.name}: ${build.name}`);
+      console.error(`>>> ${err.message} <<<`);
+      console.error(pkg);
+      console.error(build);
+      console.error(`^^^ ${err.message} ^^^`);
+      console.error(err.stack);
+      continue;
+    }
+
+    let buildsByRelease = getBuildsByRelease(build, buildsByOs, target);
+    buildsByRelease.push(build);
+  }
+
+  return buildsByOs;
+}
+
+function getBuildsByRelease(build, buildsByOs, target) {
+  let archLibc = `${target.arch}-${target.libc}`;
+
+  if (!buildsByOs[target.os]) {
+    buildsByOs[target.os] = {};
+  }
+
+  let buildsByVersion = buildsByOs[target.os];
+  if (!buildsByVersion[build.version]) {
+    buildsByVersion[build.version] = {};
+  }
+
+  let buildsByArchLibc = buildsByVersion[build.version];
+  if (!buildsByArchLibc[archLibc]) {
+    buildsByArchLibc[archLibc] = [];
+  }
+
+  let buildsByRelease = buildsByArchLibc[archLibc];
+  return buildsByRelease;
+}
+
+function matchBuildsByTarget(pkg, buildsTree, target) {
+  let oses = [];
+  let targetOs = target.os;
+  if (target.os === 'windows') {
+    oses = ['ANYOS', 'windows'];
+    //buildsByOs = buildsTree.ANYOS || buildsTree[target.os];
+  } else if (target.os === 'android') {
+    oses = ['ANYOS', 'posix_2017', 'android', 'linux'];
+    // buildsByOs =
+    //   buildsTree.ANYOS || buildsTree.posix_2017 || buildsTree[target.os];
+    // if (!buildsByOs) {
+    //   targetOs = 'linux';
+    //   buildsByOs = buildsTree.linux;
+    // }
+  } else {
+    oses = ['ANYOS', 'posix_2017', target.os];
+    // buildsByOs =
+    //   buildsTree.ANYOS || buildsTree.posix_2017 || buildsTree[target.os];
+  }
+
+  // TODO can we move sortByOsAndArchLibc(builds, anything) down to the lib?
+  //     and then the matcher
+  //     and make the waterfall more optional?
+
+  let waterfall = HostTargets.WATERFALL[target.os] || {};
+  let arches = waterfall[target.arch] ||
+    HostTargets.WATERFALL.ANYOS[target.arch] || [target.arch];
+  arches = ['ANYARCH'].concat(arches);
+  let libcs = waterfall[target.libc] ||
+    HostTargets.WATERFALL.ANYOS[target.libc] || [target.libc];
+
+  //console.log('waterfalls', arches, libcs);
+
+  // TODO flatten earlier and precache?
+  let duplets = [];
+  for (let arch of arches) {
+    for (let libc of libcs) {
+      let duplet = `${arch}-${libc}`;
+      duplets.push(duplet);
+    }
+  }
+
+  let duplet;
+  let targetBuilds;
+  for (let os of oses) {
+    let buildsByOs = buildsTree[os];
+    if (!buildsByOs) {
+      continue;
+    }
+
+    // TODO
+    //   - latest supported triplets
+    //   - historical supported triplets
+
+    // TODO sort versions first, get channel (or 'stable' or 'latest') from user
+    for (let version of pkg.versions) {
+      let versionBuilds = buildsByOs[version];
+      if (!versionBuilds) {
+        continue;
+      }
+
+      for (let _duplet of duplets) {
+        targetBuilds = versionBuilds[_duplet];
+        //console.log(`    duplet: ${_duplet}`, versionBuilds, targetBuilds);
+        if (targetBuilds?.length > 0) {
+          targetOs = os;
+          duplet = _duplet;
+          break;
+        }
+      }
+      if (targetBuilds?.length > 0) {
+        break;
+      }
+    }
+    if (targetBuilds?.length > 0) {
+      break;
+    }
+  }
+
+  if (!targetBuilds?.length) {
+    // console.log('    no builds:', buildsByOs);
+    targetBuilds = [];
+  }
+
+  let match = { triplet: `${targetOs}-${duplet}`, builds: targetBuilds };
+  return match;
+}
+
+async function main() {
+  // TODO
+  //     node ./_webi/lint-builds.js caddy@beta 'x86_64/unknown Darwin libc'
+  //
+  // let [pkgName, userAgent] = process.argv[2].slice(0);
+  // create test case for zoxide, caddy, rg
+
+  let dirs = await bc.getPackages();
+  showDirs(dirs);
+  console.info('');
+
+  bc.freshenRandomPackage(600 * 1000);
+
+  let rows = [];
+  let triples = [];
+  let valids = Object.keys(dirs.valid);
+
+  let index = valids.indexOf('webi');
+  if (index >= 0) {
+    // TODO fix the webi faux package
+    // (not sure why I even created it)
+    void valids.splice(index, 1);
+  }
+
+  let parallel = 25;
+  //valids = ['atomicparsley', 'caddy', 'macos'];
+  //valids = ['atomicparsley'];
+  let packages = await getPackagesWithBuilds(INSTALLERS_DIR, valids, parallel);
+
+  console.info(`Fetching builds for`);
+  for (let pkg of packages) {
+    console.info(`    ${pkg.name}`);
+
+    let nStr = pkg.releases.length.toString();
+    let n = nStr.padStart(5, ' ');
+    let row = `##### ${n}\t${pkg.name}\tv`;
+    rows.push(row);
+
+    // ignore known, non-package extensions
+    for (let build of pkg.releases) {
+      let target = bc.classify(pkg, build);
+      if (!target) {
+        // non-build file
+        continue;
+      }
+      if (target.error) {
+        let e = target.error;
+        if (e.code === 'E_BUILD_NO_PATTERN') {
+          console.warn(`>>> ${e.message} <<<`);
+          console.warn(pkg);
+          console.warn(build);
+          console.warn(`^^^ ${e.message} ^^^`);
+        }
+        throw e;
+      }
+
+      triples.push(target.triplet);
+      // if (!build.version) {
+      //   throw new Error(`no version for ${pkg.name} ${build.name}`);
+      // }
+      // // For debug printing versions
+      // console.error(build.version);
+      rows.push(`${target.triplet}\t${pkg.name}\t${build.version}`);
+    }
+  }
+
+  let packagesTree = getBuildsByTarget(packages);
+  //console.log(`packagesTree`, packagesTree);
+
+  for (let pkg of packages) {
+    console.log('');
+    console.log('');
+    console.log('pkg', pkg.name);
+    let buildsTree = packagesTree[pkg.name];
+    console.log(buildsTree);
+    for (let target of uaTargets) {
+      let libc = target.libc || 'libc';
+      let hostTriplet = `${target.os}-${target.arch}-${libc}`;
+      console.log('');
+      console.log(`target: ${hostTriplet}`);
+      let match = matchBuildsByTarget(pkg, buildsTree, target);
+      if (!match) {
+        console.log(
+          `    pkg: ${pkg.name}: missing build for os '${target.os}'`,
+        );
+        continue;
+      }
+
+      if (match.builds.length === 0) {
+        console.log(
+          `    pkg: ${pkg.name}: missing build for os '${target.os}-${target.arch}-${libc}'`,
+        );
+      } else if (match.triplet === hostTriplet) {
+        console.log(`    selected ${match.builds.length}`);
+      } else {
+        console.log(
+          `    selected ${match.builds.length} (${match.triplet} fallback)`,
+        );
+      }
+    }
+  }
+
+  let tsv = rows.join('\n');
+  console.info('');
+  console.info('#rows', rows.length);
+  await Fs.writeFile('builds.tsv', tsv, 'utf8');
+
+  console.info('');
+  console.info('Triplets Detected:');
+  let triplets = Object.keys(bc._triplets);
+  if (triplets.length) {
+    triplets.sort();
+    console.info('   ', triplets.join('\n    '));
+  } else {
+    console.info('    (none)');
+  }
+
+  console.info('');
+  console.info('New / Unknown Terms:');
+  let unknowns = Object.keys(bc.unknownTerms);
+  if (unknowns.length) {
+    unknowns.sort();
+    console.warn('   ', unknowns.join('\n    '));
+  } else {
+    console.info('    (none)');
+  }
+
+  console.info('');
+  console.info('Unused Terms:');
+  let unuseds = Object.keys(bc.orphanTerms);
+  if (unuseds.length) {
+    unuseds.sort();
+    console.warn('   ', unuseds.join('\n    '));
+  } else {
+    console.info('    (none)');
+  }
+
+  console.info('');
+  // sort -u -k1 builds.tsv | rg -v '^#|^https?:' | rg -i arm
+  // cut -f1 builds.tsv | sort -u -k1 | rg -v '^#|^https?:' | rg -i arm
+}
+
+if (module === require.main) {
+  let times = [];
+  let now = Date.now();
+  main()
+    .then(async function () {
+      let then = Date.now();
+      let delta = then - now;
+      times.push(delta);
+      now = then;
+      await main();
+      then = Date.now();
+      delta = then - now;
+      times.push(delta);
+    })
+    .then(function () {
+      console.info('');
+      console.info('Run times');
+      for (let delta of times) {
+        let s = delta / 1000;
+        console.info(`    ${s}`);
+      }
+
+      function forceExit() {
+        console.warn(`warn: dangling event loop reference`);
+        process.exit(0);
+      }
+      let exitTimeout = setTimeout(forceExit, 250);
+      exitTimeout.unref();
+    })
+    .catch(function (err) {
+      console.error(err.stack || err);
+      process.exit(1);
+    });
+}

_webi/parallel.js

@@ -0,0 +1,44 @@
+'use strict';
+
+var Parallel = module.exports;
+Parallel.run = async function (limit, arr, fn) {
+  let index = 0;
+  let actives = [];
+  let results = [];
+  limit = Math.min(limit, arr.length);
+
+  function launch() {
+    let _index = index;
+    let p = fn(arr[_index], _index, arr);
+
+    // some tasks may be synchronous
+    // so we must push before removing
+    actives.push(p);
+
+    p.then(function _resolve(result) {
+      let i = actives.indexOf(p);
+      actives.splice(i, 1);
+      results[_index] = result;
+    });
+
+    index += 1;
+  }
+
+  // start tasks in parallel, up to limit
+  for (; actives.length < limit; ) {
+    launch();
+  }
+
+  // keep the task queue full
+  for (; index < arr.length; ) {
+    // wait for one task to complete
+    await Promise.race(actives);
+    // add one task again
+    launch();
+  }
+
+  // wait for all remaining tasks
+  await Promise.all(actives);
+
+  return results;
+};