wip: ffprobe

.github/workflows/node.js.yml

@@ -23,9 +23,6 @@ jobs:
           sh ./_scripts/install-ci-deps
           echo "${HOME}/.local/bin" >> $GITHUB_PATH
           echo "${HOME}/.local/opt/pwsh" >> $GITHUB_PATH
-      - run: |
-          git submodule init
-          git submodule update
       - run: shfmt --version
       - run: shellcheck -V
       - run: node --version

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "_webi/build-classifier"]
-	path = _webi/build-classifier
-	url = https://github.com/webinstall/webi-build-classifier.git

_common/git-tag.js

@@ -6,12 +6,11 @@ var Crypto = require('crypto');
 var util = require('util');
 var exec = util.promisify(require('child_process').exec);
 var Fs = require('node:fs/promises');
-var FsSync = require('node:fs');
 var Path = require('node:path');
 
 var repoBaseDir = process.env.REPO_BASE_DIR || '';
 if (!repoBaseDir) {
-  repoBaseDir = Path.resolve('./_repos');
+  repoBaseDir = Path.resolve('./repos');
   // for stderr
   console.error(`[Warn] REPO_BASE_DIR= not set, ${repoBaseDir}`);
 }
@@ -21,29 +20,8 @@ var Repos = {};
 Repos.clone = async function (repoPath, gitUrl) {
   let uuid = Crypto.randomUUID();
   let tmpPath = `${repoPath}.${uuid}.tmp`;
-  let bakPath = `${repoPath}.${uuid}.dup`;
   await exec(`git clone --bare --filter=tree:0 ${gitUrl} ${tmpPath}`);
-
+  await Fs.rename(tmpPath, repoPath);
-  try {
-    FsSync.accessSync(repoPath);
-    return;
-  } catch (e) {
-    if (e.code !== 'ENOENT') {
-      throw e;
-    }
-  }
-
-  // sync to avoid race conditions
-  try {
-    FsSync.renameSync(repoPath, bakPath);
-  } catch (e) {
-    if (e.code !== 'ENOENT') {
-      throw e;
-    }
-  }
-  FsSync.renameSync(tmpPath, repoPath);
-
-  await Fs.rm(bakPath, { force: true, recursive: true });
 };
 
 Repos.checkExists = async function (repoPath) {

_webi/builds-cacher.js

@@ -1,892 +0,0 @@
-'use strict';
-
-var BuildsCacher = module.exports;
-
-let Fs = require('node:fs/promises');
-let Path = require('node:path');
-
-let HostTargets = require('./build-classifier/host-targets.js');
-let Lexver = require('./build-classifier/lexver.js');
-let Triplet = require('./build-classifier/triplet.js');
-
-let request = require('@root/request');
-
-var ALIAS_RE = /^alias: (\w+)$/m;
-
-var LEGACY_ARCH_MAP = {
-  '*': 'ANYARCH',
-  arm64: 'aarch64',
-  armv6l: 'armv6',
-  armv7l: 'armv7',
-  amd64: 'x86_64',
-  mipsle: 'mipsel',
-  mips64le: 'mips64el',
-  mipsr6le: 'mipsr6el',
-  mips64r6le: 'mips64r6el',
-  // yes... el for arm and mips, but le for ppc
-  // (perhaps the joke got old?)
-  ppc64el: 'ppc64le',
-  386: 'x86',
-};
-var LEGACY_OS_MAP = {
-  '*': 'ANYOS',
-  macos: 'darwin',
-  posix: 'posix_2017',
-};
-
-var TERMS_META = [
-  // pattern
-  '{ARCH}',
-  '{EXT}',
-  '{LIBC}',
-  '{NAME}',
-  '{OS}',
-  '{VENDOR}',
-  // // os-/arch-indepedent
-  // 'ANYARCH',
-  // 'ANYOS',
-  // // libc
-  // 'none',
-  // channel
-  'beta',
-  'dev',
-  'preview',
-  'stable',
-];
-
-async function getPartialHeader(path) {
-  let readme = `${path}/README.md`;
-  let head = await readFirstBytes(readme).catch(function (err) {
-    if (err.code !== 'ENOENT') {
-      console.warn(`warn: ${path}: ${err.message}`);
-    }
-    return null;
-  });
-
-  return head;
-}
-
-// let fsOpen = util.promisify(Fs.open);
-// let fsRead = util.promisify(Fs.read);
-async function readFirstBytes(path) {
-  let start = 0;
-  let n = 1024;
-  let fh = await Fs.open(path, 'r');
-  let buf = new Buffer.alloc(n);
-  let result = await fh.read(buf, start, n);
-  let str = result.buffer.toString('utf8');
-  await fh.close();
-
-  return str;
-}
-
-let promises = {};
-async function getLatestBuilds(Releases, installersDir, cacheDir, name, date) {
-  if (!Releases) {
-    Releases = require(`${installersDir}/${name}/releases.js`);
-  }
-  // TODO update all releases files with module.exports.xxxx = 'foo';
-  if (!Releases.latest) {
-    Releases.latest = Releases;
-  }
-
-  let id = `${cacheDir}/${name}`;
-  if (!promises[id]) {
-    promises[id] = Promise.resolve();
-  }
-
-  promises[id] = promises[id].then(async function () {
-    return await getLatestBuildsInner(Releases, cacheDir, name, date);
-  });
-
-  return await promises[id];
-}
-
-async function getLatestBuildsInner(Releases, cacheDir, name, date) {
-  let data = await Releases.latest(request);
-
-  if (!date) {
-    date = new Date();
-  }
-  let isoDate = date.toISOString();
-  let yearMonth = isoDate.slice(0, 7);
-
-  // TODO hash file
-  let dataFile = `${cacheDir}/${yearMonth}/${name}.json`;
-  // TODO fsstat releases.js vs require-ing time as well
-  let tsFile = `${cacheDir}/${yearMonth}/${name}.updated.txt`;
-
-  let dirPath = Path.dirname(dataFile);
-  await Fs.mkdir(dirPath, { recursive: true });
-
-  let json = JSON.stringify(data, null, 2);
-  await Fs.writeFile(dataFile, json, 'utf8');
-
-  let seconds = date.valueOf();
-  let ms = seconds / 1000;
-  let msStr = ms.toFixed(3);
-  await Fs.writeFile(tsFile, msStr, 'utf8');
-
-  return data;
-}
-
-BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
-  let installersDir = installers;
-  let cacheDir = caches;
-
-  if (!ALL_TERMS) {
-    ALL_TERMS = Triplet.TERMS_PRIMARY_MAP;
-  }
-
-  let bc = {};
-  bc.usedTerms = {};
-  bc.orphanTerms = Object.assign({}, ALL_TERMS);
-  bc.unknownTerms = {};
-  bc.formats = [];
-  bc._triplets = {};
-  bc._targetsByBuildIdCache = {};
-  bc._caches = {};
-  bc._staleAge = 15 * 60 * 1000;
-  bc._allFormats = {};
-  bc._allTriplets = {};
-
-  for (let term of TERMS_META) {
-    delete bc.orphanTerms[term];
-  }
-
-  bc.getProjectsByType = async function () {
-    let dirs = {
-      hidden: {},
-      errors: {},
-      alias: {},
-      invalid: {},
-      selfhosted: {},
-      valid: {},
-    };
-
-    let entries = await Fs.readdir(installersDir, { withFileTypes: true });
-    for (let entry of entries) {
-      let meta = await bc.getProjectTypeByEntry(entry);
-      if (meta.type === 'not_found') {
-        let err = meta.detail;
-        console.error('');
-        console.error('PROBLEM');
-        console.error(`    ${err.message}`);
-        console.error('');
-        console.error('SOLUTION');
-        console.error('    npm clean-install');
-        console.error('');
-        throw new Error(
-          '[SANITY FAIL] should never have missing modules in prod',
-        );
-      }
-      dirs[meta.type][entry.name] = meta.detail;
-    }
-
-    return dirs;
-  };
-
-  /**
-   * Get project type and detail - alias, selfhosted, valid (and the invalids)
-   * @param {String} name - filename
-   */
-  bc.getProjectType = async function (name) {
-    let filepath = Path.join(installersDir, name);
-    let entry;
-    try {
-      entry = await Fs.lstat(filepath);
-      Object.assign(entry, { name: name });
-    } catch (e) {
-      return { type: 'errors', detail: 'not found' };
-    }
-    let info = await bc.getProjectTypeByEntry(entry);
-
-    return info;
-  };
-
-  /**
-   * Get project type and detail - alias, selfhosted, valid (and the invalids)
-   * @param {fs.Stats|fs.Dirent} entry
-   */
-  bc.getProjectTypeByEntry = async function (entry) {
-    let path = Path.join(installersDir, entry.name);
-
-    // skip non-installer dirs
-    if (entry.isSymbolicLink()) {
-      let link = await Fs.readlink(path);
-      return { type: 'alias', detail: link };
-    }
-
-    if (!entry.isDirectory()) {
-      return { type: 'hidden', detail: '!directory' };
-    }
-    if (entry.name === 'node_modules') {
-      return { type: 'hidden', detail: 'node_modules' };
-    }
-    if (entry.name.startsWith('_')) {
-      return { type: 'hidden', detail: '_*' };
-    }
-    if (entry.name.startsWith('.')) {
-      return { type: 'hidden', detail: '.*' };
-    }
-    if (entry.name.startsWith('~')) {
-      return { type: 'hidden', detail: '~*' };
-    }
-    if (entry.name.endsWith('~')) {
-      return { type: 'hidden', detail: '*~' };
-    }
-
-    // skip invalid installers
-    let head = await getPartialHeader(path);
-    if (!head) {
-      return { type: 'invalid', detail: '!README.md' };
-    }
-
-    let alias = head.match(ALIAS_RE);
-    if (alias) {
-      let link = alias[1];
-      return { type: 'alias', detail: link };
-    }
-
-    let releasesPath = Path.join(path, 'releases.js');
-    try {
-      void require(releasesPath);
-    } catch (err) {
-      if (err.code !== 'MODULE_NOT_FOUND') {
-        return { type: 'errors', detail: err };
-      }
-
-      if (err.message.includes(`Cannot find module '${releasesPath}'`)) {
-        return { type: 'selfhosted', detail: true };
-      }
-
-      return { type: 'not_found', detail: err };
-    }
-
-    return { type: 'valid', detail: true };
-  };
-
-  // Typically a package is organized by release (ex: go has 1.20, 1.21, etc),
-  // but we will organize by the build (ex: go1.20-darwin-arm64.tar.gz, etc).
-  bc.getPackages = async function ({ Releases, name, date }) {
-    if (!date) {
-      date = new Date();
-    }
-    let isoDate = date.toISOString();
-    let yearMonth = isoDate.slice(0, 7);
-    let dataFile = `${cacheDir}/${yearMonth}/${name}.json`;
-
-    // let secondsStr = await Fs.readFile(tsFile, 'ascii').catch(function (err) {
-    //   if (err.code !== 'ENOENT') {
-    //     throw err;
-    //   }
-    //   return '0';
-    // });
-    // secondsStr = secondsStr.trim();
-    // let seconds = parseFloat(secondsStr) || 0;
-
-    // let age = now - seconds;
-
-    let projInfo = bc._caches[name];
-
-    let meta = {
-      // version info
-      versions: projInfo?.versions || [],
-      lexvers: projInfo?.lexvers || [],
-      lexversMap: projInfo?.lexversMap || {},
-      // culled release assets
-      packages: projInfo?.packages || [],
-      releasesByTriplet: projInfo?.releasesByTriplet || {},
-      // target info
-      triplets: projInfo?.triplets || [],
-      oses: projInfo?.oses || [],
-      arches: projInfo?.arches || [],
-      libcs: projInfo?.libcs || [],
-      formats: projInfo?.formats || [],
-      // TODO channels: projInfo?.channels || [],
-    };
-
-    if (!projInfo) {
-      let json = await Fs.readFile(dataFile, 'ascii').catch(
-        async function (err) {
-          if (err.code !== 'ENOENT') {
-            throw err;
-          }
-
-          return null;
-        },
-      );
-
-      try {
-        projInfo = JSON.parse(json);
-      } catch (e) {
-        console.error(`error: ${dataFile}:\n\t${e.message}`);
-        projInfo = null;
-      }
-    }
-    if (!projInfo) {
-      projInfo = await getLatestBuilds(Releases, installersDir, cacheDir, name);
-    }
-
-    process.nextTick(async function () {
-      let now = date.valueOf();
-      let age = now - projInfo.updated;
-      if (age < bc._staleAge) {
-        return;
-      }
-
-      projInfo = await getLatestBuilds(Releases, installersDir, cacheDir, name);
-      transformAndUpdate(name, projInfo, meta, date);
-    });
-
-    transformAndUpdate(name, projInfo, meta, date);
-    return projInfo;
-  };
-
-  function transformAndUpdate(name, projInfo, meta, date) {
-    meta.packages = [];
-
-    let updated = date.valueOf();
-    Object.assign(projInfo, { name, updated }, meta);
-    for (let build of projInfo.releases) {
-      let buildTarget = bc.classify(projInfo, build);
-      if (!buildTarget) {
-        // ignore known, non-package extensions
-        continue;
-      }
-
-      if (buildTarget.error) {
-        let err = buildTarget.error;
-        let code = err.code || '';
-        console.error(`[ERROR]: ${code} ${projInfo.name}: ${build.name}`);
-        console.error(`>>> ${err.message} <<<`);
-        console.error(projInfo);
-        console.error(build);
-        console.error(`^^^ ${err.message} ^^^`);
-        console.error(err.stack);
-        continue;
-      }
-
-      build.target = buildTarget;
-      meta.packages.push(build);
-    }
-
-    updateReleasesByTriplet(meta);
-    updateAndSortVersions(projInfo, meta);
-
-    Object.assign(projInfo, { name, updated }, meta);
-    bc._caches[name] = projInfo;
-  }
-
-  function updateReleasesByTriplet(meta) {
-    for (let build of meta.packages) {
-      let target = build.target;
-
-      let triplet = `${target.os}-${target.arch}-${target.libc}`;
-      if (!meta.releasesByTriplet[triplet]) {
-        meta.releasesByTriplet[triplet] = {};
-      }
-
-      let buildsByRelease = meta.releasesByTriplet[triplet];
-      if (!buildsByRelease[build.version]) {
-        buildsByRelease[build.version] = [];
-      }
-
-      let packages = buildsByRelease[build.version];
-      packages.push(build);
-    }
-  }
-
-  // TODO
-  //   - tag channels
-  function updateAndSortVersions(projInfo, meta) {
-    for (let build of projInfo.packages) {
-      let hasVersion = meta.versions.includes(build.version);
-      if (!hasVersion) {
-        build.lexver = Lexver.parseVersion(build.version);
-        meta.lexversMap[build.lexver] = build.version;
-      }
-    }
-
-    meta.lexvers = Object.keys(meta.lexversMap);
-    meta.lexvers.sort();
-    meta.lexvers.reverse();
-
-    meta.versions = [];
-    for (let lexver of meta.lexvers) {
-      let version = meta.lexversMap[lexver];
-      meta.versions.push(version);
-    }
-
-    projInfo.packages.sort(function (a, b) {
-      if (a.lexver > b.lexver) {
-        return -1;
-      }
-      if (a.lexver < b.lexver) {
-        return 1;
-      }
-      return 0;
-    });
-  }
-
-  // Makes sure that packages are updated once an hour, on average
-  bc._staleNames = [];
-  bc._freshenTimeout = null;
-  bc.freshenRandomPackage = async function (minDelay) {
-    if (!minDelay) {
-      minDelay = 15 * 1000;
-    }
-
-    if (bc._staleNames.length === 0) {
-      let dirs = await bc.getProjectsByType();
-      bc._staleNames = Object.keys(dirs.valid);
-      bc._staleNames.sort(function () {
-        return 0.5 - Math.random();
-      });
-    }
-
-    let name = bc._staleNames.pop();
-    void (await bc.getPackages({
-      //Releases: Releases,
-      name: name,
-      date: new Date(),
-    }));
-
-    let hour = 60 * 60 * 1000;
-    let delay = minDelay;
-    let spread = hour / bc._staleNames.length;
-    let seed = Math.random();
-    delay += seed * spread;
-
-    clearTimeout(bc._freshenTimeout);
-    bc._freshenTimeout = setTimeout(bc.freshenRandomPackage, delay);
-    bc._freshenTimeout.unref();
-  };
-
-  bc.classify = function (projInfo, build) {
-    /* jshint maxcomplexity: 25 */
-    let maybeInstallable = Triplet.maybeInstallable(projInfo, build);
-    if (!maybeInstallable) {
-      return null;
-    }
-
-    if (LEGACY_OS_MAP[build.os]) {
-      build.os = LEGACY_OS_MAP[build.os];
-    }
-    if (LEGACY_ARCH_MAP[build.arch]) {
-      build.arch = LEGACY_ARCH_MAP[build.arch];
-    }
-
-    // because some packages are shimmed to match a single download against
-    let preTarget = Object.assign({ os: '', arch: '', libc: '' }, build);
-
-    let targetId = `${preTarget.os}:${preTarget.arch}:${preTarget.libc}`;
-    let buildId = `${projInfo.name}:${targetId}@${build.download}`;
-    let target = bc._targetsByBuildIdCache[buildId];
-    if (target) {
-      Object.assign(build, { target: target, triplet: target.triplet });
-      return target;
-    }
-
-    let pattern = Triplet.toPattern(projInfo, build);
-    if (!pattern) {
-      let err = new Error(`no pattern generated for ${projInfo.name}`);
-      err.code = 'E_BUILD_NO_PATTERN';
-      target = { error: err };
-      bc._targetsByBuildIdCache[buildId] = target;
-      return target;
-    }
-
-    let rawTerms = pattern.split(/[_\{\}\/\.\-]+/g);
-    for (let term of rawTerms) {
-      delete bc.orphanTerms[term];
-      bc.usedTerms[term] = true;
-    }
-
-    // {NAME}/{NAME}-{VER}-Windows-x86_64_v2-musl.exe =>
-    //     {NAME}.windows.x86_64v2.musl.exe
-    let terms = Triplet.patternToTerms(pattern);
-    if (!terms.length) {
-      let err = new Error(`'${terms}' was trimmed to ''`);
-      target = { error: err };
-      bc._targetsByBuildIdCache[buildId] = target;
-      return target;
-    }
-
-    for (let term of terms) {
-      if (!term) {
-        continue;
-      }
-
-      if (ALL_TERMS[term]) {
-        delete bc.orphanTerms[term];
-        bc.usedTerms[term] = true;
-        continue;
-      }
-
-      bc.unknownTerms[term] = true;
-    }
-
-    // {NAME}.windows.x86_64v2.musl.exe
-    //     windows-x86_64_v2-musl
-    target = { triplet: '' };
-    void Triplet.termsToTarget(target, projInfo, build, terms);
-
-    target.triplet = `${target.arch}-${target.vendor}-${target.os}-${target.libc}`;
-
-    {
-      // TODO I don't love this hidden behavior
-      // perhaps classify should just happen when the package is loaded
-      // (and the sanity error should be removed, or thrown after the loop is complete)
-      let hasTriplet = projInfo.triplets.includes(target.triplet);
-      if (!hasTriplet) {
-        projInfo.triplets.push(target.triplet);
-      }
-      let hasOs = projInfo.oses.includes(target.os);
-      if (!hasOs) {
-        projInfo.oses.push(target.os);
-      }
-      let hasArch = projInfo.arches.includes(target.arch);
-      if (!hasArch) {
-        projInfo.arches.push(target.arch);
-      }
-      let hasLibc = projInfo.libcs.includes(target.libc);
-      if (!hasLibc) {
-        projInfo.libcs.push(target.libc);
-      }
-
-      if (!build.ext) {
-        build.ext = Triplet.buildToPackageType(build);
-      }
-      if (build.ext) {
-        if (!build.ext.startsWith('.')) {
-          build.ext = `.${build.ext}`;
-        }
-      }
-      let hasExt = projInfo.formats.includes(build.ext);
-      if (!hasExt) {
-        projInfo.formats.push(build.ext);
-      }
-      let hasGlobalExt = bc.formats.includes(build.ext);
-      if (!hasGlobalExt) {
-        bc.formats.push(build.ext);
-      }
-    }
-
-    bc._triplets[target.triplet] = true;
-    bc._targetsByBuildIdCache[buildId] = target;
-
-    let triple = [target.arch, target.vendor, target.os, target.libc];
-    for (let term of triple) {
-      if (!ALL_TERMS[term]) {
-        throw new Error(
-          `[SANITY FAIL] '${projInfo.name}' '${target.triplet}' generated unknown term '${term}'`,
-        );
-      }
-
-      delete bc.orphanTerms[term];
-      bc.usedTerms[term] = true;
-    }
-
-    return target;
-  };
-
-  /**
-   * Given a list of acceptable formats, get the sorted list of of formats.
-   * Actually used (as per node _webi/lint-builds.js):
-   *     .7z
-   *     .app.zip
-   *     .dmg
-   *     .exe
-   *     .exe.xz
-   *     .git
-   *     .gz
-   *     .msi
-   *     .pkg
-   *     .pkg.tar.zst
-   *     .sh
-   *     .tar.gz
-   *     .tar.xz
-   *     .xz
-   *     .zip
-   */
-  bc.getSortedFormats = function (formats) {
-    /* jshint maxcomplexity: 25 */
-    formats.sort();
-    let id = formats.join(',');
-    if (bc._allFormats[id]) {
-      return bc._allFormats[id];
-    }
-
-    // we don't know how to handle any of these yet
-    // let exclude = [];
-    // let isAndroid = false;
-    // if (!isAndroid) {
-    //   exclude.push('.apk');
-    // }
-    // let isDebian = false;
-    // if (!isDebian) {
-    //   exclude.push('.deb');
-    // }
-    // let isEnterpriseLinux = false;
-    // if (!isEnterpriseLinux) {
-    //   exclude.push('.rpm');
-    // }
-    // let isArch = false;
-    // if (!isArch) {
-    //   exclude.push('.pkg.tar.zst');
-    // }
-
-    let hasExe = formats.includes('exe') || formats.includes('.exe');
-
-    /** @type {Array<String>} */
-    let exts = [];
-
-    let hasXz = formats.includes('xz') || formats.includes('.xz');
-    if (hasXz) {
-      exts.push('.tar.xz');
-      if (hasExe) {
-        exts.push('.exe.xz');
-      }
-      exts.push('.xz');
-    }
-    let hasZst = formats.includes('zst') || formats.includes('.zst');
-    if (hasZst) {
-      exts.push('.tar.zst');
-      exts.push('.zst');
-    }
-    let hasZip = formats.includes('zip') || formats.includes('.zip');
-    if (hasZip) {
-      exts.push('.zip');
-    }
-    let has7z = false;
-    if (has7z) {
-      exts.push('.7z');
-    }
-    // let hasBz2 = formats.includes('bz2') || formats.includes('.bz2');
-    // if (hasBz2) {
-    // 	exts.push('.bz2');
-    // }
-    if (hasExe) {
-      if (!hasZip) {
-        exts.push('.zip');
-      }
-      exts.push('.tar.gz');
-      exts.push('.gz');
-      exts.push('.exe');
-      exts.push('.msi');
-      //exts.push('.msixbundle');
-    } else {
-      exts.push('.tar.gz');
-      exts.push('.gz');
-      exts.push('.sh');
-    }
-    exts.push('.git');
-
-    // Fallbacks
-    exts.push('.app.zip');
-    exts.push('.dmg');
-    exts.push('.pkg');
-    if (!hasXz) {
-      exts.push('.tar.xz');
-      if (hasExe) {
-        exts.push('.exe.xz');
-      }
-      exts.push('.xz');
-    }
-    if (!hasZip) {
-      if (!hasExe) {
-        exts.push('.zip');
-      }
-    }
-    if (!hasZst) {
-      exts.push('.tar.zst');
-      exts.push('.zst');
-    }
-    if (!has7z) {
-      exts.push('.7z');
-    }
-    // if (!hasRar) {
-    // 	exts.push('.rar');
-    // }
-    // exts.push('.tar.bz2');
-    // exts.push('.bz2');
-
-    bc._allFormats[id] = exts;
-    return exts;
-  };
-
-  bc.selectPackage = function (packages, formats) {
-    if (packages.length === 1) {
-      return packages[0];
-    }
-
-    let exts = bc.getSortedFormats(formats);
-    for (let ext of exts) {
-      for (let build of packages) {
-        if (build.ext === ext) {
-          return build;
-        }
-      }
-    }
-
-    return packages[0];
-  };
-
-  /**
-   * @param {Object} target
-   * @param {String} target.os - linux, darwin, windows, *bsd
-   * @param {String} target.arch - arm64, x86_64, ppc64le
-   * @param {String} target.libc - none, libc, gnu, musl, bionic, msvc
-   * @param {String} target.triplet - os-vendor-arch-libc
-   * @param {Error} target.error
-   */
-  bc.findMatchingPackages = function (pkgInfo, hostTarget, verTarget) {
-    let matchInfo = bc._enumerateVersions(pkgInfo, verTarget.version);
-    let triplets = bc._enumerateTriplets(hostTarget);
-    //console.log('dbg: matchInfo', matchInfo);
-
-    if (matchInfo) {
-      for (let _triplet of triplets) {
-        let targetReleases = pkgInfo.releasesByTriplet[_triplet];
-        if (!targetReleases) {
-          continue;
-        }
-
-        // Make sure that these releases are the expected version
-        // (ex: jq1.7 => darwin-arm64-libc, jq1.6 => darwin-x86_64-libc)
-        for (let matchver of matchInfo.matches) {
-          let ver = pkgInfo.lexversMap[matchver] || matchver;
-          let packages = targetReleases[ver];
-          if (!packages) {
-            continue;
-          }
-
-          let match = {
-            triplet: _triplet,
-            packages: packages,
-            latest: pkgInfo.versions[0],
-            version: ver,
-            versions: matchInfo,
-          };
-          return match;
-        }
-      }
-
-      return null;
-    }
-
-    for (let _triplet of triplets) {
-      let targetReleases = pkgInfo.releasesByTriplet[_triplet];
-      if (!targetReleases) {
-        continue;
-      }
-
-      let versions = Object.keys(targetReleases);
-      //console.log('dbg: targetRelease versions', versions);
-      let lexvers = [];
-      for (let version of versions) {
-        let lexPrefix = Lexver.parseVersion(version);
-        lexvers.push(lexPrefix);
-      }
-      lexvers.sort();
-      lexvers.reverse();
-      // TODO get the other matchInfo props
-
-      // Make sure that these releases are the expected version
-      // (ex: jq1.7 => darwin-arm64-libc, jq1.6 => darwin-x86_64-libc)
-      for (let matchver of lexvers) {
-        let ver = pkgInfo.lexversMap[matchver] || matchver;
-        let packages = targetReleases[ver];
-        //console.log('dbg: packages', packages);
-        if (!packages) {
-          continue;
-        }
-
-        let pkg = packages[0];
-        if (verTarget.lts) {
-          if (!pkg.lts) {
-            continue;
-          }
-
-          let match = {
-            triplet: _triplet,
-            packages: packages,
-            latest: pkgInfo.versions[0],
-            version: ver,
-            versions: matchInfo,
-          };
-          return match;
-        }
-
-        let wantChannel = verTarget.channel || 'stable';
-        let isChannel = pkg.channel || 'stable';
-        if (wantChannel === 'stable') {
-          if (isChannel !== 'stable') {
-            continue;
-          }
-        }
-        // latest, beta, alpha, rc, preview
-
-        let match = {
-          triplet: _triplet,
-          packages: packages,
-          latest: pkgInfo.versions[0],
-          version: ver,
-          versions: matchInfo,
-        };
-        return match;
-      }
-    }
-
-    return null;
-  };
-
-  bc._enumerateTriplets = function (hostTarget) {
-    let id = [hostTarget.os, hostTarget.arch, hostTarget.libc].join(',');
-    let triplets = bc._allTriplets[id] || [];
-    if (triplets.length > 0) {
-      return triplets;
-    }
-
-    let oses = [];
-    if (hostTarget.os === 'windows') {
-      oses = ['ANYOS', 'windows'];
-    } else if (hostTarget.os === 'android') {
-      oses = ['ANYOS', 'posix_2017', 'android', 'linux'];
-    } else {
-      oses = ['ANYOS', 'posix_2017', hostTarget.os];
-    }
-
-    let waterfall = HostTargets.WATERFALL[hostTarget.os] || {};
-    let arches = waterfall[hostTarget.arch] ||
-      HostTargets.WATERFALL.ANYOS[hostTarget.arch] || [hostTarget.arch];
-    arches = ['ANYARCH'].concat(arches);
-    let libcs = waterfall[hostTarget.libc] ||
-      HostTargets.WATERFALL.ANYOS[hostTarget.libc] || [hostTarget.libc];
-
-    for (let os of oses) {
-      for (let arch of arches) {
-        for (let libc of libcs) {
-          let triplet = `${os}-${arch}-${libc}`;
-          triplets.push(triplet);
-        }
-      }
-    }
-    bc._allTriplets[id] = triplets;
-
-    return triplets;
-  };
-
-  bc._enumerateVersions = function (pkgInfo, ver) {
-    if (!ver) {
-      return null;
-    }
-    let lexPrefix = Lexver.parsePrefix(ver);
-    let matchInfo = Lexver.matchSorted(pkgInfo.lexvers, lexPrefix);
-
-    return matchInfo;
-  };
-
-  return bc;
-};

_webi/builds.js

@@ -1,40 +0,0 @@
-'use strict';
-
-let Builds = module.exports;
-
-let Path = require('node:path');
-
-let BuildsCacher = require('./builds-cacher.js');
-// let HostTargets = require('./build-classifier/host-targets.js');
-let Parallel = require('./parallel.js');
-
-var INSTALLERS_DIR = Path.join(__dirname, '..');
-var CACHE_DIR = Path.join(__dirname, '../_cache');
-
-let bc = BuildsCacher.create({
-  caches: CACHE_DIR,
-  installers: INSTALLERS_DIR,
-});
-bc.freshenRandomPackage(600 * 1000);
-
-Builds.init = async function () {
-  bc.freshenRandomPackage(600 * 1000);
-
-  let dirs = await bc.getProjects();
-  let projNames = Object.keys(dirs.valid);
-
-  let parallel = 25;
-  await Parallel.run(parallel, projNames, getAll);
-  async function getAll(name) {
-    void (await bc.getPackages({
-      //Releases: Releases,
-      name: name,
-      date: new Date(),
-    }));
-  }
-};
-
-Builds.getProjectType = bc.getProjectType;
-Builds.getPackage = bc.getPackages;
-Builds.findMatchingPackages = bc.findMatchingPackages;
-Builds.selectPackage = bc.selectPackage;

_webi/curl-pipe-bootstrap.tpl.sh

@@ -27,42 +27,18 @@ fn_show_welcome() { (
     echo ""
     echo "    $(t_attn 'Success')? Star it!   $(t_url 'https://github.com/webinstall/webi-installers')"
     echo "    $(t_attn 'Problem')? Report it: $(t_url 'https://github.com/webinstall/webi-installers/issues')"
-    echo "                        $(t_dim "(your system is") $(t_host "$(fn_get_os)")/$(t_host "$(uname -m)") $(t_dim "with") $(t_host "$(fn_get_libc)") $(t_dim "&") $(t_host "$(fn_get_http_client_name)")$(t_dim ")")"
+    echo "                        $(t_dim "(your system is") $(t_host "$(uname -s)")/$(t_host "$(uname -m)") $(t_dim "with") $(t_host "$(fn_get_libc)") $(t_dim "&") $(t_host "$(fn_get_http_client_name)")$(t_dim ")")"
 
     sleep 0.2
 ); }
 
-fn_get_os() { (
-    # Ex:
-    #     GNU/Linux
-    #     Android
-    #     Linux (often Alpine, musl)
-    #     Darwin
-    b_os="$(uname -o 2> /dev/null || echo '')"
-    b_sys="$(uname -s)"
-    if test -z "${b_os}" || test "${b_os}" = "${b_sys}"; then
-        # ex: 'Darwin' (and plain, non-GNU 'Linux')
-        echo "${b_sys}"
-        return 0
-    fi
-
-    if echo "${b_os}" | grep -q "${b_sys}"; then
-        # ex: 'GNU/Linux'
-        echo "${b_os}"
-        return 0
-    fi
-
-    # ex: 'Android/Linux'
-    echo "${b_os}/${b_sys}"
-); }
-
 fn_get_libc() { (
     # Ex:
     #     musl
     #     libc
     if ldd /bin/ls 2> /dev/null | grep -q 'musl' 2> /dev/null; then
         echo 'musl'
-    elif fn_get_os | grep -q 'GNU|Linux'; then
+    elif uname -o | grep -q 'GNU' || uname -s | grep -q 'Linux'; then
         echo 'gnu'
     else
         echo 'libc'
@@ -201,9 +177,9 @@ fn_curl() { (
 
 fn_get_target_triple_user_agent() { (
     # Ex:
-    #     x86_64/unknown GNU/Linux/5.15.107-2-pve gnu
+    #     x86_64/unknown Linux/5.15.107-2-pve gnu
     #     arm64/unknown Darwin/22.6.0 libc
-    echo "$(uname -m)/unknown $(fn_get_os)/$(uname -r) $(fn_get_libc)"
+    echo "$(uname -m)/unknown $(uname -s)/$(uname -r) $(fn_get_libc)"
 ); }
 
 fn_download_to_path() { (

_webi/lint-builds.js

@@ -1,330 +0,0 @@
-#!/usr/bin/env node
-'use strict';
-
-let Fs = require('node:fs/promises');
-let Path = require('node:path');
-
-let BuildsCacher = require('./builds-cacher.js');
-let HostTargets = require('./build-classifier/host-targets.js');
-let Parallel = require('./parallel.js');
-
-var INSTALLERS_DIR = Path.join(__dirname, '..');
-var CACHE_DIR = Path.join(__dirname, '../_cache');
-
-let UserAgentsMap = require('./build-classifier/uas.json');
-let uas = Object.keys(UserAgentsMap);
-let uaTargetsMap = {};
-for (let ua of uas) {
-  let terms = ua.split(/[\s\/]+/g);
-  let target = {};
-  void HostTargets.termsToTarget(target, terms);
-  if (!target) {
-    continue;
-  }
-  if (target.errors.length) {
-    throw target.errors[0];
-  }
-  if (!target.os) {
-    // TODO make target null, or create error for this
-    console.warn(`no os for terms: ${terms}`);
-    //throw new Error(`terms: ${terms}`);
-    continue;
-  }
-  if (!target.arch) {
-    // TODO make target null, or create error for this
-    console.warn(`no arch for terms: ${terms}`);
-    //throw new Error(`terms: ${terms}`);
-    continue;
-  }
-  if (!target.libc) {
-    // TODO make target null, or create error for this
-    console.warn(`no libc for terms: ${terms}`);
-    //throw new Error(`terms: ${terms}`);
-    continue;
-  }
-  let triplet = `${target.os}-${target.arch}-${target.libc}`;
-  uaTargetsMap[triplet] = target;
-}
-let uaTargets = [];
-let triplets = Object.keys(uaTargetsMap);
-for (let triplet of triplets) {
-  let target = uaTargetsMap[triplet];
-  uaTargets.push(target);
-}
-
-function showDirs(dirs) {
-  {
-    let errors = Object.keys(dirs.errors);
-    console.error('');
-    console.error(`Errors: ${errors.length}`);
-    for (let name of errors) {
-      let err = dirs.errors[name];
-      console.error(`${name}/: ${err.message}`);
-    }
-  }
-
-  {
-    let hidden = Object.keys(dirs.hidden);
-    console.debug('');
-    console.debug(`Hidden: ${hidden.length}`);
-    for (let name of hidden) {
-      let kind = dirs.hidden[name];
-      if (kind === '!directory') {
-        console.debug(`    ${name}`);
-      } else {
-        console.debug(`    ${name}/`);
-      }
-    }
-  }
-
-  {
-    let alias = Object.keys(dirs.alias);
-    console.debug('');
-    console.debug(`Alias: ${alias.length}`);
-    for (let name of alias) {
-      let kind = dirs.alias[name];
-      if (kind === 'symlink') {
-        console.debug(`    ${name} => ...`);
-      } else {
-        console.debug(`    ${name}/`);
-      }
-    }
-  }
-
-  {
-    let invalids = Object.keys(dirs.invalid);
-    console.warn('');
-    console.warn(`Invalid: ${invalids.length}`);
-    for (let name of invalids) {
-      console.warn(`    ${name}/`);
-    }
-  }
-
-  {
-    let selfhosted = Object.keys(dirs.selfhosted);
-    console.info('');
-    console.info(`Self-Hosted: ${selfhosted.length}`);
-    for (let name of selfhosted) {
-      console.info(`    ${name}/`);
-    }
-  }
-
-  {
-    let valids = Object.keys(dirs.valid);
-    console.info('');
-    console.info(`Found: ${valids.length}`);
-    for (let name of valids) {
-      console.info(`    ${name}/`);
-    }
-  }
-}
-
-let bc = BuildsCacher.create({
-  caches: CACHE_DIR,
-  installers: INSTALLERS_DIR,
-});
-
-async function main() {
-  // TODO
-  //     node ./_webi/lint-builds.js caddy@beta 'x86_64/unknown Darwin libc'
-  //
-  //let [projName, userAgent] = process.argv.slice(2);
-  let projName = process.argv[2];
-  // create test case for zoxide, goreleaser, go, yq, caddy, rg
-
-  let dirs = await bc.getProjectsByType();
-  if (!projName) {
-    showDirs(dirs);
-    console.info('');
-  }
-
-  bc.freshenRandomPackage(600 * 1000);
-
-  let rows = [];
-  let triples = [];
-  let valids = Object.keys(dirs.valid);
-
-  if (projName) {
-    if (!valids.includes(projName)) {
-      throw new Error(`'${projName}' is not a valid installable project`);
-    }
-    valids = [projName];
-  }
-  //valids = ['atomicparsley', 'caddy', 'macos'];
-  //valids = ['atomicparsley'];
-
-  console.info('');
-  console.info(`Fetching project release assets`);
-  let parallel = 25;
-  let projects = [];
-  await Parallel.run(parallel, valids, getAll);
-  async function getAll(name, i) {
-    console.info(`    ${name}`);
-    let projInfo = await bc.getPackages({
-      //Releases: Releases,
-      name: name,
-      date: new Date(),
-    });
-    projects[i] = projInfo;
-  }
-
-  console.info(`Classifying build assets for...`);
-  for (let projInfo of projects) {
-    console.info(`    ${projInfo.name}`);
-
-    let nStr = projInfo.releases.length.toString();
-    let n = nStr.padStart(5, ' ');
-    let row = `##### ${n}\t${projInfo.name}\tv`;
-    rows.push(row);
-
-    // ignore known, non-package extensions
-    for (let build of projInfo.releases) {
-      let target = bc.classify(projInfo, build);
-      if (!target) {
-        // non-build file
-        continue;
-      }
-      if (target.error) {
-        let e = target.error;
-        if (e.code === 'E_BUILD_NO_PATTERN') {
-          console.warn(`>>> ${e.message} <<<`);
-          console.warn(projInfo);
-          console.warn(build);
-          console.warn(`^^^ ${e.message} ^^^`);
-        }
-        throw e;
-      }
-
-      triples.push(target.triplet);
-      // if (!build.version) {
-      //   throw new Error(`no version for ${pkg.name} ${build.name}`);
-      // }
-      // // For debug printing versions
-      // console.error(build.version);
-      rows.push(`${target.triplet}\t${projInfo.name}\t${build.version}`);
-    }
-  }
-
-  console.info(`Fetching builds for`);
-  for (let projInfo of projects) {
-    console.info('');
-    console.info('');
-    console.info(`    ${projInfo.name}`);
-
-    for (let target of uaTargets) {
-      let libc = target.libc || 'libc';
-      let hostTriplet = `${target.os}-${target.arch}-${libc}`;
-      console.info('');
-      console.info(`    target: ${hostTriplet}`);
-      let match = bc.findMatchingPackages(projInfo, target, {
-        ver: '',
-      });
-      if (!match) {
-        console.info(
-          `    project: ${projInfo.name}: missing build for os '${target.os}'`,
-        );
-        continue;
-      }
-
-      if (!match.releases) {
-        console.info(
-          `    project: ${projInfo.name}: missing build for os '${target.os}-${target.arch}-${libc}'`,
-        );
-      } else if (match.triplet === hostTriplet) {
-        let releaseNames = Object.keys(match.releases);
-        console.info(`    selected ${releaseNames.length}`);
-      } else {
-        let releaseNames = Object.keys(match.releases);
-        console.info(
-          `    selected ${releaseNames.length} (${match.triplet} fallback)`,
-        );
-      }
-    }
-  }
-
-  let tsv = rows.join('\n');
-  console.info('');
-  console.info('#rows', rows.length);
-  await Fs.writeFile('builds.tsv', tsv, 'utf8');
-
-  console.info('');
-  console.info('Triplets Detected:');
-  let triplets = Object.keys(bc._triplets);
-  if (triplets.length) {
-    triplets.sort();
-    console.info('   ', triplets.join('\n    '));
-  } else {
-    console.info('    (none)');
-  }
-
-  console.info('');
-  console.info('New / Unknown Terms:');
-  let unknowns = Object.keys(bc.unknownTerms);
-  if (unknowns.length) {
-    unknowns.sort();
-    console.warn('   ', unknowns.join('\n    '));
-  } else {
-    console.info('    (none)');
-  }
-
-  console.info('');
-  console.info('Unused Terms:');
-  let unuseds = Object.keys(bc.orphanTerms);
-  if (unuseds.length) {
-    unuseds.sort();
-    console.warn('   ', unuseds.join('\n    '));
-  } else {
-    console.info('    (none)');
-  }
-
-  console.info('');
-  console.info('Formats:');
-  if (bc.formats.length) {
-    let formats = bc.formats.slice();
-    formats.sort();
-    if (!formats[0]) {
-      formats[0] = '(bin)';
-    }
-    console.warn('   ', formats.join('\n    '));
-  } else {
-    console.info('    (none)');
-  }
-
-  // sort -u -k1 builds.tsv | rg -v '^#|^https?:' | rg -i arm
-  // cut -f1 builds.tsv | sort -u -k1 | rg -v '^#|^https?:' | rg -i arm
-}
-
-if (module === require.main) {
-  let times = [];
-  let now = Date.now();
-  main()
-    .then(async function () {
-      let then = Date.now();
-      let delta = then - now;
-      times.push(delta);
-      now = then;
-      await main();
-      then = Date.now();
-      delta = then - now;
-      times.push(delta);
-    })
-    .then(function () {
-      console.info('');
-      console.info('Run times');
-      for (let delta of times) {
-        let s = delta / 1000;
-        console.info(`    ${s}`);
-      }
-
-      function forceExit() {
-        console.warn(`warn: dangling event loop reference`);
-        process.exit(0);
-      }
-      let exitTimeout = setTimeout(forceExit, 250);
-      exitTimeout.unref();
-    })
-    .catch(function (err) {
-      console.error(err.stack || err);
-      process.exit(1);
-    });
-}

_webi/package-install.tpl.sh

@@ -26,7 +26,6 @@ __bootstrap_webi() {
     #PKG_ARCHES=
     #PKG_LIBCS=
     #PKG_FORMATS=
-    #PKG_LATEST=
     WEBI_PKG_DOWNLOAD=""
     WEBI_DOWNLOAD_DIR="${HOME}/Downloads"
     if command -v xdg-user-dir > /dev/null; then
@@ -126,13 +125,8 @@ __bootstrap_webi() {
         {
             echo ""
             echo "    $(t_err "Error: no '${PKG_NAME:-"Unknown Package"}@${WEBI_TAG:-"Unknown Tag"}' release for '${WEBI_OS:-"Unknown OS"}' (${WEBI_LIBC:-"Unknown Libc"}) on '${WEBI_ARCH:-"Unknown CPU"}' as one of '${WEBI_FORMATS:-"Unknown File Type"}'")"
-            echo ""
+            echo "      '$PKG_NAME' is available for '$PKG_OSES' ($PKG_LIBCS) on '$PKG_ARCHES' as one of '$PKG_FORMATS'"
-            echo "        Latest Version: ${PKG_LATEST}"
+            echo "      (check that the package name and version are correct)"
-            echo "        CPUs: $PKG_ARCHES"
-            echo "        OSes: $PKG_OSES"
-            echo "        libcs: $PKG_LIBCS"
-            echo "        Package Formats: $PKG_FORMATS"
-            echo "        (check that the package name and version are correct)"
 
             echo ""
             my_release_url="$(echo "$WEBI_RELEASES" | sed 's:?.*::')"
@@ -539,42 +533,18 @@ fn_show_welcome_back() { (
     echo ""
     echo "    $(t_attn 'Success')? Star it!   $(t_url 'https://github.com/webinstall/webi-installers')"
     echo "    $(t_attn 'Problem')? Report it: $(t_url 'https://github.com/webinstall/webi-installers/issues')"
-    echo "                        $(t_dim "(your system is") $(t_host "$(fn_get_os)")/$(t_host "$(uname -m)") $(t_dim "with") $(t_host "$(fn_get_libc)") $(t_dim "&") $(t_host "$(fn_get_http_client_name)")$(t_dim ")")"
+    echo "                        $(t_dim "(your system is") $(t_host "$(uname -s)")/$(t_host "$(uname -m)") $(t_dim "with") $(t_host "$(fn_get_libc)") $(t_dim "&") $(t_host "$(fn_get_http_client_name)")$(t_dim ")")"
 
     sleep 0.2
 ); }
 
-fn_get_os() { (
-    # Ex:
-    #     GNU/Linux
-    #     Android
-    #     Linux (often Alpine, musl)
-    #     Darwin
-    b_os="$(uname -o 2> /dev/null || echo '')"
-    b_sys="$(uname -s)"
-    if test -z "${b_os}" || test "${b_os}" = "${b_sys}"; then
-        # ex: 'Darwin' (and plain, non-GNU 'Linux')
-        echo "${b_sys}"
-        return 0
-    fi
-
-    if echo "${b_os}" | grep -q "${b_sys}"; then
-        # ex: 'GNU/Linux'
-        echo "${b_os}"
-        return 0
-    fi
-
-    # ex: 'Android/Linux'
-    echo "${b_os}/${b_sys}"
-); }
-
 fn_get_libc() { (
     # Ex:
     #     musl
     #     libc
     if ldd /bin/ls 2> /dev/null | grep -q 'musl' 2> /dev/null; then
         echo 'musl'
-    elif fn_get_os | grep -q 'GNU|Linux'; then
+    elif uname -o | grep -q 'GNU' || uname -s | grep -q 'Linux'; then
         echo 'gnu'
     else
         echo 'libc'
@@ -713,9 +683,9 @@ fn_curl() { (
 
 fn_get_target_triple_user_agent() { (
     # Ex:
-    #     x86_64/unknown GNU/Linux/5.15.107-2-pve gnu
+    #     x86_64/unknown Linux/5.15.107-2-pve gnu
     #     arm64/unknown Darwin/22.6.0 libc
-    echo "$(uname -m)/unknown $(fn_get_os)/$(uname -r) $(fn_get_libc)"
+    echo "$(uname -m)/unknown $(uname -s)/$(uname -r) $(fn_get_libc)"
 ); }
 
 fn_download_to_path() { (

_webi/packages.js

@@ -1 +0,0 @@
-projects.js

_webi/packages.js

@@ -0,0 +1,96 @@
+'use strict';
+
+var frontmarker = require('./frontmarker.js');
+var fs = require('fs');
+var path = require('path');
+
+var pkgs = module.exports;
+pkgs.create = function (Pkgs, basepath) {
+  if (!Pkgs) {
+    Pkgs = {};
+  }
+  if (!basepath) {
+    basepath = path.join(__dirname, '../');
+  }
+
+  Pkgs.all = function () {
+    return fs.promises.readdir(basepath).then(function (nodes) {
+      var items = [];
+      return nodes
+        .reduce(function (p, node) {
+          return p.then(function () {
+            return pkgs.get(node).then(function (meta) {
+              if (meta && '_' !== node[0]) {
+                meta.name = node;
+                items.push(meta);
+              }
+            });
+          });
+        }, Promise.resolve())
+        .then(function () {
+          return items;
+        });
+    });
+  };
+
+  Pkgs.get = function (node) {
+    return fs.promises.access(path.join(basepath, node)).then(function () {
+      return Pkgs._get(node);
+    });
+  };
+  Pkgs._get = function (node) {
+    var curlbash = path.join(basepath, node, 'install.sh');
+    var readme = path.join(basepath, node, 'README.md');
+    var winstall = path.join(basepath, node, 'install.ps1');
+    return Promise.all([
+      fs.promises
+        .readFile(readme, 'utf-8')
+        .then(function (txt) {
+          // TODO
+          return frontmarker.parse(txt);
+        })
+        .catch(function (e) {
+          if ('ENOENT' !== e.code && 'ENOTDIR' !== e.code) {
+            console.error("failed to read '" + node + "/README.md'");
+            console.error(e);
+          }
+        }),
+      fs.promises.access(curlbash).catch(function (e) {
+        // no *nix installer
+        curlbash = '';
+        if ('ENOENT' !== e.code && 'ENOTDIR' !== e.code) {
+          console.error("failed to parse '" + node + "/install.sh'");
+          console.error(e);
+        }
+      }),
+      fs.promises.access(winstall).catch(function (e) {
+        // no winstaller
+        winstall = '';
+        if ('ENOENT' !== e.code && 'ENOTDIR' !== e.code) {
+          console.error("failed to read '" + node + "/install.ps1'");
+          console.error(e);
+        }
+      }),
+    ]).then(function (items) {
+      var meta = items[0] || items[1];
+      if (!meta) {
+        // doesn't exist
+        return;
+      }
+      meta.windows = !!winstall;
+      meta.bash = !!curlbash;
+
+      return meta;
+    });
+  };
+
+  return Pkgs;
+};
+pkgs.create(pkgs);
+
+if (module === require.main) {
+  pkgs.all().then(function (data) {
+    console.info('package info:');
+    console.info(data);
+  });
+}

_webi/parallel.js

@@ -1,44 +0,0 @@
-'use strict';
-
-var Parallel = module.exports;
-Parallel.run = async function (limit, arr, fn) {
-  let index = 0;
-  let actives = [];
-  let results = [];
-  limit = Math.min(limit, arr.length);
-
-  function launch() {
-    let _index = index;
-    let p = fn(arr[_index], _index, arr);
-
-    // some tasks may be synchronous
-    // so we must push before removing
-    actives.push(p);
-
-    p.then(function _resolve(result) {
-      let i = actives.indexOf(p);
-      actives.splice(i, 1);
-      results[_index] = result;
-    });
-
-    index += 1;
-  }
-
-  // start tasks in parallel, up to limit
-  for (; actives.length < limit; ) {
-    launch();
-  }
-
-  // keep the task queue full
-  for (; index < arr.length; ) {
-    // wait for one task to complete
-    await Promise.race(actives);
-    // add one task again
-    launch();
-  }
-
-  // wait for all remaining tasks
-  await Promise.all(actives);
-
-  return results;
-};

_webi/projects.js

@@ -1,96 +0,0 @@
-'use strict';
-
-var frontmarker = require('./frontmarker.js');
-var fs = require('fs');
-var path = require('path');
-
-var pkgs = module.exports;
-pkgs.create = function (Pkgs, basepath) {
-  if (!Pkgs) {
-    Pkgs = {};
-  }
-  if (!basepath) {
-    basepath = path.join(__dirname, '../');
-  }
-
-  Pkgs.all = function () {
-    return fs.promises.readdir(basepath).then(function (nodes) {
-      var items = [];
-      return nodes
-        .reduce(function (p, node) {
-          return p.then(function () {
-            return pkgs.get(node).then(function (meta) {
-              if (meta && '_' !== node[0]) {
-                meta.name = node;
-                items.push(meta);
-              }
-            });
-          });
-        }, Promise.resolve())
-        .then(function () {
-          return items;
-        });
-    });
-  };
-
-  Pkgs.get = function (node) {
-    return fs.promises.access(path.join(basepath, node)).then(function () {
-      return Pkgs._get(node);
-    });
-  };
-  Pkgs._get = function (node) {
-    var curlbash = path.join(basepath, node, 'install.sh');
-    var readme = path.join(basepath, node, 'README.md');
-    var winstall = path.join(basepath, node, 'install.ps1');
-    return Promise.all([
-      fs.promises
-        .readFile(readme, 'utf-8')
-        .then(function (txt) {
-          // TODO
-          return frontmarker.parse(txt);
-        })
-        .catch(function (e) {
-          if ('ENOENT' !== e.code && 'ENOTDIR' !== e.code) {
-            console.error("failed to read '" + node + "/README.md'");
-            console.error(e);
-          }
-        }),
-      fs.promises.access(curlbash).catch(function (e) {
-        // no *nix installer
-        curlbash = '';
-        if ('ENOENT' !== e.code && 'ENOTDIR' !== e.code) {
-          console.error("failed to parse '" + node + "/install.sh'");
-          console.error(e);
-        }
-      }),
-      fs.promises.access(winstall).catch(function (e) {
-        // no winstaller
-        winstall = '';
-        if ('ENOENT' !== e.code && 'ENOTDIR' !== e.code) {
-          console.error("failed to read '" + node + "/install.ps1'");
-          console.error(e);
-        }
-      }),
-    ]).then(function (items) {
-      var meta = items[0] || items[1];
-      if (!meta) {
-        // doesn't exist
-        return;
-      }
-      meta.windows = !!winstall;
-      meta.bash = !!curlbash;
-
-      return meta;
-    });
-  };
-
-  return Pkgs;
-};
-pkgs.create(pkgs);
-
-if (module === require.main) {
-  pkgs.all().then(function (data) {
-    console.info('package info:');
-    console.info(data);
-  });
-}

_webi/releases.js

@@ -1,24 +1,46 @@
 'use strict';
 
-var Installers = module.exports;
-
 var Crypto = require('crypto');
 var Fs = require('node:fs/promises');
 var path = require('node:path');
+var request = require('@root/request');
+
+var _normalize = require('../_webi/normalize.js');
 
 var reInstallTpl = /\s*#?\s*{{ installer }}/;
 
+var Releases = module.exports;
+Releases.get = async function (pkgdir) {
+  let get;
+  try {
+    get = require(path.join(pkgdir, 'releases.js'));
+  } catch (e) {
+    let err = new Error('no releases.js for', pkgdir.split(/[\/\\]+/).pop());
+    err.code = 'E_NO_RELEASE';
+    throw err;
+  }
+
+  let all = await get(request);
+
+  return _normalize(all);
+};
+
 function padScript(txt) {
   return txt.replace(/^/g, '        ');
 }
 
 var BAD_SH_RE = /[<>'"`$\\]/;
-Installers.renderBash = async function (
+Releases.renderBash = async function (
-  baseurl,
+  pkgdir,
-  posixTemplate,
+  rel,
-  buildRequest,
+  { baseurl, pkg, tag, ver, os = '', arch = '', libc = '', formats },
-  buildMatch,
 ) {
+  if (!Array.isArray(formats)) {
+    formats = [];
+  }
+  if (!tag) {
+    tag = '';
+  }
   let installTxt = await Fs.readFile(path.join(pkgdir, 'install.sh'), 'utf8');
   installTxt = padScript(installTxt);
   var vers = rel.version.split('.');
@@ -68,7 +90,7 @@ Installers.renderBash = async function (
     .join(',')
     .replace(/'/g, '');
 
-  let webiChecksum = await Installers.getWebiShChecksum();
+  let webiChecksum = await Releases.getWebiShChecksum();
   let envReplacements = [
     ['WEBI_CHECKSUM', webiChecksum],
     ['WEBI_PKG', webiPkg],
@@ -77,7 +99,7 @@ Installers.renderBash = async function (
     ['WEBI_ARCH', arch],
     ['WEBI_LIBC', libc],
     ['WEBI_TAG', tag],
-    ['WEBI_RELEASES', `${baseurl}${releaseUrl}`],
+    ['WEBI_RELEASES', `${baseurl}/${releaseUrl}`],
     ['WEBI_CSV', releaseCsv],
     ['WEBI_VERSION', rel.version],
     ['WEBI_MAJOR', v.major],
@@ -94,11 +116,10 @@ Installers.renderBash = async function (
     ['WEBI_PKG_PATHNAME', pkgFile],
     ['WEBI_PKG_FILE', pkgFile], // TODO replace with pathname
     ['PKG_NAME', pkg],
-    ['PKG_OSES', (rel.oses || []).join(' ')],
+    ['PKG_OSES', rel.oses],
-    ['PKG_ARCHES', (rel.arches || []).join(' ')],
+    ['PKG_ARCHES', rel.arches],
-    ['PKG_LIBCS', (rel.libcs || []).join(' ')],
+    ['PKG_LIBCS', rel.libcs],
-    ['PKG_FORMATS', (rel.formats || []).join(' ')],
+    ['PKG_FORMATS', (rel.formats || []).join(',')],
-    ['PKG_LATEST', latest],
   ];
 
   for (let env of envReplacements) {
@@ -126,18 +147,17 @@ Installers.renderBash = async function (
   return tplTxt;
 };
 
-/**
+Releases.renderPowerShell = async function (
- * @param {String} pkgdir
+  pkgdir,
- * @param {String} baseurl
+  rel,
- * @param {BuildRequest} buildRequest
+  { baseurl, pkg, tag, ver, os, arch, libc = '', formats },
- * @param {BuildMatch} buildMatch
- */
-Installers.renderPowerShell = async function (
-  baseurl,
-  pwshTemplate,
-  buildRequest,
-  buildMatch,
 ) {
+  if (!Array.isArray(formats)) {
+    formats = [];
+  }
+  if (!tag) {
+    tag = '';
+  }
   let installTxt = await Fs.readFile(path.join(pkgdir, 'install.ps1'), 'utf8');
   installTxt = padScript(installTxt);
   /*
@@ -204,7 +224,7 @@ var _webiShMeta = {
   checksum: '',
   mtime: 0,
 };
-Installers.getWebiShChecksum = async function () {
+Releases.getWebiShChecksum = async function () {
   let now = Date.now();
   let ago = now - _webiShMeta.updated_at;
   if (ago <= _webiShMeta.stale) {

_webi/serve-installer.js

@@ -1,250 +1,141 @@
 'use strict';
 
-var InstallerServer = module.exports;
+var Installers = module.exports;
 
-let Fs = require('fs/promises');
+var Fs = require('fs/promises');
-let Path = require('path');
+var path = require('path');
 
-let HostTargets = require('./build-classifier/host-targets.js');
+var uaDetect = require('./ua-detect.js');
-let Builds = require('./builds.js');
+var packages = require('./packages.js');
-let Installers = require('./installers.js');
+var Releases = require('./releases.js');
 
-InstallerServer.INSTALLERS_DIR = Path.join(__dirname, '..');
+// handlers caching and transformation, probably should be broken down
+var getReleases = require('./transform-releases.js');
 
-/**
+Installers.INSTALLERS_DIR = path.join(__dirname, '..');
- * @typedef BuildRequest
+Installers.serveInstaller = async function (
- * @prop {String} unameAgent
- * @prop {String} projectName
- * @prop {String} tag
- * @prop {Array<String>} formats
- */
-
-InstallerServer.serveInstaller = async function (
   baseurl,
-  ua, // TODO nix
+  ua,
-  unameAgent,
+  pkg,
-  pkg, // TODO nix
-  projectName,
   tag,
-  ext, // TODO nix
+  ext,
-  installerType,
   formats,
+  libc,
 ) {
-  if (!unameAgent) {
+  let [rel, opts] = await Installers.helper({
-    unameAgent = ua;
+    ua,
-  }
+    pkg,
-  if (!projectName) {
-    projectName = pkg;
-  }
-  if (!installerType) {
-    installerType = ext;
-  }
-  let buildRequest = {
-    projectName,
     tag,
-    unameAgent,
     formats,
-  };
+    libc,
+  });
+  Object.assign(opts, {
+    baseurl,
+  });
 
-  let hostTarget = {};
+  var pkgdir = path.join(Installers.INSTALLERS_DIR, pkg);
-  let buildMatch;
-  {
-    let terms = unameAgent.split(/[\s\/]+/g);
-    void HostTargets.termsToTarget(hostTarget, terms);
-    buildMatch = await InstallerServer.getBuild(buildRequest, hostTarget);
-  }
-  Object.assign(tmplParams, { baseurl, });
-
-  console.log('dbg: buildPkg', rel);
-  console.log('dbg: tmplParams', tmplParams);
-
-  var pkgdir = Path.join(InstallerServer.INSTALLERS_DIR, projectName);
   if ('ps1' === ext) {
-    return Installers.renderPowerShell(pkgdir, rel, tmplParams);
+    return Releases.renderPowerShell(pkgdir, rel, opts);
   }
-  return Installers.renderBash(pkgdir, rel, tmplParams);
+  return Releases.renderBash(pkgdir, rel, opts);
 };
+Installers.helper = async function ({ ua, pkg, tag, formats, libc }) {
+  // TODO put some of this in a middleware? or common function?
 
-// TODO put some of this in a middleware? or common function?
+  // TODO maybe move package/version/lts/channel detection into getReleases
-// TODO maybe move package/version/lts/channel detection into getReleases
+  var ver = tag.replace(/^v/, '');
-/**
+  var lts;
- * @param {BuildRequest}
+  var channel;
- * @returns {BuildMatch}
- */
-InstallerServer.helper = async function ({
-  unameAgent,
-  projectName,
-  tag,
-  formats,
-}) {
-  console.log(`dbg: Installer User-Agent: ${unameAgent}`);
 
-  let releaseTarget = toReleaseTarget(tag);
+  switch (ver) {
-  let hostFormats = formats;
+    case 'latest':
-  let terms = unameAgent.split(/[\s\/]+/g);
+      ver = '';
-  let hostTarget = {};
+      channel = 'stable';
-  try {
+      break;
-    void HostTargets.termsToTarget(hostTarget, terms);
+    case 'lts':
-  } catch (e) {
+      lts = true;
-    // if we can't guarantee the results...
+      channel = 'stable';
-    // "in the face of ambiguity, refuse the temptation to guess"
+      ver = '';
-    throw e;
+      break;
-  }
+    case 'stable':
-  console.log(`dbg: Installer Host Target:`);
+      channel = 'stable';
-  console.log(hostTarget);
+      ver = '';
-
+      break;
-  if (!hostTarget.os) {
+    case 'beta':
-    throw new Error(`OS could not be identified by User-Agent '${unameAgent}'`);
+      channel = 'beta';
+      ver = '';
+      break;
+    case 'dev':
+      channel = 'dev';
+      ver = '';
+      break;
   }
 
-  console.log(`dbg: Get Project Installer Type for '${projectName}':`);
+  var myOs = uaDetect.os(ua);
-  let proj = await Builds.getProjectType(projectName);
+  var myArch = uaDetect.arch(ua);
-  console.log(proj);
+  var myLibc;
-
+  if (libc) {
-  let validTypes = ['alias', 'selfhosted', 'valid'];
+    myLibc = uaDetect.libc(libc);
-  if (!validTypes.includes(proj.type)) {
-    let msg = `'${projectName}' doesn't have an installer: '${proj.type}': '${proj.detail}'`;
-    let err = new Error(msg);
-    err.code = 'ENOENT';
-    throw err;
   }
-  if (proj.type === 'alias') {
+  if (!myLibc) {
-    projectName = proj.detail;
+    myLibc = uaDetect.libc(ua);
+  }
+  if (!myLibc) {
+    myLibc = 'libc';
   }
 
-  let tmplParams = {
+  let cfg = await packages.get(pkg);
-    pkg: projectName,
+  let releaseQuery = {
-    tag: tag,
+    pkg: cfg.alias || pkg,
-    os: hostTarget.os,
+    ver,
-    arch: hostTarget.arch,
+    os: myOs,
-    libc: hostTarget.libc,
+    arch: myArch,
-    formats: hostFormats,
+    libc: myLibc,
+    lts,
+    channel,
+    // TODO use formats for sorting, not exclusion
+    // (it's better to install xz or report an error to install zip)
+    formats,
     limit: 1,
   };
-  Object.assign(tmplParams, releaseTarget);
-  console.log('tmplParams', tmplParams);
 
-  let errPackage = {
+  let rels = await getReleases(releaseQuery);
-    name: 'doesntexist.ext',
+
-    version: '0.0.0',
+  var rel = rels.releases[0];
-    lts: '-',
+  var opts = {
-    channel: 'error',
+    pkg: cfg.alias || pkg,
-    date: '1970-01-01',
+    ver,
-    os: hostTarget.os || '-',
+    tag,
-    arch: hostTarget.arch || '-',
+    os: myOs,
-    libc: hostTarget.libc || '-',
+    arch: myArch,
-    ext: 'err',
+    libc: myLibc,
-    download: 'https://example.com/doesntexist.ext',
+    lts,
-    comment:
+    channel,
-      'No matches found. Could be bad or missing version info' +
+    formats,
-      ',' +
+    limit: 1,
-      "Check query parameters. Should be something like '/api/releases/{package}@{version}.tab?os={macos|linux|windows|-}&arch={amd64|x86|aarch64|arm64|armv7l|-}&libc={musl|gnu|msvc|libc|static}&limit=10'",
   };
-
+  rel = Object.assign(
-  if (proj.type === 'selfhosted') {
+    {
-    return [errPackage, tmplParams];
+      oses: rels.oses,
-  }
+      arches: rels.arches,
-
+      libcs: rels.libcs,
-  let projInfo = await Builds.getPackage({
+      formats: rels.formats,
-    name: projectName,
+    },
-    date: new Date(),
+    rel,
-  });
-  let latest = projInfo.versions[0];
-  Object.assign(tmplParams, { latest });
-  //console.log('projInfo', projInfo);
-
-  let buildTargetInfo = {
-    triplets: projInfo.triplets,
-    oses: projInfo.oses,
-    arches: projInfo.arches,
-    libcs: projInfo.libcs,
-    formats: projInfo.formats,
-  };
-
-  let hasOs = projInfo.oses.includes(hostTarget.os);
-  if (!hasOs) {
-    let pkg1 = Object.assign(buildTargetInfo, errPackage);
-    return [pkg1, tmplParams];
-  }
-
-  let targetRelease = Builds.findMatchingPackages(
-    projInfo,
-    hostTarget,
-    releaseTarget,
   );
-  // { triplet: `${os}-${arch}-${libc}`, packages: targetPackages
-  // , latest: projInfo.versions[0], versions: matchInfo
-  // }
 
-  if (!targetRelease?.packages) {
+  return [rel, opts];
-    let pkg1 = Object.assign(buildTargetInfo, errPackage);
-    return [pkg1, tmplParams];
-  }
-
-  let buildPkg = Builds.selectPackage(targetRelease.packages, hostFormats);
-  let ext = buildPkg.ext || '.exe';
-  if (ext.startsWith('.')) {
-    ext = ext.slice(1);
-  }
-
-  let version = targetRelease.version;
-  if (version.startsWith('v')) {
-    version = version.slice(1);
-  }
-  let ver = version;
-
-  buildPkg = Object.assign(buildTargetInfo, buildPkg, { ext, version, tag });
-  Object.assign(tmplParams, { tag, ext, ver, version });
-  //console.log(`VERSION: ${version}`, ext);
-  return [buildPkg, tmplParams];
 };
 
-let channelNames = [
+var CURL_PIPE_PS1_BOOT = path.join(__dirname, 'curl-pipe-bootstrap.tpl.ps1');
-  'stable',
+var CURL_PIPE_SH_BOOT = path.join(__dirname, 'curl-pipe-bootstrap.tpl.sh');
-  // 'hotfix',
-  'latest',
-  'rc',
-  'preview',
-  'pre',
-  'dev',
-  'beta',
-  'alpha',
-];
-
-function toReleaseTarget(tag) {
-  tag = tag.replace(/^v/, '');
-
-  let releaseTarget = {
-    channel: '',
-    lts: false,
-    version: '',
-  };
-
-  if (tag === 'lts') {
-    releaseTarget.lts = true;
-    releaseTarget.channel = 'stable';
-  } else if (channelNames.includes(tag)) {
-    releaseTarget.channel = tag;
-  } else {
-    releaseTarget.version = tag;
-  }
-
-  return releaseTarget;
-}
-
-var CURL_PIPE_PS1_BOOT = Path.join(__dirname, 'curl-pipe-bootstrap.tpl.ps1');
-var CURL_PIPE_SH_BOOT = Path.join(__dirname, 'curl-pipe-bootstrap.tpl.sh');
 var BAD_SH_RE = /[<>'"`$\\]/;
 
-InstallerServer.getPosixCurlPipeBootstrap = async function ({
+Installers.getPosixCurlPipeBootstrap = async function ({ baseurl, pkg, ver }) {
-  baseurl,
-  pkg,
-  ver,
-}) {
   let bootTxt = await Fs.readFile(CURL_PIPE_SH_BOOT, 'utf8');
 
   var webiPkg = [pkg, ver].filter(Boolean).join('@');
-  var webiChecksum = await Installers.getWebiShChecksum();
+  var webiChecksum = await Releases.getWebiShChecksum();
   var envReplacements = [
     ['WEBI_PKG', webiPkg],
     ['WEBI_HOST', baseurl],
@@ -255,6 +146,7 @@ InstallerServer.getPosixCurlPipeBootstrap = async function ({
     let name = env[0];
     let value = env[1];
 
+    // TODO create REs once, in higher scope
     let envRe = new RegExp(
       `^[ \\t]*#?[ \\t]*(export[ \\t])?[ \\t]*(${name})=.*`,
       'm',
@@ -272,7 +164,7 @@ InstallerServer.getPosixCurlPipeBootstrap = async function ({
   return bootTxt;
 };
 
-InstallerServer.getPwshCurlPipeBootstrap = async function ({
+Installers.getPwshCurlPipeBootstrap = async function ({
   baseurl,
   pkg,
   ver,
@@ -281,7 +173,7 @@ InstallerServer.getPwshCurlPipeBootstrap = async function ({
   let bootTxt = await Fs.readFile(CURL_PIPE_PS1_BOOT, 'utf8');
 
   var webiPkg = [pkg, ver].filter(Boolean).join('@');
-  //var webiChecksum = await InstallerServer.getWebiPs1Checksum();
+  //var webiChecksum = await Releases.getWebiPs1Checksum();
   var envReplacements = [
     ['Env:WEBI_PKG', webiPkg],
     ['Env:WEBI_HOST', baseurl],
@@ -302,6 +194,9 @@ InstallerServer.getPwshCurlPipeBootstrap = async function ({
     let tplRe = new RegExp(`{{ (${name}) }}`, 'g');
     bootTxt = bootTxt.replace(tplRe, `${value}`);
 
+    // let envRe = new RegExp(`^[ \\t]*#?[ \\t]*($$${name})[ \\t]*=.*`, 'im');
+    // bootTxt = bootTxt.replace(envRe, `$$${name} = '${value}'`);
+
     let setRe = new RegExp(
       `(#[ \\t]*)?(\\$${name})[ \\t]*=[ \\t]['"].*['"][ \\t]`,
       'im',

_webi/test.js

@@ -32,8 +32,7 @@ if (/\b-?-h(elp)?\b/.test(process.argv.join(' '))) {
 var os = require('os');
 var fs = require('fs');
 var path = require('path');
-var Releases = require('./transform-releases.js');
+var Releases = require('./releases.js');
-var Installers = require('./installers.js');
 var ServeInstaller = require('./serve-installer.js');
 
 var pkg = process.argv[2].split('@');
@@ -81,29 +80,20 @@ Releases.get(path.join(process.cwd(), pkgdir)).then(async function (all) {
   }
   var formats = ['exe', 'xz', 'tar', 'zip', 'git'];
 
-  let unameAgent = `${nodeOs}/${nodeOsRelease} ${nodeArch}/unknown ${nodeLibc}`;
-  console.log(`DEBUG: ${unameAgent}`);
   let [rel, opts] = await ServeInstaller.helper({
-    unameAgent: unameAgent,
+    ua: `${nodeOs}/${nodeOsRelease} ${nodeArch}/unknown ${nodeLibc}`,
-    projectName: pkgname,
+    pkg: pkgname,
     tag: pkgtag || '',
     formats: formats,
     libc: nodeLibc,
   });
-  console.log('DEBUG opts:');
-  console.log(opts);
   Object.assign(
-    rel,
     {
-      version: '{test-version}',
+      ver: '',
-      git_tag: '{test-git-tag}',
-      git_commit_hash: '{test-git-commit-hash}',
       lts: null,
-      channel: '{test-channel}',
+      channel: '',
-      date: '1970-01-01T00:00:00Z',
+      os: '',
-      os: '{test-os}',
+      arch: '',
-      arch: '{test-arch}',
-      ext: '{test-ext}',
       limit: 0,
     },
     opts,
@@ -126,8 +116,8 @@ Releases.get(path.join(process.cwd(), pkgdir)).then(async function (all) {
   console.info('');
 
   return Promise.all([
-    Installers.renderBash(pkgdir, rel, opts).catch(function () {}),
+    Releases.renderBash(pkgdir, rel, opts).catch(function () {}),
-    Installers.renderPowerShell(pkgdir, rel, opts).catch(function () {}),
+    Releases.renderPowerShell(pkgdir, rel, opts).catch(function () {}),
   ]).then(function (scripts) {
     var bashTxt = scripts[0];
     var ps1Txt = scripts[1];

_webi/transform-releases.js

@@ -1,10 +1,7 @@
 'use strict';
 
-var Releases = module.exports;
-
 var path = require('path');
-var request = require('@root/request');
+var Releases = require('./releases.js');
-var _normalize = require('./normalize.js');
 
 var cache = {};
 //var staleAge = 5 * 1000;
@@ -14,21 +11,6 @@ var expiredAge = 15 * 60 * 1000;
 
 let installerDir = path.join(__dirname, '..');
 
-Releases.get = async function (pkgdir) {
-  let get;
-  try {
-    get = require(path.join(pkgdir, 'releases.js'));
-  } catch (e) {
-    let err = new Error('no releases.js for', pkgdir.split(/[\/\\]+/).pop());
-    err.code = 'E_NO_RELEASE';
-    throw err;
-  }
-
-  let all = await get(request);
-
-  return _normalize(all);
-};
-
 // TODO needs a proper test, and more accurate (though perhaps far less simple) code
 function createFormatsSorter(formats) {
   return function sortByVerExt(a, b) {
@@ -140,7 +122,7 @@ async function getCachedReleases(pkg) {
           cache[pkg].all = all;
           complete = true;
         }),
-      sleep(15000).then(function () {
+      sleep(5000).then(function () {
         if (complete) {
           return;
         }
@@ -260,7 +242,7 @@ async function filterReleases(
   return sortedRels.slice(0, limit || 1000);
 }
 
-Releases.getReleases = function ({
+module.exports = function getReleases({
   _count,
   pkg,
   ver,
@@ -308,7 +290,7 @@ Releases.getReleases = function ({
         if (_count < 1) {
           // Apple Silicon M1 hacky-do workaround fix
           if ('macos' === os && 'arm64' === arch) {
-            return Releases.getReleases({
+            return getReleases({
               pkg,
               ver,
               os,
@@ -322,7 +304,7 @@ Releases.getReleases = function ({
           }
           // Windows ARM hacky-do workaround fix
           if ('windows' === os && 'arm64' === arch) {
-            return Releases.getReleases({
+            return getReleases({
               pkg,
               ver,
               os,
@@ -336,7 +318,7 @@ Releases.getReleases = function ({
           }
           // Raspberry Pi 3+ on Ubuntu arm64 (via Bionic?)
           if ('linux' === os && 'arm64' === arch) {
-            return Releases.getReleases({
+            return getReleases({
               _count: _count + 1,
               pkg,
               ver,
@@ -351,7 +333,7 @@ Releases.getReleases = function ({
           }
           // armv7 can run armv6
           if ('linux' === os && 'armv7l' === arch) {
-            return Releases.getReleases({
+            return getReleases({
               _count: _count + 1,
               pkg,
               ver,
@@ -369,7 +351,7 @@ Releases.getReleases = function ({
           // Raspberry Pi 3+ on Raspbian arm7 (not Ubuntu arm64)
           // hail mary
           if ('linux' === os && 'armv7l' === arch) {
-            return Releases.getReleases({
+            return getReleases({
               _count: _count + 1,
               pkg,
               ver,

ffprobe/releases.js

@@ -0,0 +1,45 @@
+'use strict';
+
+var path = require('path');
+
+var github = require('../_common/github.js');
+var owner = 'eugeneware';
+var repo = 'ffmpeg-static';
+
+module.exports = function (request) {
+  return github(request, owner, repo).then(function (all) {
+    all.releases = all.releases
+      .filter(function (rel) {
+        let isFfmpeg = rel.name.includes('ffmpeg');
+        if (!isFfmpeg) {
+          return;
+        }
+
+        // remove README and LICENSE
+        return !['.README', '.LICENSE'].includes(path.extname(rel.name));
+      })
+      .map(function (rel) {
+        rel.version = rel.version.replace(/^b/, '');
+
+        if (/win32/.test(rel.name)) {
+          rel.os = 'windows';
+          rel.ext = 'exe';
+        }
+        if (/ia32/.test(rel.name)) {
+          rel.arch = '386';
+        } else if (/x64/.test(rel.name)) {
+          rel.arch = 'amd64';
+        }
+
+        return rel;
+      });
+    return all;
+  });
+};
+
+if (module === require.main) {
+  module.exports(require('@root/request')).then(function (all) {
+    all = require('../_webi/normalize.js')(all);
+    console.info(JSON.stringify(all));
+  });
+}

goreleaser/releases.js

@@ -6,7 +6,6 @@ var repo = 'goreleaser';
 
 module.exports = function (request) {
   return github(request, owner, repo).then(function (all) {
-    all._names = ['goreleaser', '1'];
     return all;
   });
 };

ollama/releases.js

@@ -7,25 +7,19 @@ var repo = 'ollama';
 module.exports = async function (request) {
   let all = await github(request, owner, repo);
 
+  // TODO why are the 0.0.x releases sorting so high?
   let releases = [];
   for (let rel of all.releases) {
-    // this is a janky, sudo-wantin' .app
+    let isLow = rel.version.startsWith('v0.0.');
-    let isJank = rel.name.startsWith('Ollama-darwin');
+    if (isLow) {
-    if (isJank) {
       continue;
     }
 
-    let isUniversal = rel.name === 'ollama-darwin';
-    if (isUniversal) {
-      let x64 = Object.assign({ arch: 'x86_64' }, rel);
-      releases.push(x64);
-
-      rel.arch = 'aarch64';
-    }
     releases.push(rel);
   }
   all.releases = releases;
 
+  all._names = ['Ollama', 'ollama'];
   return all;
 };
 

package.json

@@ -5,7 +5,6 @@
   "description": "webinstall script repository",
   "main": "_webi/",
   "scripts": {
-    "bump": "npm version -m \"chore(release): bump to v%s\"",
     "fmt": "npm run prettier && npm run shfmt && npm run pwsh-fmt",
     "lint": "npm run shellcheck && npm run jshint && npm run pwsh-lint",
     "prepare": "npm run tooling-init && npm run git-hooks-init",

pwsh/releases.js

@@ -32,7 +32,7 @@ module.exports = function (request) {
       let isMusl = rel.download.match(/(\b|_)(musl|alpine)(\b|_)/i);
       if (isMusl) {
         // not a fully static build, not gnu-compatible
-        rel.libc = 'musl';
+        rel.arch = 'musl';
       }
 
       return true;

sd/install.sh

@@ -20,22 +20,10 @@ __init_sd() {
 
     # pkg_install must be defined by every package
     pkg_install() {
+        # ~/.local/opt/sd-v0.99.9/bin
+        mkdir -p "$(dirname "$pkg_src_cmd")"
         # mv ./sd-*/sd "$pkg_src_cmd"
-        if test -f sd-*; then
+        mv sd-* "$pkg_src_cmd"
-            # ~/.local/opt/sd-v0.99.9/bin
-            mkdir -p "$(dirname "$pkg_src_cmd")"
-            mv sd-* "$pkg_src_cmd"
-        elif test -f sd-*/sd; then
-            # ~/.local/opt/sd-v0.99.9/bin
-            mkdir -p "$(dirname "$pkg_src_cmd")"
-            mv sd-*/sd "$pkg_src_cmd"
-            if test -f sd-*/sd.1; then
-                mkdir -p "$pkg_src_dir/share/man/man1"
-                mv sd-*/sd.1 "$pkg_src_dir/share/man/man1"
-            fi
-        elif test -d sd-*/bin; then
-            mv sd-* "$pkg_src_dir"
-        fi
     }
 
     # pkg_get_current_version is recommended, but (soon) not required

shellcheck/releases.js

@@ -6,6 +6,27 @@ var repo = 'shellcheck';
 
 module.exports = function (request) {
   return github(request, owner, repo).then(function (all) {
+    all.releases = all.releases.filter(function (rel) {
+      // don't include meta versions as actual versions
+      if (
+        ['latest', 'stable'].includes(rel.version) ||
+        'v' !== rel.version[0]
+      ) {
+        return false;
+      }
+      return true;
+    });
+
+    all.releases.forEach(function (rel) {
+      // if there is no os or arch or source designation, and it's a .zip, it's Windows amd64
+      if (
+        !/(darwin|mac|linux|x86_64|arm|src|source)/i.test(rel.name) &&
+        /\.zip$/.test(rel.name)
+      ) {
+        rel.os = 'windows';
+        rel.arch = 'amd64';
+      }
+    });
     return all;
   });
 };

watchexec/releases.js

@@ -6,13 +6,6 @@ var repo = 'watchexec';
 
 module.exports = function (request) {
   return github(request, owner, repo).then(function (all) {
-    let builds = [];
-    for (let build of all.releases) {
-      build.version = build.version.replace(/^cli-/, '');
-      builds.push(build);
-    }
-    all.releases = builds;
-
     return all;
   });
 };

webi/releases.js

@@ -0,0 +1,43 @@
+'use strict';
+
+module.exports = async function () {
+  return {
+    releases: [
+      {
+        name: 'webi.tar',
+        version: '0.0.0',
+        lts: false,
+        channel: 'stable',
+        date: '',
+        os: 'linux',
+        arch: 'amd64',
+        ext: 'tar',
+        download: '',
+      },
+      {
+        name: 'webi.tar',
+        version: '0.0.0',
+        lts: false,
+        channel: 'stable',
+        date: '',
+        os: 'macos',
+        arch: 'amd64',
+        ext: 'tar',
+        download: '',
+      },
+      {
+        name: 'webi.tar',
+        version: '0.0.0',
+        lts: false,
+        channel: 'stable',
+        date: '',
+        os: 'windows',
+        arch: 'amd64',
+        ext: 'tar',
+        download: '',
+      },
+    ],
+    formats: [],
+    downloads: '',
+  };
+};

zig/releases.js

@@ -1,7 +1,6 @@
 'use strict';
 
 var NON_BUILDS = ['bootstrap', 'src'];
-var ODDITIES = NON_BUILDS.concat(['armv6kz-linux']);
 
 module.exports = function (request) {
   return request({
@@ -28,8 +27,8 @@ module.exports = function (request) {
           return;
         }
 
-        let isOdd = ODDITIES.includes(platform);
+        let isNonBuild = NON_BUILDS.includes(platform);
-        if (isOdd) {
+        if (isNonBuild) {
           return;
         }
 

zoxide/releases.js

@@ -6,6 +6,11 @@ var repo = 'zoxide';
 
 module.exports = function (request) {
   return github(request, owner, repo).then(function (all) {
+    all.releases.forEach(function (rel) {
+      if (/-arm-/.test(rel.download)) {
+        rel.arch = 'armv6l';
+      }
+    });
     return all;
   });
 };