WIP: ref(installer): use new names for template vars

.github/workflows/node.js.yml

@@ -23,6 +23,9 @@ jobs:
           sh ./_scripts/install-ci-deps
           echo "${HOME}/.local/bin" >> $GITHUB_PATH
           echo "${HOME}/.local/opt/pwsh" >> $GITHUB_PATH
+      - run: |
+          git submodule init
+          git submodule update
       - run: shfmt --version
       - run: shellcheck -V
       - run: node --version

_webi/builds-cacher.js

@@ -5,9 +5,12 @@ var BuildsCacher = module.exports;
 let Fs = require('node:fs/promises');
 let Path = require('node:path');
 
-let request = require('@root/request');
+let HostTargets = require('./build-classifier/host-targets.js');
+let Lexver = require('./build-classifier/lexver.js');
 let Triplet = require('./build-classifier/triplet.js');
 
+let request = require('@root/request');
+
 var ALIAS_RE = /^alias: (\w+)$/m;
 
 var LEGACY_ARCH_MAP = {
@@ -78,7 +81,15 @@ async function readFirstBytes(path) {
 }
 
 let promises = {};
-async function getLatestBuilds(Releases, cacheDir, name, date) {
+async function getLatestBuilds(Releases, installersDir, cacheDir, name, date) {
+  if (!Releases) {
+    Releases = require(`${installersDir}/${name}/releases.js`);
+  }
+  // TODO update all releases files with module.exports.xxxx = 'foo';
+  if (!Releases.latest) {
+    Releases.latest = Releases;
+  }
+
   let id = `${cacheDir}/${name}`;
   if (!promises[id]) {
     promises[id] = Promise.resolve();
@@ -120,6 +131,9 @@ async function getLatestBuildsInner(Releases, cacheDir, name, date) {
 }
 
 BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
+  let installersDir = installers;
+  let cacheDir = caches;
+
   if (!ALL_TERMS) {
     ALL_TERMS = Triplet.TERMS_PRIMARY_MAP;
   }
@@ -128,16 +142,19 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
   bc.usedTerms = {};
   bc.orphanTerms = Object.assign({}, ALL_TERMS);
   bc.unknownTerms = {};
+  bc.formats = [];
   bc._triplets = {};
   bc._targetsByBuildIdCache = {};
   bc._caches = {};
   bc._staleAge = 15 * 60 * 1000;
+  bc._allFormats = {};
+  bc._allTriplets = {};
 
   for (let term of TERMS_META) {
     delete bc.orphanTerms[term];
   }
 
-  bc.getPackages = async function () {
+  bc.getProjectsByType = async function () {
     let dirs = {
       hidden: {},
       errors: {},
@@ -147,66 +164,11 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
       valid: {},
     };
 
-    let entries = await Fs.readdir(installers, { withFileTypes: true });
+    let entries = await Fs.readdir(installersDir, { withFileTypes: true });
     for (let entry of entries) {
-      // skip non-installer dirs
-      if (entry.isSymbolicLink()) {
-        dirs.alias[entry.name] = 'symlink';
-        continue;
-      }
-      if (!entry.isDirectory()) {
-        dirs.hidden[entry.name] = '!directory';
-        continue;
-      }
-      if (entry.name === 'node_modules') {
-        dirs.hidden[entry.name] = 'node_modules';
-        continue;
-      }
-      if (entry.name.startsWith('_')) {
-        dirs.hidden[entry.name] = '_*';
-        continue;
-      }
-      if (entry.name.startsWith('.')) {
-        dirs.hidden[entry.name] = '.*';
-        continue;
-      }
-      if (entry.name.startsWith('~')) {
-        dirs.hidden[entry.name] = '~*';
-        continue;
-      }
-      if (entry.name.endsWith('~')) {
-        dirs.hidden[entry.name] = '*~';
-        continue;
-      }
-
-      // skip invalid installers
-      let path = Path.join(installers, entry.name);
-      let head = await getPartialHeader(path);
-      if (!head) {
-        dirs.invalid[entry.name] = '!README.md';
-        continue;
-      }
-
-      let alias = head.match(ALIAS_RE);
-      if (alias) {
-        dirs.alias[entry.name] = true;
-        continue;
-      }
-
-      let releasesPath = Path.join(path, 'releases.js');
-      let releases;
-      try {
-        releases = require(releasesPath);
-      } catch (err) {
-        if (err.code !== 'MODULE_NOT_FOUND') {
-          dirs.errors[entry.name] = err;
-          continue;
-        }
-        if (err.requireStack.length === 2) {
-          dirs.selfhosted[entry.name] = true;
-          continue;
-        }
-        // err.requireStack.length > 1
+      let meta = await bc.getProjectTypeByEntry(entry);
+      if (meta.type === 'not_found') {
+        let err = meta.detail;
         console.error('');
         console.error('PROBLEM');
         console.error(`    ${err.message}`);
@@ -218,27 +180,95 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
           '[SANITY FAIL] should never have missing modules in prod',
         );
       }
-
-      dirs.valid[entry.name] = true;
+      dirs[meta.type][entry.name] = meta.detail;
     }
 
     return dirs;
   };
 
+  /**
+   * Get project type and detail - alias, selfhosted, valid (and the invalids)
+   * @param {String} name - filename
+   */
+  bc.getProjectType = async function (name) {
+    let filepath = Path.join(installersDir, name);
+    let entry;
+    try {
+      entry = await Fs.lstat(filepath);
+      Object.assign(entry, { name: name });
+    } catch (e) {
+      return { type: 'errors', detail: 'not found' };
+    }
+    let info = await bc.getProjectTypeByEntry(entry);
+
+    return info;
+  };
+
+  /**
+   * Get project type and detail - alias, selfhosted, valid (and the invalids)
+   * @param {fs.Stats|fs.Dirent} entry
+   */
+  bc.getProjectTypeByEntry = async function (entry) {
+    let path = Path.join(installersDir, entry.name);
+
+    // skip non-installer dirs
+    if (entry.isSymbolicLink()) {
+      let link = await Fs.readlink(path);
+      return { type: 'alias', detail: link };
+    }
+
+    if (!entry.isDirectory()) {
+      return { type: 'hidden', detail: '!directory' };
+    }
+    if (entry.name === 'node_modules') {
+      return { type: 'hidden', detail: 'node_modules' };
+    }
+    if (entry.name.startsWith('_')) {
+      return { type: 'hidden', detail: '_*' };
+    }
+    if (entry.name.startsWith('.')) {
+      return { type: 'hidden', detail: '.*' };
+    }
+    if (entry.name.startsWith('~')) {
+      return { type: 'hidden', detail: '~*' };
+    }
+    if (entry.name.endsWith('~')) {
+      return { type: 'hidden', detail: '*~' };
+    }
+
+    // skip invalid installers
+    let head = await getPartialHeader(path);
+    if (!head) {
+      return { type: 'invalid', detail: '!README.md' };
+    }
+
+    let alias = head.match(ALIAS_RE);
+    if (alias) {
+      let link = alias[1];
+      return { type: 'alias', detail: link };
+    }
+
+    let releasesPath = Path.join(path, 'releases.js');
+    try {
+      void require(releasesPath);
+    } catch (err) {
+      if (err.code !== 'MODULE_NOT_FOUND') {
+        return { type: 'errors', detail: err };
+      }
+
+      if (err.message.includes(`Cannot find module '${releasesPath}'`)) {
+        return { type: 'selfhosted', detail: true };
+      }
+
+      return { type: 'not_found', detail: err };
+    }
+
+    return { type: 'valid', detail: true };
+  };
+
   // Typically a package is organized by release (ex: go has 1.20, 1.21, etc),
   // but we will organize by the build (ex: go1.20-darwin-arm64.tar.gz, etc).
-  bc.getBuilds = async function ({ Releases, name, date }) {
-    let cacheDir = caches;
-    let installerDir = installers;
-
-    if (!Releases) {
-      Releases = require(`${installerDir}/${name}/releases.js`);
-    }
-    // TODO update all releases files with object export
-    if (!Releases.latest) {
-      Releases.latest = Releases;
-    }
-
+  bc.getPackages = async function ({ Releases, name, date }) {
     if (!date) {
       date = new Date();
     }
@@ -257,71 +287,146 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
 
     // let age = now - seconds;
 
-    let data = bc._caches[name];
-    let versions = data?.versions || [];
-    let triplets = [];
-    let now = date.valueOf();
-    if (data) {
-      process.nextTick(async function () {
-        let age = now - data.updated;
-        if (age < bc._staleAge) {
-          return;
-        }
-        data = await getLatestBuilds(Releases, cacheDir, name);
-        let updated = date.valueOf();
-        updateVersions(data, versions);
-        Object.assign(data, { name, updated, versions, triplets });
-        bc._caches[name] = data;
-      });
-      return data;
+    let projInfo = bc._caches[name];
+
+    let meta = {
+      // version info
+      versions: projInfo?.versions || [],
+      lexvers: projInfo?.lexvers || [],
+      lexversMap: projInfo?.lexversMap || {},
+      // culled release assets
+      packages: projInfo?.packages || [],
+      releasesByTriplet: projInfo?.releasesByTriplet || {},
+      // target info
+      triplets: projInfo?.triplets || [],
+      oses: projInfo?.oses || [],
+      arches: projInfo?.arches || [],
+      libcs: projInfo?.libcs || [],
+      formats: projInfo?.formats || [],
+      // TODO channels: projInfo?.channels || [],
+    };
+
+    if (!projInfo) {
+      let json = await Fs.readFile(dataFile, 'ascii').catch(
+        async function (err) {
+          if (err.code !== 'ENOENT') {
+            throw err;
+          }
+
+          return null;
+        },
+      );
+
+      try {
+        projInfo = JSON.parse(json);
+      } catch (e) {
+        console.error(`error: ${dataFile}:\n\t${e.message}`);
+        projInfo = null;
+      }
+    }
+    if (!projInfo) {
+      projInfo = await getLatestBuilds(Releases, installersDir, cacheDir, name);
     }
 
-    let json = await Fs.readFile(dataFile, 'ascii').catch(async function (err) {
-      if (err.code !== 'ENOENT') {
-        throw err;
+    process.nextTick(async function () {
+      let now = date.valueOf();
+      let age = now - projInfo.updated;
+      if (age < bc._staleAge) {
+        return;
       }
 
-      return null;
+      projInfo = await getLatestBuilds(Releases, installersDir, cacheDir, name);
+      transformAndUpdate(name, projInfo, meta, date);
     });
 
-    try {
-      data = JSON.parse(json);
-    } catch (e) {
-      console.error(`error: ${dataFile}:\n\t${e.message}`);
-      data = null;
-    }
-
-    if (!data) {
-      data = await getLatestBuilds(Releases, cacheDir, name);
-    }
-    let updated = date.valueOf();
-    updateVersions(data, versions);
-    Object.assign(data, { name, updated, versions, triplets });
-    bc._caches[name] = data;
-
-    for (let build of data.releases) {
-      if (LEGACY_OS_MAP[build.os]) {
-        build.os = LEGACY_OS_MAP[build.os];
-      }
-      if (LEGACY_ARCH_MAP[build.arch]) {
-        build.arch = LEGACY_ARCH_MAP[build.arch];
-      }
-    }
-
-    return data;
+    transformAndUpdate(name, projInfo, meta, date);
+    return projInfo;
   };
 
+  function transformAndUpdate(name, projInfo, meta, date) {
+    meta.packages = [];
+
+    let updated = date.valueOf();
+    Object.assign(projInfo, { name, updated }, meta);
+    for (let build of projInfo.releases) {
+      let buildTarget = bc.classify(projInfo, build);
+      if (!buildTarget) {
+        // ignore known, non-package extensions
+        continue;
+      }
+
+      if (buildTarget.error) {
+        let err = buildTarget.error;
+        let code = err.code || '';
+        console.error(`[ERROR]: ${code} ${projInfo.name}: ${build.name}`);
+        console.error(`>>> ${err.message} <<<`);
+        console.error(projInfo);
+        console.error(build);
+        console.error(`^^^ ${err.message} ^^^`);
+        console.error(err.stack);
+        continue;
+      }
+
+      build.target = buildTarget;
+      meta.packages.push(build);
+    }
+
+    updateReleasesByTriplet(meta);
+    updateAndSortVersions(projInfo, meta);
+
+    Object.assign(projInfo, { name, updated }, meta);
+    bc._caches[name] = projInfo;
+  }
+
+  function updateReleasesByTriplet(meta) {
+    for (let build of meta.packages) {
+      let target = build.target;
+
+      let triplet = `${target.os}-${target.arch}-${target.libc}`;
+      if (!meta.releasesByTriplet[triplet]) {
+        meta.releasesByTriplet[triplet] = {};
+      }
+
+      let buildsByRelease = meta.releasesByTriplet[triplet];
+      if (!buildsByRelease[build.version]) {
+        buildsByRelease[build.version] = [];
+      }
+
+      let packages = buildsByRelease[build.version];
+      packages.push(build);
+    }
+  }
+
   // TODO
-  //   - sort version
   //   - tag channels
-  //   - @beta not install older than stable
-  function updateVersions(data, versions) {
-    for (let release of data.releases) {
-      let hasVersion = versions.includes(release.version);
+  function updateAndSortVersions(projInfo, meta) {
+    for (let build of projInfo.packages) {
+      let hasVersion = meta.versions.includes(build.version);
       if (!hasVersion) {
-        versions.unshift(release.version);
+        build.lexver = Lexver.parseVersion(build.version);
+        meta.lexversMap[build.lexver] = build.version;
       }
     }
+
+    meta.lexvers = Object.keys(meta.lexversMap);
+    meta.lexvers.sort();
+    meta.lexvers.reverse();
+
+    meta.versions = [];
+    for (let lexver of meta.lexvers) {
+      let version = meta.lexversMap[lexver];
+      meta.versions.push(version);
+    }
+
+    projInfo.packages.sort(function (a, b) {
+      if (a.lexver > b.lexver) {
+        return -1;
+      }
+      if (a.lexver < b.lexver) {
+        return 1;
+      }
+      return 0;
+    });
   }
 
   // Makes sure that packages are updated once an hour, on average
@@ -333,7 +438,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     }
 
     if (bc._staleNames.length === 0) {
-      let dirs = await bc.getPackages();
+      let dirs = await bc.getProjectsByType();
       bc._staleNames = Object.keys(dirs.valid);
       bc._staleNames.sort(function () {
         return 0.5 - Math.random();
@@ -341,10 +446,8 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     }
 
     let name = bc._staleNames.pop();
-    let Releases = require(`${installers}/${name}/releases.js`);
-    //data = await getLatestBuilds(Releases, cacheDir, name);
-    void (await bc.getBuilds({
-      Releases: Releases,
+    void (await bc.getPackages({
+      //Releases: Releases,
       name: name,
       date: new Date(),
     }));
@@ -360,26 +463,34 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     bc._freshenTimeout.unref();
   };
 
-  bc.classify = function (pkg, build) {
-    let maybeInstallable = Triplet.maybeInstallable(pkg, build);
+  bc.classify = function (projInfo, build) {
+    /* jshint maxcomplexity: 25 */
+    let maybeInstallable = Triplet.maybeInstallable(projInfo, build);
     if (!maybeInstallable) {
       return null;
     }
 
+    if (LEGACY_OS_MAP[build.os]) {
+      build.os = LEGACY_OS_MAP[build.os];
+    }
+    if (LEGACY_ARCH_MAP[build.arch]) {
+      build.arch = LEGACY_ARCH_MAP[build.arch];
+    }
+
     // because some packages are shimmed to match a single download against
     let preTarget = Object.assign({ os: '', arch: '', libc: '' }, build);
 
     let targetId = `${preTarget.os}:${preTarget.arch}:${preTarget.libc}`;
-    let buildId = `${pkg.name}:${targetId}@${build.download}`;
+    let buildId = `${projInfo.name}:${targetId}@${build.download}`;
     let target = bc._targetsByBuildIdCache[buildId];
     if (target) {
       Object.assign(build, { target: target, triplet: target.triplet });
       return target;
     }
 
-    let pattern = Triplet.toPattern(pkg, build);
+    let pattern = Triplet.toPattern(projInfo, build);
     if (!pattern) {
-      let err = new Error(`no pattern generated for ${name}`);
+      let err = new Error(`no pattern generated for ${projInfo.name}`);
       err.code = 'E_BUILD_NO_PATTERN';
       target = { error: err };
       bc._targetsByBuildIdCache[buildId] = target;
@@ -419,15 +530,47 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     // {NAME}.windows.x86_64v2.musl.exe
     //     windows-x86_64_v2-musl
     target = { triplet: '' };
-    void Triplet.termsToTarget(target, pkg, build, terms);
+    void Triplet.termsToTarget(target, projInfo, build, terms);
 
     target.triplet = `${target.arch}-${target.vendor}-${target.os}-${target.libc}`;
-    let hasTriplet = pkg.triplets.includes(target.triplet);
-    if (!hasTriplet) {
+
+    {
       // TODO I don't love this hidden behavior
       // perhaps classify should just happen when the package is loaded
       // (and the sanity error should be removed, or thrown after the loop is complete)
-      pkg.triplets.push(target.triplet);
+      let hasTriplet = projInfo.triplets.includes(target.triplet);
+      if (!hasTriplet) {
+        projInfo.triplets.push(target.triplet);
+      }
+      let hasOs = projInfo.oses.includes(target.os);
+      if (!hasOs) {
+        projInfo.oses.push(target.os);
+      }
+      let hasArch = projInfo.arches.includes(target.arch);
+      if (!hasArch) {
+        projInfo.arches.push(target.arch);
+      }
+      let hasLibc = projInfo.libcs.includes(target.libc);
+      if (!hasLibc) {
+        projInfo.libcs.push(target.libc);
+      }
+
+      if (!build.ext) {
+        build.ext = Triplet.buildToPackageType(build);
+      }
+      if (build.ext) {
+        if (!build.ext.startsWith('.')) {
+          build.ext = `.${build.ext}`;
+        }
+      }
+      let hasExt = projInfo.formats.includes(build.ext);
+      if (!hasExt) {
+        projInfo.formats.push(build.ext);
+      }
+      let hasGlobalExt = bc.formats.includes(build.ext);
+      if (!hasGlobalExt) {
+        bc.formats.push(build.ext);
+      }
     }
 
     bc._triplets[target.triplet] = true;
@@ -437,7 +580,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     for (let term of triple) {
       if (!ALL_TERMS[term]) {
         throw new Error(
-          `[SANITY FAIL] '${pkg.name}' '${target.triplet}' generated unknown term '${term}'`,
+          `[SANITY FAIL] '${projInfo.name}' '${target.triplet}' generated unknown term '${term}'`,
         );
       }
 
@@ -448,5 +591,302 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     return target;
   };
 
+  /**
+   * Given a list of acceptable formats, get the sorted list of of formats.
+   * Actually used (as per node _webi/lint-builds.js):
+   *     .7z
+   *     .app.zip
+   *     .dmg
+   *     .exe
+   *     .exe.xz
+   *     .git
+   *     .gz
+   *     .msi
+   *     .pkg
+   *     .pkg.tar.zst
+   *     .sh
+   *     .tar.gz
+   *     .tar.xz
+   *     .xz
+   *     .zip
+   */
+  bc.getSortedFormats = function (formats) {
+    /* jshint maxcomplexity: 25 */
+    formats.sort();
+    let id = formats.join(',');
+    if (bc._allFormats[id]) {
+      return bc._allFormats[id];
+    }
+
+    // we don't know how to handle any of these yet
+    // let exclude = [];
+    // let isAndroid = false;
+    // if (!isAndroid) {
+    //   exclude.push('.apk');
+    // }
+    // let isDebian = false;
+    // if (!isDebian) {
+    //   exclude.push('.deb');
+    // }
+    // let isEnterpriseLinux = false;
+    // if (!isEnterpriseLinux) {
+    //   exclude.push('.rpm');
+    // }
+    // let isArch = false;
+    // if (!isArch) {
+    //   exclude.push('.pkg.tar.zst');
+    // }
+
+    let hasExe = formats.includes('exe') || formats.includes('.exe');
+
+    /** @type {Array<String>} */
+    let exts = [];
+
+    let hasXz = formats.includes('xz') || formats.includes('.xz');
+    if (hasXz) {
+      exts.push('.tar.xz');
+      if (hasExe) {
+        exts.push('.exe.xz');
+      }
+      exts.push('.xz');
+    }
+    let hasZst = formats.includes('zst') || formats.includes('.zst');
+    if (hasZst) {
+      exts.push('.tar.zst');
+      exts.push('.zst');
+    }
+    let hasZip = formats.includes('zip') || formats.includes('.zip');
+    if (hasZip) {
+      exts.push('.zip');
+    }
+    let has7z = false;
+    if (has7z) {
+      exts.push('.7z');
+    }
+    // let hasBz2 = formats.includes('bz2') || formats.includes('.bz2');
+    // if (hasBz2) {
+    // 	exts.push('.bz2');
+    // }
+    if (hasExe) {
+      if (!hasZip) {
+        exts.push('.zip');
+      }
+      exts.push('.tar.gz');
+      exts.push('.gz');
+      exts.push('.exe');
+      exts.push('.msi');
+      //exts.push('.msixbundle');
+    } else {
+      exts.push('.tar.gz');
+      exts.push('.gz');
+      exts.push('.sh');
+    }
+    exts.push('.git');
+
+    // Fallbacks
+    exts.push('.app.zip');
+    exts.push('.dmg');
+    exts.push('.pkg');
+    if (!hasXz) {
+      exts.push('.tar.xz');
+      if (hasExe) {
+        exts.push('.exe.xz');
+      }
+      exts.push('.xz');
+    }
+    if (!hasZip) {
+      if (!hasExe) {
+        exts.push('.zip');
+      }
+    }
+    if (!hasZst) {
+      exts.push('.tar.zst');
+      exts.push('.zst');
+    }
+    if (!has7z) {
+      exts.push('.7z');
+    }
+    // if (!hasRar) {
+    // 	exts.push('.rar');
+    // }
+    // exts.push('.tar.bz2');
+    // exts.push('.bz2');
+
+    bc._allFormats[id] = exts;
+    return exts;
+  };
+
+  bc.selectPackage = function (packages, formats) {
+    if (packages.length === 1) {
+      return packages[0];
+    }
+
+    let exts = bc.getSortedFormats(formats);
+    for (let ext of exts) {
+      for (let build of packages) {
+        if (build.ext === ext) {
+          return build;
+        }
+      }
+    }
+
+    return packages[0];
+  };
+
+  /**
+   * @param {Object} target
+   * @param {String} target.os - linux, darwin, windows, *bsd
+   * @param {String} target.arch - arm64, x86_64, ppc64le
+   * @param {String} target.libc - none, libc, gnu, musl, bionic, msvc
+   * @param {String} target.triplet - os-vendor-arch-libc
+   * @param {Error} target.error
+   */
+  bc.findMatchingPackages = function (pkgInfo, hostTarget, verTarget) {
+    let matchInfo = bc._enumerateVersions(pkgInfo, verTarget.version);
+    let triplets = bc._enumerateTriplets(hostTarget);
+    //console.log('dbg: matchInfo', matchInfo);
+
+    if (matchInfo) {
+      for (let _triplet of triplets) {
+        let targetReleases = pkgInfo.releasesByTriplet[_triplet];
+        if (!targetReleases) {
+          continue;
+        }
+
+        // Make sure that these releases are the expected version
+        // (ex: jq1.7 => darwin-arm64-libc, jq1.6 => darwin-x86_64-libc)
+        for (let matchver of matchInfo.matches) {
+          let ver = pkgInfo.lexversMap[matchver] || matchver;
+          let packages = targetReleases[ver];
+          if (!packages) {
+            continue;
+          }
+
+          let match = {
+            triplet: _triplet,
+            packages: packages,
+            latest: pkgInfo.versions[0],
+            version: ver,
+            versions: matchInfo,
+          };
+          return match;
+        }
+      }
+
+      return null;
+    }
+
+    for (let _triplet of triplets) {
+      let targetReleases = pkgInfo.releasesByTriplet[_triplet];
+      if (!targetReleases) {
+        continue;
+      }
+
+      let versions = Object.keys(targetReleases);
+      //console.log('dbg: targetRelease versions', versions);
+      let lexvers = [];
+      for (let version of versions) {
+        let lexPrefix = Lexver.parseVersion(version);
+        lexvers.push(lexPrefix);
+      }
+      lexvers.sort();
+      lexvers.reverse();
+      // TODO get the other matchInfo props
+
+      // Make sure that these releases are the expected version
+      // (ex: jq1.7 => darwin-arm64-libc, jq1.6 => darwin-x86_64-libc)
+      for (let matchver of lexvers) {
+        let ver = pkgInfo.lexversMap[matchver] || matchver;
+        let packages = targetReleases[ver];
+        //console.log('dbg: packages', packages);
+        if (!packages) {
+          continue;
+        }
+
+        let pkg = packages[0];
+        if (verTarget.lts) {
+          if (!pkg.lts) {
+            continue;
+          }
+
+          let match = {
+            triplet: _triplet,
+            packages: packages,
+            latest: pkgInfo.versions[0],
+            version: ver,
+            versions: matchInfo,
+          };
+          return match;
+        }
+
+        let wantChannel = verTarget.channel || 'stable';
+        let isChannel = pkg.channel || 'stable';
+        if (wantChannel === 'stable') {
+          if (isChannel !== 'stable') {
+            continue;
+          }
+        }
+        // latest, beta, alpha, rc, preview
+
+        let match = {
+          triplet: _triplet,
+          packages: packages,
+          latest: pkgInfo.versions[0],
+          version: ver,
+          versions: matchInfo,
+        };
+        return match;
+      }
+    }
+
+    return null;
+  };
+
+  bc._enumerateTriplets = function (hostTarget) {
+    let id = [hostTarget.os, hostTarget.arch, hostTarget.libc].join(',');
+    let triplets = bc._allTriplets[id] || [];
+    if (triplets.length > 0) {
+      return triplets;
+    }
+
+    let oses = [];
+    if (hostTarget.os === 'windows') {
+      oses = ['ANYOS', 'windows'];
+    } else if (hostTarget.os === 'android') {
+      oses = ['ANYOS', 'posix_2017', 'android', 'linux'];
+    } else {
+      oses = ['ANYOS', 'posix_2017', hostTarget.os];
+    }
+
+    let waterfall = HostTargets.WATERFALL[hostTarget.os] || {};
+    let arches = waterfall[hostTarget.arch] ||
+      HostTargets.WATERFALL.ANYOS[hostTarget.arch] || [hostTarget.arch];
+    arches = ['ANYARCH'].concat(arches);
+    let libcs = waterfall[hostTarget.libc] ||
+      HostTargets.WATERFALL.ANYOS[hostTarget.libc] || [hostTarget.libc];
+
+    for (let os of oses) {
+      for (let arch of arches) {
+        for (let libc of libcs) {
+          let triplet = `${os}-${arch}-${libc}`;
+          triplets.push(triplet);
+        }
+      }
+    }
+    bc._allTriplets[id] = triplets;
+
+    return triplets;
+  };
+
+  bc._enumerateVersions = function (pkgInfo, ver) {
+    if (!ver) {
+      return null;
+    }
+    let lexPrefix = Lexver.parsePrefix(ver);
+    let matchInfo = Lexver.matchSorted(pkgInfo.lexvers, lexPrefix);
+
+    return matchInfo;
+  };
+
   return bc;
 };

_webi/builds.js

@@ -0,0 +1,40 @@
+'use strict';
+
+let Builds = module.exports;
+
+let Path = require('node:path');
+
+let BuildsCacher = require('./builds-cacher.js');
+// let HostTargets = require('./build-classifier/host-targets.js');
+let Parallel = require('./parallel.js');
+
+var INSTALLERS_DIR = Path.join(__dirname, '..');
+var CACHE_DIR = Path.join(__dirname, '../_cache');
+
+let bc = BuildsCacher.create({
+  caches: CACHE_DIR,
+  installers: INSTALLERS_DIR,
+});
+bc.freshenRandomPackage(600 * 1000);
+
+Builds.init = async function () {
+  bc.freshenRandomPackage(600 * 1000);
+
+  let dirs = await bc.getProjects();
+  let projNames = Object.keys(dirs.valid);
+
+  let parallel = 25;
+  await Parallel.run(parallel, projNames, getAll);
+  async function getAll(name) {
+    void (await bc.getPackages({
+      //Releases: Releases,
+      name: name,
+      date: new Date(),
+    }));
+  }
+};
+
+Builds.getProjectType = bc.getProjectType;
+Builds.getPackage = bc.getPackages;
+Builds.findMatchingPackages = bc.findMatchingPackages;
+Builds.selectPackage = bc.selectPackage;

_webi/installers.js

@@ -14,16 +14,11 @@ function padScript(txt) {
 
 var BAD_SH_RE = /[<>'"`$\\]/;
 Installers.renderBash = async function (
-  pkgdir,
-  rel,
-  { baseurl, pkg, tag, ver, os = '', arch = '', libc = '', formats },
+  baseurl,
+  posixTemplate,
+  buildRequest,
+  buildMatch,
 ) {
-  if (!Array.isArray(formats)) {
-    formats = [];
-  }
-  if (!tag) {
-    tag = '';
-  }
   let installTxt = await Fs.readFile(path.join(pkgdir, 'install.sh'), 'utf8');
   installTxt = padScript(installTxt);
   var vers = rel.version.split('.');
@@ -99,10 +94,11 @@ Installers.renderBash = async function (
     ['WEBI_PKG_PATHNAME', pkgFile],
     ['WEBI_PKG_FILE', pkgFile], // TODO replace with pathname
     ['PKG_NAME', pkg],
-    ['PKG_OSES', rel.oses],
-    ['PKG_ARCHES', rel.arches],
-    ['PKG_LIBCS', rel.libcs],
-    ['PKG_FORMATS', (rel.formats || []).join(',')],
+    ['PKG_OSES', (rel.oses || []).join(' ')],
+    ['PKG_ARCHES', (rel.arches || []).join(' ')],
+    ['PKG_LIBCS', (rel.libcs || []).join(' ')],
+    ['PKG_FORMATS', (rel.formats || []).join(' ')],
+    ['PKG_LATEST', latest],
   ];
 
   for (let env of envReplacements) {
@@ -130,17 +126,18 @@ Installers.renderBash = async function (
   return tplTxt;
 };
 
+/**
+ * @param {String} pkgdir
+ * @param {String} baseurl
+ * @param {BuildRequest} buildRequest
+ * @param {BuildMatch} buildMatch
+ */
 Installers.renderPowerShell = async function (
-  pkgdir,
-  rel,
-  { baseurl, pkg, tag, ver, os, arch, libc = '', formats },
+  baseurl,
+  pwshTemplate,
+  buildRequest,
+  buildMatch,
 ) {
-  if (!Array.isArray(formats)) {
-    formats = [];
-  }
-  if (!tag) {
-    tag = '';
-  }
   let installTxt = await Fs.readFile(path.join(pkgdir, 'install.ps1'), 'utf8');
   installTxt = padScript(installTxt);
   /*

_webi/lint-builds.js

@@ -124,184 +124,19 @@ let bc = BuildsCacher.create({
   installers: INSTALLERS_DIR,
 });
 
-async function getPackagesWithBuilds(installersDir, pkgNames, parallel = 25) {
-  let packages = [];
-
-  await Parallel.run(parallel, pkgNames, getAll);
-
-  async function getAll(name, i) {
-    let Releases = require(`${installersDir}/${name}/releases.js`);
-    let pkg = await bc.getBuilds({
-      Releases: Releases,
-      name: name,
-      date: new Date(),
-    });
-    packages[i] = pkg;
-  }
-
-  return packages;
-}
-
-function getBuildsByTarget(packages) {
-  let packagesByName = {};
-
-  for (let pkg of packages) {
-    let buildsByOs = getBuildsByOs(pkg);
-    packagesByName[pkg.name] = buildsByOs;
-  }
-
-  return packagesByName;
-}
-
-function getBuildsByOs(pkg) {
-  let buildsByOs = {};
-
-  for (let build of pkg.releases) {
-    // TODO check targets cache
-    let target = bc.classify(pkg, build);
-    if (!target) {
-      // ignore known, non-package extensions
-      continue;
-    }
-
-    if (target.error) {
-      let err = target.error;
-      let code = err.code || '';
-      console.error(`[ERROR]: ${code} ${pkg.name}: ${build.name}`);
-      console.error(`>>> ${err.message} <<<`);
-      console.error(pkg);
-      console.error(build);
-      console.error(`^^^ ${err.message} ^^^`);
-      console.error(err.stack);
-      continue;
-    }
-
-    let buildsByRelease = getBuildsByRelease(build, buildsByOs, target);
-    buildsByRelease.push(build);
-  }
-
-  return buildsByOs;
-}
-
-function getBuildsByRelease(build, buildsByOs, target) {
-  let archLibc = `${target.arch}-${target.libc}`;
-
-  if (!buildsByOs[target.os]) {
-    buildsByOs[target.os] = {};
-  }
-
-  let buildsByVersion = buildsByOs[target.os];
-  if (!buildsByVersion[build.version]) {
-    buildsByVersion[build.version] = {};
-  }
-
-  let buildsByArchLibc = buildsByVersion[build.version];
-  if (!buildsByArchLibc[archLibc]) {
-    buildsByArchLibc[archLibc] = [];
-  }
-
-  let buildsByRelease = buildsByArchLibc[archLibc];
-  return buildsByRelease;
-}
-
-function matchBuildsByTarget(pkg, buildsTree, target) {
-  let oses = [];
-  let targetOs = target.os;
-  if (target.os === 'windows') {
-    oses = ['ANYOS', 'windows'];
-    //buildsByOs = buildsTree.ANYOS || buildsTree[target.os];
-  } else if (target.os === 'android') {
-    oses = ['ANYOS', 'posix_2017', 'android', 'linux'];
-    // buildsByOs =
-    //   buildsTree.ANYOS || buildsTree.posix_2017 || buildsTree[target.os];
-    // if (!buildsByOs) {
-    //   targetOs = 'linux';
-    //   buildsByOs = buildsTree.linux;
-    // }
-  } else {
-    oses = ['ANYOS', 'posix_2017', target.os];
-    // buildsByOs =
-    //   buildsTree.ANYOS || buildsTree.posix_2017 || buildsTree[target.os];
-  }
-
-  // TODO can we move sortByOsAndArchLibc(builds, anything) down to the lib?
-  //     and then the matcher
-  //     and make the waterfall more optional?
-
-  let waterfall = HostTargets.WATERFALL[target.os] || {};
-  let arches = waterfall[target.arch] ||
-    HostTargets.WATERFALL.ANYOS[target.arch] || [target.arch];
-  arches = ['ANYARCH'].concat(arches);
-  let libcs = waterfall[target.libc] ||
-    HostTargets.WATERFALL.ANYOS[target.libc] || [target.libc];
-
-  //console.log('waterfalls', arches, libcs);
-
-  // TODO flatten earlier and precache?
-  let duplets = [];
-  for (let arch of arches) {
-    for (let libc of libcs) {
-      let duplet = `${arch}-${libc}`;
-      duplets.push(duplet);
-    }
-  }
-
-  let duplet;
-  let targetBuilds;
-  for (let os of oses) {
-    let buildsByOs = buildsTree[os];
-    if (!buildsByOs) {
-      continue;
-    }
-
-    // TODO
-    //   - latest supported triplets
-    //   - historical supported triplets
-
-    // TODO sort versions first, get channel (or 'stable' or 'latest') from user
-    for (let version of pkg.versions) {
-      let versionBuilds = buildsByOs[version];
-      if (!versionBuilds) {
-        continue;
-      }
-
-      for (let _duplet of duplets) {
-        targetBuilds = versionBuilds[_duplet];
-        //console.log(`    duplet: ${_duplet}`, versionBuilds, targetBuilds);
-        if (targetBuilds?.length > 0) {
-          targetOs = os;
-          duplet = _duplet;
-          break;
-        }
-      }
-      if (targetBuilds?.length > 0) {
-        break;
-      }
-    }
-    if (targetBuilds?.length > 0) {
-      break;
-    }
-  }
-
-  if (!targetBuilds?.length) {
-    // console.log('    no builds:', buildsByOs);
-    targetBuilds = [];
-  }
-
-  let match = { triplet: `${targetOs}-${duplet}`, builds: targetBuilds };
-  return match;
-}
-
 async function main() {
   // TODO
   //     node ./_webi/lint-builds.js caddy@beta 'x86_64/unknown Darwin libc'
   //
-  // let [pkgName, userAgent] = process.argv[2].slice(0);
-  // create test case for zoxide, caddy, rg
+  //let [projName, userAgent] = process.argv.slice(2);
+  let projName = process.argv[2];
+  // create test case for zoxide, goreleaser, go, yq, caddy, rg
 
-  let dirs = await bc.getPackages();
-  showDirs(dirs);
-  console.info('');
+  let dirs = await bc.getProjectsByType();
+  if (!projName) {
+    showDirs(dirs);
+    console.info('');
+  }
 
   bc.freshenRandomPackage(600 * 1000);
 
@@ -309,30 +144,42 @@ async function main() {
   let triples = [];
   let valids = Object.keys(dirs.valid);
 
-  let index = valids.indexOf('webi');
-  if (index >= 0) {
-    // TODO fix the webi faux package
-    // (not sure why I even created it)
-    void valids.splice(index, 1);
+  if (projName) {
+    if (!valids.includes(projName)) {
+      throw new Error(`'${projName}' is not a valid installable project`);
+    }
+    valids = [projName];
   }
-
-  let parallel = 25;
   //valids = ['atomicparsley', 'caddy', 'macos'];
   //valids = ['atomicparsley'];
-  let packages = await getPackagesWithBuilds(INSTALLERS_DIR, valids, parallel);
 
-  console.info(`Fetching builds for`);
-  for (let pkg of packages) {
-    console.info(`    ${pkg.name}`);
+  console.info('');
+  console.info(`Fetching project release assets`);
+  let parallel = 25;
+  let projects = [];
+  await Parallel.run(parallel, valids, getAll);
+  async function getAll(name, i) {
+    console.info(`    ${name}`);
+    let projInfo = await bc.getPackages({
+      //Releases: Releases,
+      name: name,
+      date: new Date(),
+    });
+    projects[i] = projInfo;
+  }
 
-    let nStr = pkg.releases.length.toString();
+  console.info(`Classifying build assets for...`);
+  for (let projInfo of projects) {
+    console.info(`    ${projInfo.name}`);
+
+    let nStr = projInfo.releases.length.toString();
     let n = nStr.padStart(5, ' ');
-    let row = `##### ${n}\t${pkg.name}\tv`;
+    let row = `##### ${n}\t${projInfo.name}\tv`;
     rows.push(row);
 
     // ignore known, non-package extensions
-    for (let build of pkg.releases) {
-      let target = bc.classify(pkg, build);
+    for (let build of projInfo.releases) {
+      let target = bc.classify(projInfo, build);
       if (!target) {
         // non-build file
         continue;
@@ -341,7 +188,7 @@ async function main() {
         let e = target.error;
         if (e.code === 'E_BUILD_NO_PATTERN') {
           console.warn(`>>> ${e.message} <<<`);
-          console.warn(pkg);
+          console.warn(projInfo);
           console.warn(build);
           console.warn(`^^^ ${e.message} ^^^`);
         }
@@ -354,41 +201,42 @@ async function main() {
       // }
       // // For debug printing versions
       // console.error(build.version);
-      rows.push(`${target.triplet}\t${pkg.name}\t${build.version}`);
+      rows.push(`${target.triplet}\t${projInfo.name}\t${build.version}`);
     }
   }
 
-  let packagesTree = getBuildsByTarget(packages);
-  //console.log(`packagesTree`, packagesTree);
+  console.info(`Fetching builds for`);
+  for (let projInfo of projects) {
+    console.info('');
+    console.info('');
+    console.info(`    ${projInfo.name}`);
 
-  for (let pkg of packages) {
-    console.log('');
-    console.log('');
-    console.log('pkg', pkg.name);
-    let buildsTree = packagesTree[pkg.name];
-    console.log(buildsTree);
     for (let target of uaTargets) {
       let libc = target.libc || 'libc';
       let hostTriplet = `${target.os}-${target.arch}-${libc}`;
-      console.log('');
-      console.log(`target: ${hostTriplet}`);
-      let match = matchBuildsByTarget(pkg, buildsTree, target);
+      console.info('');
+      console.info(`    target: ${hostTriplet}`);
+      let match = bc.findMatchingPackages(projInfo, target, {
+        ver: '',
+      });
       if (!match) {
-        console.log(
-          `    pkg: ${pkg.name}: missing build for os '${target.os}'`,
+        console.info(
+          `    project: ${projInfo.name}: missing build for os '${target.os}'`,
         );
         continue;
       }
 
-      if (match.builds.length === 0) {
-        console.log(
-          `    pkg: ${pkg.name}: missing build for os '${target.os}-${target.arch}-${libc}'`,
+      if (!match.releases) {
+        console.info(
+          `    project: ${projInfo.name}: missing build for os '${target.os}-${target.arch}-${libc}'`,
         );
       } else if (match.triplet === hostTriplet) {
-        console.log(`    selected ${match.builds.length}`);
+        let releaseNames = Object.keys(match.releases);
+        console.info(`    selected ${releaseNames.length}`);
       } else {
-        console.log(
-          `    selected ${match.builds.length} (${match.triplet} fallback)`,
+        let releaseNames = Object.keys(match.releases);
+        console.info(
+          `    selected ${releaseNames.length} (${match.triplet} fallback)`,
         );
       }
     }
@@ -430,6 +278,18 @@ async function main() {
   }
 
   console.info('');
+  console.info('Formats:');
+  if (bc.formats.length) {
+    let formats = bc.formats.slice();
+    formats.sort();
+    if (!formats[0]) {
+      formats[0] = '(bin)';
+    }
+    console.warn('   ', formats.join('\n    '));
+  } else {
+    console.info('    (none)');
+  }
+
   // sort -u -k1 builds.tsv | rg -v '^#|^https?:' | rg -i arm
   // cut -f1 builds.tsv | sort -u -k1 | rg -v '^#|^https?:' | rg -i arm
 }

_webi/package-install.tpl.sh

@@ -127,6 +127,7 @@ __bootstrap_webi() {
             echo ""
             echo "    $(t_err "Error: no '${PKG_NAME:-"Unknown Package"}@${WEBI_TAG:-"Unknown Tag"}' release for '${WEBI_OS:-"Unknown OS"}' (${WEBI_LIBC:-"Unknown Libc"}) on '${WEBI_ARCH:-"Unknown CPU"}' as one of '${WEBI_FORMATS:-"Unknown File Type"}'")"
             echo ""
+            echo "        Latest Version: ${PKG_LATEST}"
             echo "        CPUs: $PKG_ARCHES"
             echo "        OSes: $PKG_OSES"
             echo "        libcs: $PKG_LIBCS"

_webi/serve-installer.js

@@ -2,133 +2,238 @@
 
 var InstallerServer = module.exports;
 
-var Fs = require('fs/promises');
-var path = require('path');
+let Fs = require('fs/promises');
+let Path = require('path');
 
-var uaDetect = require('./ua-detect.js');
-var Projects = require('./projects.js');
-var Installers = require('./installers.js');
+let HostTargets = require('./build-classifier/host-targets.js');
+let Builds = require('./builds.js');
+let Installers = require('./installers.js');
 
-// handlers caching and transformation, probably should be broken down
-var Releases = require('./transform-releases.js');
+InstallerServer.INSTALLERS_DIR = Path.join(__dirname, '..');
+
+/**
+ * @typedef BuildRequest
+ * @prop {String} unameAgent
+ * @prop {String} projectName
+ * @prop {String} tag
+ * @prop {Array<String>} formats
+ */
 
-InstallerServer.INSTALLERS_DIR = path.join(__dirname, '..');
 InstallerServer.serveInstaller = async function (
   baseurl,
-  ua,
-  pkg,
+  ua, // TODO nix
+  unameAgent,
+  pkg, // TODO nix
+  projectName,
   tag,
-  ext,
+  ext, // TODO nix
+  installerType,
   formats,
-  libc,
 ) {
-  let [rel, opts] = await InstallerServer.helper({
-    ua,
-    pkg,
+  if (!unameAgent) {
+    unameAgent = ua;
+  }
+  if (!projectName) {
+    projectName = pkg;
+  }
+  if (!installerType) {
+    installerType = ext;
+  }
+  let buildRequest = {
+    projectName,
     tag,
+    unameAgent,
     formats,
-    libc,
-  });
-  Object.assign(opts, {
-    baseurl,
-  });
+  };
 
-  var pkgdir = path.join(InstallerServer.INSTALLERS_DIR, pkg);
+  let hostTarget = {};
+  let buildMatch;
+  {
+    let terms = unameAgent.split(/[\s\/]+/g);
+    void HostTargets.termsToTarget(hostTarget, terms);
+    buildMatch = await InstallerServer.getBuild(buildRequest, hostTarget);
+  }
+  Object.assign(tmplParams, { baseurl, });
+
+  console.log('dbg: buildPkg', rel);
+  console.log('dbg: tmplParams', tmplParams);
+
+  var pkgdir = Path.join(InstallerServer.INSTALLERS_DIR, projectName);
   if ('ps1' === ext) {
-    return Installers.renderPowerShell(pkgdir, rel, opts);
+    return Installers.renderPowerShell(pkgdir, rel, tmplParams);
   }
-  return Installers.renderBash(pkgdir, rel, opts);
+  return Installers.renderBash(pkgdir, rel, tmplParams);
 };
-InstallerServer.helper = async function ({ ua, pkg, tag, formats, libc }) {
-  // TODO put some of this in a middleware? or common function?
 
-  // TODO maybe move package/version/lts/channel detection into getReleases
-  var ver = tag.replace(/^v/, '');
-  var lts;
-  var channel;
+// TODO put some of this in a middleware? or common function?
+// TODO maybe move package/version/lts/channel detection into getReleases
+/**
+ * @param {BuildRequest}
+ * @returns {BuildMatch}
+ */
+InstallerServer.helper = async function ({
+  unameAgent,
+  projectName,
+  tag,
+  formats,
+}) {
+  console.log(`dbg: Installer User-Agent: ${unameAgent}`);
 
-  switch (ver) {
-    case 'latest':
-      ver = '';
-      channel = 'stable';
-      break;
-    case 'lts':
-      lts = true;
-      channel = 'stable';
-      ver = '';
-      break;
-    case 'stable':
-      channel = 'stable';
-      ver = '';
-      break;
-    case 'beta':
-      channel = 'beta';
-      ver = '';
-      break;
-    case 'dev':
-      channel = 'dev';
-      ver = '';
-      break;
+  let releaseTarget = toReleaseTarget(tag);
+  let hostFormats = formats;
+  let terms = unameAgent.split(/[\s\/]+/g);
+  let hostTarget = {};
+  try {
+    void HostTargets.termsToTarget(hostTarget, terms);
+  } catch (e) {
+    // if we can't guarantee the results...
+    // "in the face of ambiguity, refuse the temptation to guess"
+    throw e;
+  }
+  console.log(`dbg: Installer Host Target:`);
+  console.log(hostTarget);
+
+  if (!hostTarget.os) {
+    throw new Error(`OS could not be identified by User-Agent '${unameAgent}'`);
   }
 
-  var myOs = uaDetect.os(ua);
-  var myArch = uaDetect.arch(ua);
-  var myLibc;
-  if (libc) {
-    myLibc = uaDetect.libc(libc);
+  console.log(`dbg: Get Project Installer Type for '${projectName}':`);
+  let proj = await Builds.getProjectType(projectName);
+  console.log(proj);
+
+  let validTypes = ['alias', 'selfhosted', 'valid'];
+  if (!validTypes.includes(proj.type)) {
+    let msg = `'${projectName}' doesn't have an installer: '${proj.type}': '${proj.detail}'`;
+    let err = new Error(msg);
+    err.code = 'ENOENT';
+    throw err;
   }
-  if (!myLibc) {
-    myLibc = uaDetect.libc(ua);
-  }
-  if (!myLibc) {
-    myLibc = 'libc';
+  if (proj.type === 'alias') {
+    projectName = proj.detail;
   }
 
-  let cfg = await Projects.get(pkg);
-  let releaseQuery = {
-    pkg: cfg.alias || pkg,
-    ver,
-    os: myOs,
-    arch: myArch,
-    libc: myLibc,
-    lts,
-    channel,
-    // TODO use formats for sorting, not exclusion
-    // (it's better to install xz or report an error to install zip)
-    formats,
+  let tmplParams = {
+    pkg: projectName,
+    tag: tag,
+    os: hostTarget.os,
+    arch: hostTarget.arch,
+    libc: hostTarget.libc,
+    formats: hostFormats,
     limit: 1,
   };
+  Object.assign(tmplParams, releaseTarget);
+  console.log('tmplParams', tmplParams);
 
-  let rels = await Releases.getReleases(releaseQuery);
-
-  var rel = rels.releases[0];
-  var opts = {
-    pkg: cfg.alias || pkg,
-    ver,
-    tag,
-    os: myOs,
-    arch: myArch,
-    libc: myLibc,
-    lts,
-    channel,
-    formats,
-    limit: 1,
+  let errPackage = {
+    name: 'doesntexist.ext',
+    version: '0.0.0',
+    lts: '-',
+    channel: 'error',
+    date: '1970-01-01',
+    os: hostTarget.os || '-',
+    arch: hostTarget.arch || '-',
+    libc: hostTarget.libc || '-',
+    ext: 'err',
+    download: 'https://example.com/doesntexist.ext',
+    comment:
+      'No matches found. Could be bad or missing version info' +
+      ',' +
+      "Check query parameters. Should be something like '/api/releases/{package}@{version}.tab?os={macos|linux|windows|-}&arch={amd64|x86|aarch64|arm64|armv7l|-}&libc={musl|gnu|msvc|libc|static}&limit=10'",
   };
-  rel = Object.assign(
-    {
-      oses: rels.oses,
-      arches: rels.arches,
-      libcs: rels.libcs,
-      formats: rels.formats,
-    },
-    rel,
+
+  if (proj.type === 'selfhosted') {
+    return [errPackage, tmplParams];
+  }
+
+  let projInfo = await Builds.getPackage({
+    name: projectName,
+    date: new Date(),
+  });
+  let latest = projInfo.versions[0];
+  Object.assign(tmplParams, { latest });
+  //console.log('projInfo', projInfo);
+
+  let buildTargetInfo = {
+    triplets: projInfo.triplets,
+    oses: projInfo.oses,
+    arches: projInfo.arches,
+    libcs: projInfo.libcs,
+    formats: projInfo.formats,
+  };
+
+  let hasOs = projInfo.oses.includes(hostTarget.os);
+  if (!hasOs) {
+    let pkg1 = Object.assign(buildTargetInfo, errPackage);
+    return [pkg1, tmplParams];
+  }
+
+  let targetRelease = Builds.findMatchingPackages(
+    projInfo,
+    hostTarget,
+    releaseTarget,
   );
+  // { triplet: `${os}-${arch}-${libc}`, packages: targetPackages
+  // , latest: projInfo.versions[0], versions: matchInfo
+  // }
 
-  return [rel, opts];
+  if (!targetRelease?.packages) {
+    let pkg1 = Object.assign(buildTargetInfo, errPackage);
+    return [pkg1, tmplParams];
+  }
+
+  let buildPkg = Builds.selectPackage(targetRelease.packages, hostFormats);
+  let ext = buildPkg.ext || '.exe';
+  if (ext.startsWith('.')) {
+    ext = ext.slice(1);
+  }
+
+  let version = targetRelease.version;
+  if (version.startsWith('v')) {
+    version = version.slice(1);
+  }
+  let ver = version;
+
+  buildPkg = Object.assign(buildTargetInfo, buildPkg, { ext, version, tag });
+  Object.assign(tmplParams, { tag, ext, ver, version });
+  //console.log(`VERSION: ${version}`, ext);
+  return [buildPkg, tmplParams];
 };
 
-var CURL_PIPE_PS1_BOOT = path.join(__dirname, 'curl-pipe-bootstrap.tpl.ps1');
-var CURL_PIPE_SH_BOOT = path.join(__dirname, 'curl-pipe-bootstrap.tpl.sh');
+let channelNames = [
+  'stable',
+  // 'hotfix',
+  'latest',
+  'rc',
+  'preview',
+  'pre',
+  'dev',
+  'beta',
+  'alpha',
+];
+
+function toReleaseTarget(tag) {
+  tag = tag.replace(/^v/, '');
+
+  let releaseTarget = {
+    channel: '',
+    lts: false,
+    version: '',
+  };
+
+  if (tag === 'lts') {
+    releaseTarget.lts = true;
+    releaseTarget.channel = 'stable';
+  } else if (channelNames.includes(tag)) {
+    releaseTarget.channel = tag;
+  } else {
+    releaseTarget.version = tag;
+  }
+
+  return releaseTarget;
+}
+
+var CURL_PIPE_PS1_BOOT = Path.join(__dirname, 'curl-pipe-bootstrap.tpl.ps1');
+var CURL_PIPE_SH_BOOT = Path.join(__dirname, 'curl-pipe-bootstrap.tpl.sh');
 var BAD_SH_RE = /[<>'"`$\\]/;
 
 InstallerServer.getPosixCurlPipeBootstrap = async function ({
@@ -150,7 +255,6 @@ InstallerServer.getPosixCurlPipeBootstrap = async function ({
     let name = env[0];
     let value = env[1];
 
-    // TODO create REs once, in higher scope
     let envRe = new RegExp(
       `^[ \\t]*#?[ \\t]*(export[ \\t])?[ \\t]*(${name})=.*`,
       'm',
@@ -198,9 +302,6 @@ InstallerServer.getPwshCurlPipeBootstrap = async function ({
     let tplRe = new RegExp(`{{ (${name}) }}`, 'g');
     bootTxt = bootTxt.replace(tplRe, `${value}`);
 
-    // let envRe = new RegExp(`^[ \\t]*#?[ \\t]*($$${name})[ \\t]*=.*`, 'im');
-    // bootTxt = bootTxt.replace(envRe, `$$${name} = '${value}'`);
-
     let setRe = new RegExp(
       `(#[ \\t]*)?(\\$${name})[ \\t]*=[ \\t]['"].*['"][ \\t]`,
       'im',

_webi/test.js

@@ -81,20 +81,29 @@ Releases.get(path.join(process.cwd(), pkgdir)).then(async function (all) {
   }
   var formats = ['exe', 'xz', 'tar', 'zip', 'git'];
 
+  let unameAgent = `${nodeOs}/${nodeOsRelease} ${nodeArch}/unknown ${nodeLibc}`;
+  console.log(`DEBUG: ${unameAgent}`);
   let [rel, opts] = await ServeInstaller.helper({
-    ua: `${nodeOs}/${nodeOsRelease} ${nodeArch}/unknown ${nodeLibc}`,
-    pkg: pkgname,
+    unameAgent: unameAgent,
+    projectName: pkgname,
     tag: pkgtag || '',
     formats: formats,
     libc: nodeLibc,
   });
+  console.log('DEBUG opts:');
+  console.log(opts);
   Object.assign(
+    rel,
     {
-      ver: '',
+      version: '{test-version}',
+      git_tag: '{test-git-tag}',
+      git_commit_hash: '{test-git-commit-hash}',
       lts: null,
-      channel: '',
-      os: '',
-      arch: '',
+      channel: '{test-channel}',
+      date: '1970-01-01T00:00:00Z',
+      os: '{test-os}',
+      arch: '{test-arch}',
+      ext: '{test-ext}',
       limit: 0,
     },
     opts,

package.json

@@ -5,6 +5,7 @@
   "description": "webinstall script repository",
   "main": "_webi/",
   "scripts": {
+    "bump": "npm version -m \"chore(release): bump to v%s\"",
     "fmt": "npm run prettier && npm run shfmt && npm run pwsh-fmt",
     "lint": "npm run shellcheck && npm run jshint && npm run pwsh-lint",
     "prepare": "npm run tooling-init && npm run git-hooks-init",

webi/releases.js

@@ -1,43 +0,0 @@
-'use strict';
-
-module.exports = async function () {
-  return {
-    releases: [
-      {
-        name: 'webi.tar',
-        version: '0.0.0',
-        lts: false,
-        channel: 'stable',
-        date: '',
-        os: 'linux',
-        arch: 'amd64',
-        ext: 'tar',
-        download: '',
-      },
-      {
-        name: 'webi.tar',
-        version: '0.0.0',
-        lts: false,
-        channel: 'stable',
-        date: '',
-        os: 'macos',
-        arch: 'amd64',
-        ext: 'tar',
-        download: '',
-      },
-      {
-        name: 'webi.tar',
-        version: '0.0.0',
-        lts: false,
-        channel: 'stable',
-        date: '',
-        os: 'windows',
-        arch: 'amd64',
-        ext: 'tar',
-        download: '',
-      },
-    ],
-    formats: [],
-    downloads: '',
-  };
-};