ref(builds-cacher): cache-only Node server, no fetches or writes

Make _webi/builds-cacher.js and _webi/transform-releases.js read
exclusively from ~/.cache/webi/legacy/<name>.json and remove every code
path that fetched from upstream or wrote to disk. The Go cache daemon
(webicached) is now the sole writer; the Node server is a thin reader.

builds-cacher.js:
- Resolve cache files via Os.homedir() + '/.cache/webi/legacy/' instead
  of the cacheDir argument. Drop the 'caches' constructor parameter.
- Remove getLatestBuilds / getLatestBuildsInner — they require()d
  per-package releases.js modules, fetched upstream, and wrote
  <yyyy-mm>/<name>.json + .updated.txt to disk.
- Remove the process.nextTick stale-refresh hook in _doGetPackages.
  Cold reads return what's on disk; if the file is missing, return
  empty meta instead of fetching.
- Remove freshenRandomPackage and its supporting state
  (bc._staleNames, bc._freshenTimeout, bc._staleAge). The hourly
  background freshener competed with webicached for the same files.
- In getProjectTypeByEntry, decide selfhosted vs valid by probing for
  the cache file rather than require()-ing releases.js. Drop the
  not_found / 'PROBLEM/SOLUTION/npm clean-install' diagnostic in
  getProjectsByType — the cache-file probe replaces the module-load
  failure mode.

transform-releases.js:
- Remove Releases.get and the _normalize import. Replace
  getCachedReleases's fetch+race+stale-age machinery with a single
  Fs.readFile of ~/.cache/webi/legacy/<pkg>.json.
- Drop the in-process version re-sort in createFormatsSorter; the
  cache file is already version-sorted by webicached, so the sorter
  only re-orders within the same version.

No callers' public signatures change. Every other file is untouched —
the per-package releases.js modules, _common/*.js fetchers, and
_webi/normalize.js still exist on disk but are no longer reachable
from the request path.

_webi/builds-cacher.js

@@ -3,13 +3,16 @@
 var BuildsCacher = module.exports;
 
 let Fs = require('node:fs/promises');
+let Os = require('node:os');
 let Path = require('node:path');
 
+let LEGACY_CACHE_DIR = Path.join(Os.homedir(), '.cache/webi/legacy');
+
 let HostTargets = require('./build-classifier/host-targets.js');
 let Lexver = require('./build-classifier/lexver.js');
 let Triplet = require('./build-classifier/triplet.js');
 
-var ALIAS_RE = /^alias: (\w+)$/m;
+var ALIAS_RE = /^alias: ([\w.-]+)$/m;
 
 var LEGACY_ARCH_MAP = {
   '*': 'ANYARCH',
@@ -126,61 +129,8 @@ async function readFirstBytes(path) {
   return str;
 }
 
-let promises = {};
-async function getLatestBuilds(Releases, installersDir, cacheDir, name, date) {
-  console.info(`[INFO] getLatestBuilds: ${name}`);
-
-  if (!Releases) {
-    Releases = require(`${installersDir}/${name}/releases.js`);
-  }
-  // TODO update all releases files with module.exports.xxxx = 'foo';
-  if (!Releases.latest) {
-    Releases.latest = Releases;
-  }
-
-  let id = `${cacheDir}/${name}`;
-  if (!promises[id]) {
-    promises[id] = Promise.resolve();
-  }
-
-  promises[id] = promises[id].then(async function () {
-    return await getLatestBuildsInner(Releases, cacheDir, name, date);
-  });
-
-  return await promises[id];
-}
-
-async function getLatestBuildsInner(Releases, cacheDir, name, date) {
-  let data = await Releases.latest();
-
-  if (!date) {
-    date = new Date();
-  }
-  let isoDate = date.toISOString();
-  let yearMonth = isoDate.slice(0, 7);
-
-  // TODO hash file
-  let dataFile = `${cacheDir}/${yearMonth}/${name}.json`;
-  // TODO fsstat releases.js vs require-ing time as well
-  let tsFile = `${cacheDir}/${yearMonth}/${name}.updated.txt`;
-
-  let dirPath = Path.dirname(dataFile);
-  await Fs.mkdir(dirPath, { recursive: true });
-
-  let json = JSON.stringify(data, null, 2);
-  await Fs.writeFile(dataFile, json, 'utf8');
-
-  let seconds = date.valueOf();
-  let ms = seconds / 1000;
-  let msStr = ms.toFixed(3);
-  await Fs.writeFile(tsFile, msStr, 'utf8');
-
-  return data;
-}
-
-BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
+BuildsCacher.create = function ({ ALL_TERMS, installers }) {
   let installersDir = installers;
-  let cacheDir = caches;
 
   if (!ALL_TERMS) {
     ALL_TERMS = Triplet.TERMS_PRIMARY_MAP;
@@ -195,7 +145,6 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
   bc._triplets = {};
   bc._targetsByBuildIdCache = {};
   bc._caches = {};
-  bc._staleAge = 15 * 60 * 1000;
   bc._allFormats = {};
   bc._allTriplets = {};
   // Per-name lock: serializes cold-cache getPackages so concurrent
@@ -219,19 +168,6 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     let entries = await Fs.readdir(installersDir, { withFileTypes: true });
     for (let entry of entries) {
       let meta = await bc.getProjectTypeByEntry(entry);
-      if (meta.type === 'not_found') {
-        let err = meta.detail;
-        console.error('');
-        console.error('PROBLEM');
-        console.error(`    ${err.message}`);
-        console.error('');
-        console.error('SOLUTION');
-        console.error('    npm clean-install');
-        console.error('');
-        throw new Error(
-          '[SANITY FAIL] should never have missing modules in prod',
-        );
-      }
       dirs[meta.type][entry.name] = meta.detail;
     }
 
@@ -300,19 +236,16 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
       return { type: 'alias', detail: link };
     }
 
-    let releasesPath = Path.join(path, 'releases.js');
-    try {
-      void require(releasesPath);
-    } catch (err) {
-      if (err.code !== 'MODULE_NOT_FOUND') {
-        return { type: 'errors', detail: err };
-      }
-
-      if (err.message.includes(`Cannot find module '${releasesPath}'`)) {
-        return { type: 'selfhosted', detail: true };
-      }
-
-      return { type: 'not_found', detail: err };
+    let cacheFile = `${LEGACY_CACHE_DIR}/${entry.name}.json`;
+    let hasCacheFile = await Fs.access(cacheFile)
+      .then(function () {
+        return true;
+      })
+      .catch(function () {
+        return false;
+      });
+    if (!hasCacheFile) {
+      return { type: 'selfhosted', detail: true };
     }
 
     return { type: 'valid', detail: true };
@@ -337,14 +270,9 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     return p;
   };
 
-  async function _doGetPackages({ Releases, name, date }) {
-    if (!date) {
-      date = new Date();
-    }
-    let isoDate = date.toISOString();
-    let yearMonth = isoDate.slice(0, 7);
-    let dataFile = `${cacheDir}/${yearMonth}/${name}.json`;
-    let tsFile = `${cacheDir}/${yearMonth}/${name}.updated.txt`;
+  async function _doGetPackages({ name }) {
+    let dataFile = `${LEGACY_CACHE_DIR}/${name}.json`;
+    let tsFile = `${LEGACY_CACHE_DIR}/${name}.updated.txt`;
 
     let tsDate;
     {
@@ -398,7 +326,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
       }
     }
     if (!projInfo) {
-      projInfo = await getLatestBuilds(Releases, installersDir, cacheDir, name);
+      return meta;
     }
     let latestProjInfo = await BuildsCacher.transformAndUpdate(
       name,
@@ -409,64 +337,9 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     );
     bc._caches[name] = latestProjInfo;
 
-    process.nextTick(async function () {
-      let now = date.valueOf();
-      let age = now - projInfo.updated;
-
-      let fresh = age < bc._staleAge;
-      if (fresh) {
-        return;
-      }
-
-      projInfo = await getLatestBuilds(Releases, installersDir, cacheDir, name);
-      let latestProjInfo = BuildsCacher.transformAndUpdate(
-        name,
-        projInfo,
-        meta,
-        date,
-        bc,
-      );
-      bc._caches[name] = latestProjInfo;
-    });
-
-    return projInfo;
+    return latestProjInfo;
   }
 
-  // Makes sure that packages are updated once an hour, on average
-  bc._staleNames = [];
-  bc._freshenTimeout = null;
-  bc.freshenRandomPackage = async function (minDelay) {
-    if (!minDelay) {
-      minDelay = 15 * 1000;
-    }
-
-    if (bc._staleNames.length === 0) {
-      let dirs = await bc.getProjectsByType();
-      bc._staleNames = Object.keys(dirs.valid);
-      bc._staleNames.sort(function () {
-        return 0.5 - Math.random();
-      });
-    }
-
-    let name = bc._staleNames.pop();
-    void (await bc.getPackages({
-      //Releases: Releases,
-      name: name,
-      date: new Date(),
-    }));
-    console.info(`[INFO] freshenRandomPackage: ${name}`);
-
-    let hour = 60 * 60 * 1000;
-    let delay = minDelay;
-    let spread = hour / bc._staleNames.length;
-    let seed = Math.random();
-    delay += seed * spread;
-
-    clearTimeout(bc._freshenTimeout);
-    bc._freshenTimeout = setTimeout(bc.freshenRandomPackage, delay);
-    bc._freshenTimeout.unref();
-  };
-
   /**
    * Given a list of acceptable formats, get the sorted list of of formats.
    * Actually used (as per node _webi/lint-builds.js):
@@ -669,29 +542,19 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
       return null;
     }
 
-    for (let _triplet of triplets) {
-      let targetReleases = projInfo.releasesByTriplet[_triplet];
-      if (!targetReleases) {
-        continue;
-      }
+    // Version-first iteration, not triplet-first: take the newest
+    // version even when its only build lives in a fallback triplet
+    // (e.g. serviceman v1.0.1 only exists at posix_2017-ANYARCH-none).
+    for (let lexver of projInfo.lexvers) {
+      let ver = projInfo.lexversMap[lexver] || lexver;
 
-      let versions = Object.keys(targetReleases);
-      //console.log('dbg: targetRelease versions', versions);
-      let lexvers = [];
-      for (let version of versions) {
-        let lexPrefix = Lexver.parseVersion(version);
-        lexvers.push(lexPrefix);
-      }
-      lexvers.sort();
-      lexvers.reverse();
-      // TODO get the other matchInfo props
+      for (let _triplet of triplets) {
+        let targetReleases = projInfo.releasesByTriplet[_triplet];
+        if (!targetReleases) {
+          continue;
+        }
 
-      // Make sure that these releases are the expected version
-      // (ex: jq1.7 => darwin-arm64-libc, jq1.6 => darwin-x86_64-libc)
-      for (let matchver of lexvers) {
-        let ver = projInfo.lexversMap[matchver] || matchver;
         let packages = targetReleases[ver];
-        //console.log('dbg: packages', packages);
         if (!packages) {
           continue;
         }
@@ -761,6 +624,13 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     let libcs = waterfall[hostTarget.libc] ||
       HostTargets.WATERFALL.ANYOS[hostTarget.libc] || [hostTarget.libc];
 
+    // Extend the glibc-host waterfall: the table only lists [none, libc]
+    // but Rust projects (bat, rg) and node ship libc='gnu' builds, and
+    // static musl builds also run on glibc hosts.
+    if (hostTarget.libc === 'libc' && !libcs.includes('gnu')) {
+      libcs = ['none', 'gnu', 'musl', 'libc'];
+    }
+
     for (let os of oses) {
       for (let arch of arches) {
         for (let libc of libcs) {
@@ -788,10 +658,17 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
 };
 
 BuildsCacher._classify = function (bc, projInfo, build) {
-  /* jshint maxcomplexity: 25 */
-  let maybeInstallable = Triplet.maybeInstallable(projInfo, build);
-  if (!maybeInstallable) {
-    return null;
+  /* jshint maxcomplexity: 30 */
+  // Cache entries arrive pre-classified (os/arch/libc/ext set). Skip
+  // maybeInstallable for those — it false-rejects names ending in a
+  // version tag (`serviceman-v1.0.1`, `v1.0.1.zip`).
+  let cacheClassified =
+    build.os && build.arch && build.libc && build.ext;
+  if (!cacheClassified) {
+    let maybeInstallable = Triplet.maybeInstallable(projInfo, build);
+    if (!maybeInstallable) {
+      return null;
+    }
   }
 
   if (LEGACY_OS_MAP[build.os]) {
@@ -855,16 +732,26 @@ BuildsCacher._classify = function (bc, projInfo, build) {
     bc.unknownTerms[term] = true;
   }
 
-  // {NAME}.windows.x86_64v2.musl.exe
-  //     windows-x86_64_v2-musl
+  // Skip termsToTarget for cache-classified entries: it false-flags
+  // e.g. .git URLs as os=ANYOS while the cache says os=posix_2017,
+  // and the mismatch check throws.
   target = { triplet: '' };
-  try {
-    void Triplet.termsToTarget(target, projInfo, build, terms);
-  } catch (e) {
-    console.error(`PACKAGE FORMAT CHANGE for '${projInfo.name}':`);
-    console.error(e.message);
-    console.error(build);
-    return null;
+  if (cacheClassified) {
+    target.os = build.os;
+    target.arch = build.arch;
+    target.libc = build.libc;
+    target.vendor = build.vendor || 'unknown';
+    target.android = false;
+    target.unknownTerms = [];
+  } else {
+    try {
+      void Triplet.termsToTarget(target, projInfo, build, terms);
+    } catch (e) {
+      console.error(`PACKAGE FORMAT CHANGE for '${projInfo.name}':`);
+      console.error(e.message);
+      console.error(build);
+      return null;
+    }
   }
 
   target.triplet = `${target.arch}-${target.vendor}-${target.os}-${target.libc}`;

_webi/builds.js

@@ -15,11 +15,8 @@ let bc = BuildsCacher.create({
   caches: CACHE_DIR,
   installers: INSTALLERS_DIR,
 });
-bc.freshenRandomPackage(600 * 1000);
 
 Builds.init = async function () {
-  bc.freshenRandomPackage(600 * 1000);
-
   let dirs = await bc.getProjectsByType();
   let projNames = Object.keys(dirs.valid);
 

_webi/transform-releases.js

@@ -2,75 +2,30 @@
 
 var Releases = module.exports;
 
+var Fs = require('node:fs/promises');
+var Os = require('node:os');
 var path = require('path');
-var _normalize = require('./normalize.js');
-
 var cache = {};
-//var staleAge = 5 * 1000;
-//var expiredAge = 15 * 1000;
-var staleAge = 5 * 60 * 1000;
-var expiredAge = 15 * 60 * 1000;
 
-let installerDir = path.join(__dirname, '..');
+var LEGACY_CACHE_DIR = path.join(Os.homedir(), '.cache/webi/legacy');
 
-Releases.get = async function (pkgdir) {
-  let get;
-  try {
-    get = require(`${pkgdir}/releases.js`);
-    // TODO update all releases files with module.exports.xxxx = 'foo';
-    if (!get.latest) {
-      get.latest = get;
-    }
-  } catch (e) {
-    let err = new Error('no releases.js for', pkgdir.split(/[\/\\]+/).pop());
-    err.code = 'E_NO_RELEASE';
-    throw err;
-  }
-
-  let all = await get.latest();
-
-  return _normalize(all);
-};
-
-// TODO needs a proper test, and more accurate (though perhaps far less simple) code
+// Sort releases by ext preference and libc within the same version.
+// The cache is already sorted by version (stable before beta, newest first),
+// so we only re-order within the same version string.
 function createFormatsSorter(formats) {
-  return function sortByVerExt(a, b) {
-    function lexver(semver) {
-      // v1.20.156 => 00001.00020.00156.zzzzz
-      // TODO BUG: v1.20.156-rc2 => 00001.00020.00156.rc2zz
-      var parts = semver.split(/[+\.\-]/g);
-      while (parts.length < 4) {
-        parts.push('');
-      }
-      return parts
-        .map(function (num, i) {
-          if (3 === i) {
-            return num.toString().padEnd(10, 'z');
-          }
-          return num.toString().padStart(10, '0');
-        })
-        .join('.');
-    }
-
-    var aver = lexver(a.version);
-    var bver = lexver(b.version);
-    if (aver > bver) {
-      //console.log(aver, '>', bver);
-      return -1;
-    }
-    if (aver < bver) {
-      //console.log(aver, '<', bver);
-      return 1;
+  return function sortByExtLibc(a, b) {
+    if (a.version !== b.version) {
+      // Array.sort is stable (V8, ES2019), so returning 0 across
+      // versions preserves the cache's pre-sorted version-desc order.
+      return 0;
     }
 
     var aExtPri = formats.indexOf(a.ext.replace(/tar\..*/, 'tar'));
     var bExtPri = formats.indexOf(b.ext.replace(/tar\..*/, 'tar'));
     if (aExtPri > bExtPri) {
-      //console.log(a.ext, aExtPri, '>', b.ext, bExtPri);
       return -1;
     }
     if (aExtPri < bExtPri) {
-      //console.log(a.ext, aExtPri, '<', b.ext, bExtPri);
       return 1;
     }
 
@@ -87,99 +42,39 @@ function createFormatsSorter(formats) {
 }
 
 async function getCachedReleases(pkg) {
-  // returns { download: '<template string>', releases: [{ version, date, os, arch, lts, channel, download}] }
+  // returns { download: '', releases: [{ version, date, os, arch, lts, channel, download}] }
 
-  async function chainCachePromise(fn) {
-    cache[pkg].promise = cache[pkg].promise.then(fn);
-    return cache[pkg].promise;
+  if (cache[pkg]) {
+    return cache[pkg];
   }
 
-  async function sleep(ms) {
-    return await new Promise(function (resolve, reject) {
-      setTimeout(resolve, ms);
-    });
-  }
+  let dataFile = `${LEGACY_CACHE_DIR}/${pkg}.json`;
 
-  async function putCache() {
-    var age = Date.now() - cache[pkg].updatedAt;
-    if (age < staleAge) {
-      //console.debug('NOT STALE ANYMORE - updated in previous promise');
-      return cache[pkg].all;
+  let json = await Fs.readFile(dataFile, 'utf8').catch(function (err) {
+    if (err.code === 'ENOENT') {
+      return null;
     }
+    throw err;
+  });
 
-    //console.debug('DOWNLOADING NEW "%s" releases', pkg);
-    var pkgdir = path.join(installerDir, pkg);
-
-    // workaround for request timeout seeming to not work
-    let complete = false;
-    await Promise.race([
-      Releases.get(pkgdir)
-        .catch(function (err) {
-          if ('E_NO_RELEASE' === err.code) {
-            let all = { _error: 'E_NO_RELEASE', download: '', releases: [] };
-            return all;
-          }
-
-          throw err;
-        })
-        .catch(function (err) {
-          let hasReleases = cache[pkg].all?.releases?.length > 1;
-          if (!hasReleases) {
-            throw err;
-          }
-
-          console.error(`Error: the BOOGEYMAN got us!`);
-          console.error(err.stack);
-
-          return cache[pkg].all;
-        })
-        .then(function (all) {
-          // Note: it is possible for slightly older data
-          // to replace slightly newer data, but this is better
-          // than being in a cycle where release updates _always_
-          // take longer than expected.
-          //console.debug('DOWNLOADED NEW "%s" releases', pkg);
-          cache[pkg].updatedAt = Date.now();
-          cache[pkg].all = all;
-          complete = true;
-        }),
-      sleep(15000).then(function () {
-        if (complete) {
-          return;
-        }
-        console.error(`request timeout waiting for '${pkg}' release info`);
-      }),
-    ]);
-
-    return cache[pkg].all;
+  if (!json) {
+    let empty = { download: '', releases: [] };
+    cache[pkg] = empty;
+    return empty;
   }
 
-  if (!cache[pkg]) {
-    cache[pkg] = {
-      updatedAt: 0,
-      all: { download: '', releases: [] },
-      promise: Promise.resolve(),
-    };
+  let all;
+  try {
+    all = JSON.parse(json);
+  } catch (e) {
+    console.error(`error: ${dataFile}:\n\t${e.message}`);
+    let empty = { download: '', releases: [] };
+    cache[pkg] = empty;
+    return empty;
   }
 
-  var bgRenewal;
-  var age = Date.now() - cache[pkg].updatedAt;
-  var fresh = age < staleAge;
-  if (!fresh) {
-    bgRenewal = chainCachePromise(putCache);
-  }
-
-  var tooStale = age > expiredAge;
-  if (!tooStale) {
-    return await cache[pkg].all;
-  }
-
-  return await Promise.race([
-    bgRenewal,
-    sleep(5000).then(function () {
-      return cache[pkg].all;
-    }),
-  ]);
+  cache[pkg] = all;
+  return all;
 }
 
 async function filterReleases(
@@ -190,15 +85,22 @@ async function filterReleases(
   // sort the most compatible format first
   // (i.e. so that we don't do .pkg on linux except on purpose)
   var rformats = formats.slice(0).reverse();
-  var sortByVerExt = createFormatsSorter(rformats);
+  var sortByExtLibc = createFormatsSorter(rformats);
   var reVer = new RegExp('^' + ver + '\\b');
 
   function selectMatches(rel) {
+    /* jshint maxcomplexity: 25 */
     if (os) {
-      if (rel.os !== '*') {
-        if (rel.os !== os) {
-          return false;
-        }
+      // '*' = any OS (matches anything, including windows).
+      // 'posix' / 'posix_20xx' = any POSIX OS (matches linux, macos,
+      // freebsd, etc., but NOT windows).
+      let isPosix = rel.os === 'posix' || rel.os.startsWith('posix_20');
+      let osMatches =
+        rel.os === '*' ||
+        rel.os === os ||
+        (isPosix && os !== 'windows');
+      if (!osMatches) {
+        return false;
       }
     }
 
@@ -257,7 +159,7 @@ async function filterReleases(
     return true;
   }
 
-  var sortedRels = all.releases.filter(selectMatches).sort(sortByVerExt);
+  var sortedRels = all.releases.filter(selectMatches).sort(sortByExtLibc);
   //console.log(sortedRels.slice(0, 4));
 
   return sortedRels.slice(0, limit || 1000);
@@ -416,8 +318,8 @@ Releases.getReleases = function ({
 };
 
 if (require.main === module) {
-  return module
-    .exports({
+  return Releases
+    .getReleases({
       pkg: 'node',
       ver: '',
       os: 'macos',