feat(tools): add development and testing utilities (classify, comparecache, e2etest, fetchraw, inspect, uaparse)

eugeneware/ffmpeg-static uses non-standard filenames (x64, ia32, win32,
arm) and ships both bare binaries and .gz variants. The generic classifier
mishandles the arch mapping and the installer has no bare .gz handler.

Add ffmpegdist custom classifier that:
- Maps non-standard OS/arch names to canonical values
- Filters to bare binaries only (skips .gz, ffprobe, LICENSE, README)
- Strips 'b' version prefix from tags

Also fix installerconf to allow explicit 'source' to override the
inferred source when both are present (e.g. source=ffmpegdist +
github_releases for fetching).

Closes #947

.gitignore

@@ -3,6 +3,16 @@ install-*.sh
 install-*.bat
 install-*.ps1
 
+# Go build outputs (from go run/build in repo root)
+/classify
+/e2etest
+/fetchraw
+/inspect
+/uaparse
+/webicached
+/zigtest
+/distributables.csv
+
 # local config
 .env.*
 *.env
@@ -18,7 +28,13 @@ node_modules/
 *.bak
 *.bak.*
 
+# agent session files
+agents/
+
 # other
 .DS_Store
 desktop.ini
 .directory
+LIVE_cache
+/webid
+bin/

AGENTS.md

@@ -370,3 +370,123 @@ Commit messages: `feat(<pkg>): add installer`, `fix(<pkg>): update install.sh`,
   `arch`, or `ext` explicitly in `releases.js`.
 - **Goreleaser archives**: Typically contain a bare binary at the archive root
   (not nested in a directory). Use `mv ./cmd "$pkg_src_cmd"`.
+
+---
+
+## Go Cache Daemon
+
+The Go pipeline (`cmd/webicached`) replaces the Node.js release-fetching code.
+It reads `releases.conf` files, fetches upstream release metadata, classifies
+build assets, and writes to `~/.cache/webi/legacy/` in the format the Node.js server expects.
+
+### Canonical Vocabulary
+
+The classifier MUST use exactly these strings. They match the production API.
+
+**OS**: `macos` (NOT `darwin`), `linux`, `windows`, `freebsd`, `openbsd`,
+`netbsd`, `dragonfly`, `aix`, `illumos`, `plan9`, `solaris`, `posix_2017`
+
+**Arch** — exact equivalences:
+- `amd64` (NOT `x86_64`), `x86` (NOT `i386`/`i686`/`386`)
+- `arm64` (NOT `aarch64`)
+- `armv7l` (NOT `armv7`), `armv6l` (NOT `armv6`)
+- `mipsle` (NOT `mipsel`), `mips64le` (NOT `mips64el`)
+
+**Arch** — compatibility downcasts:
+- `armhf` → `armv7l`, `armv7a` → `armv7l`, `armel` → `arm`
+
+**Arch** — other: `arm`, `ppc64le`, `ppc64`, `loong64`, `riscv64`, `s390x`,
+`mips`, `mips64`
+
+**Libc**: `none` (never empty), `gnu`, `musl`, `msvc`
+
+**Ext**: `tar.gz`, `tar.xz`, `zip`, `exe`, `7z`, `pkg`, `msi`
+(no leading dot; `exe` for bare binaries)
+
+### releases.conf
+
+Each package directory contains a `releases.conf` that tells the daemon where
+to fetch releases. Format is `key = value`, one per line. `#` comments and
+blank lines are ignored.
+
+#### Source types (mutually exclusive — pick one)
+
+```ini
+# GitHub binary releases (most common)
+github_releases = sharkdp/bat
+
+# GitHub source tarballs (with optional git fallback)
+github_sources = bnnanet/serviceman
+git_url = https://github.com/bnnanet/serviceman.git
+
+# Git tag enumeration (vim plugins, shell scripts — git_url alone)
+git_url = https://github.com/tpope/vim-commentary.git
+
+# Gitea (full URL required, or short form + base_url)
+gitea_releases = https://git.rootprojects.org/root/pathman
+
+# GitLab (defaults to gitlab.com)
+gitlab_releases = owner/repo
+
+# HashiCorp releases API
+hashicorp_product = terraform
+
+# Custom source (servicemandist, nodedist, zigdist, etc.)
+source = nodedist
+url = https://nodejs.org/download/release
+```
+
+#### Filtering, versioning, and platform
+
+```ini
+tag_prefix = bun-                    # monorepo: strip prefix from version
+version_prefixes = jq-               # strip from version string (space-separated)
+asset_filter = MinGit                # filename must contain this substring
+exclude = busybox -src- -docs-       # skip assets containing these (space-separated)
+os = posix_2017                      # restrict ALL versions to this OS (blanket)
+alias_of = rg                        # mirrors another package's releases
+```
+
+#### Design rules
+
+- `os` is a blanket tag on ALL versions. Only use for packages that are always
+  POSIX-only. For version-dependent OS tagging, use a custom `TagVariants` in
+  `internal/releases/{pkg}/variants.go`.
+- `git_url` can be primary (gittag source when it's the only key) or secondary
+  fallback alongside `github_sources`/`gitea_sources`.
+- Full URL forms accepted for github/gitea/gitlab (e.g.
+  `github_releases = https://github.com/sharkdp/bat`).
+
+### Testing
+
+Test tools: `cmd/e2etest` (pipeline comparison), `cmd/comparecache` (cache diff),
+`cmd/inspect` (single-package debug). Run each with `--help` for usage.
+
+### Classifier vs Per-Package Tagger
+
+The general classifier (`internal/classify/`) handles patterns common across
+many projects. It MUST NOT contain one-off logic for a single package.
+
+Per-package taggers (`internal/releases/{pkg}/variants.go`) handle
+project-specific knowledge. Read the existing taggers for conventions.
+
+MUST: Derive arch/OS from concrete evidence — not blanket defaults.
+MUST: New general classifier patterns must apply to 2-3+ packages.
+
+### Deploying
+
+```sh
+./scripts/deploy-webicached.sh beta.webi.sh
+./scripts/deploy-webicached.sh next.webi.sh
+```
+
+First-time setup on a new host uses `serviceman`:
+
+```sh
+serviceman add --name webicached \
+  --workdir ~/srv/webid/installers/ -- \
+  ~/bin/webicached \
+    --envfile ~/srv/webid/.env.secret \
+    --conf ~/srv/webid/installers/ \
+    --raw ~/.cache/webi/raw
+```

_common/brew.js

@@ -1,62 +0,0 @@
-'use strict';
-
-let Fetcher = require('../_common/fetcher.js');
-
-/**
- * Gets releases from 'brew'.
- *
- * @param {null} _
- * @param {string} formula
- * @returns {Promise<any>}
- */
-async function getDistributables(_, formula) {
-  if (!formula) {
-    return Promise.reject('missing formula for brew');
-  }
-
-  let resp;
-  try {
-    let url = `https://formulae.brew.sh/api/formula/${formula}.json`;
-    resp = await Fetcher.fetch(url, {
-      headers: { Accept: 'application/json' },
-    });
-  } catch (e) {
-    /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error
-    let err = e;
-    if (err.code === 'E_FETCH_RELEASES') {
-      err.message = `failed to fetch '${formula}' release data from 'brew': ${err.response.status} ${err.response.body}`;
-    }
-    throw e;
-  }
-  let body = JSON.parse(resp.body);
-
-  var ver = body.versions.stable;
-  var dl = (
-    body.bottle.stable.files.high_sierra || body.bottle.stable.files.catalina
-  ).url.replace(new RegExp(ver.replace(/\./g, '\\.'), 'g'), '{{ v }}');
-  return [
-    {
-      version: ver,
-      download: dl.replace(/{{ v }}/g, ver),
-    },
-  ].concat(
-    body.versioned_formulae.map(
-      /** @param {String} f */
-      function (f) {
-        var ver = f.replace(/.*@/, '');
-        return {
-          version: ver,
-          download: dl,
-        };
-      },
-    ),
-  );
-}
-
-module.exports = getDistributables;
-
-if (module === require.main) {
-  getDistributables(null, 'mariadb').then(function (all) {
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

_common/fetcher.js

@@ -1,56 +0,0 @@
-'use strict';
-
-let Fetcher = module.exports;
-
-/**
- * @typedef ResponseSummary
- * @prop {Boolean} ok
- * @prop {Headers} headers
- * @prop {Number} status
- * @prop {String} body
- */
-
-/**
- * @param {String} url
- * @param {RequestInit} opts
- * @returns {Promise<ResponseSummary>}
- */
-Fetcher.fetch = async function (url, opts) {
-  let resp = await fetch(url, opts);
-  let summary = Fetcher.throwIfNotOk(resp);
-
-  return summary;
-};
-
-/**
- * @param {Response} resp
- * @returns {Promise<ResponseSummary>}
- */
-Fetcher.throwIfNotOk = async function (resp) {
-  let text = await resp.text();
-
-  if (!resp.ok) {
-    let headers = Array.from(resp.headers);
-    console.error('[Fetcher] error: Response Headers:', headers);
-    console.error('[Fetcher] error: Response Text:', text);
-    let err = new Error(`fetch was not ok`);
-    Object.assign({
-      status: 503,
-      code: 'E_FETCH_RELEASES',
-      response: {
-        status: resp.status,
-        headers: headers,
-        body: text,
-      },
-    });
-    throw err;
-  }
-
-  let summary = {
-    ok: resp.ok,
-    headers: resp.headers,
-    status: resp.status,
-    body: text,
-  };
-  return summary;
-};

_common/git-tag.js

@@ -1,218 +0,0 @@
-'use strict';
-
-require('dotenv').config({ path: '.env' });
-
-var Crypto = require('crypto');
-var util = require('util');
-var exec = util.promisify(require('child_process').exec);
-var Fs = require('node:fs/promises');
-var FsSync = require('node:fs');
-var Path = require('node:path');
-
-var repoBaseDir = process.env.REPO_BASE_DIR || '';
-if (!repoBaseDir) {
-  repoBaseDir = Path.resolve('./_repos');
-  // for stderr
-  console.error(`[Warn] REPO_BASE_DIR= not set, ${repoBaseDir}`);
-}
-
-var Repos = {};
-
-Repos.clone = async function (repoPath, gitUrl) {
-  let uuid = Crypto.randomUUID();
-  let tmpPath = `${repoPath}.${uuid}.tmp`;
-  let bakPath = `${repoPath}.${uuid}.dup`;
-  await exec(`git clone --bare --filter=tree:0 ${gitUrl} ${tmpPath}`);
-
-  try {
-    FsSync.accessSync(repoPath);
-    return;
-  } catch (e) {
-    if (e.code !== 'ENOENT') {
-      throw e;
-    }
-  }
-
-  // sync to avoid race conditions
-  try {
-    FsSync.renameSync(repoPath, bakPath);
-  } catch (e) {
-    if (e.code !== 'ENOENT') {
-      throw e;
-    }
-  }
-  FsSync.renameSync(tmpPath, repoPath);
-
-  await Fs.rm(bakPath, { force: true, recursive: true });
-};
-
-Repos.checkExists = async function (repoPath) {
-  let err = await Fs.access(repoPath).catch(Object);
-  if (!err) {
-    return true;
-  }
-
-  if (err.code !== 'ENOENT') {
-    throw err;
-  }
-  return false;
-};
-
-Repos.fetch = async function (repoPath) {
-  await exec(`git --git-dir=${repoPath} fetch`);
-};
-
-Repos.getTags = async function (repoPath) {
-  var { stdout } = await exec(`git --git-dir=${repoPath} tag`);
-  var rawTags = stdout.trim().split('\n');
-
-  let tags = [];
-  for (let tag of rawTags) {
-    // ex: v1, v2, v1.1, 1.1.0-rc
-    let maybeVersionRe = /^(v\d+|v?\d+\.\d+)/;
-    let maybeVersion = maybeVersionRe.test(tag);
-    if (maybeVersion) {
-      tags.push(tag);
-    }
-  }
-
-  tags = tags.reverse();
-  return tags;
-};
-
-Repos.getTipInfo = async function (repoPath) {
-  var { stdout } = await exec(
-    `git --git-dir=${repoPath} rev-parse --abbrev-ref HEAD`,
-  );
-  var branch = stdout.trim();
-
-  var info = await Repos.getCommitInfo(repoPath, 'HEAD');
-  info.commitish = branch;
-
-  return info;
-};
-
-Repos.getCommitInfo = async function (repoPath, commitish) {
-  var { stdout } = await exec(
-    `git --git-dir=${repoPath} log -1 --format="%h %H %ad %cd" --date=iso-strict ${commitish}`,
-  );
-  stdout = stdout.trim();
-  var commitParts = stdout.split(/\s+/g);
-  return {
-    commitish: commitish,
-    commit_id: `${commitParts[0]}`,
-    commit: `${commitParts[1]}`,
-    date: commitParts[2],
-    date_authored: commitParts[3],
-  };
-};
-
-/**
- * Lists GitHub Releases (w/ uploaded assets)
- *
- * @param request
- * @param {string} owner
- * @param {string} gitUrl
- * @returns {PromiseLike<any> | Promise<any>}
- */
-async function getDistributables(gitUrl) {
-  let all = {
-    releases: [],
-    download: '',
-  };
-
-  let repoName = gitUrl.split('/').pop();
-  repoName = repoName.replace(/\.git$/, '');
-
-  let repoPath = `${repoBaseDir}/${repoName}.git`;
-
-  let isCloned = await Repos.checkExists(repoPath);
-  if (!isCloned) {
-    await Repos.clone(repoPath, gitUrl);
-  } else {
-    await Repos.fetch(repoPath);
-  }
-
-  let commitInfos = [];
-  let tags = await Repos.getTags(repoPath);
-  for (let tag of tags) {
-    let commitInfo = await Repos.getCommitInfo(repoPath, tag);
-    Object.assign(commitInfo, { version: tag, channel: '' });
-    commitInfos.push(commitInfo);
-  }
-
-  {
-    let tipInfo = await Repos.getTipInfo(repoPath);
-    // "2024-01-01T00:00:00-05:00" => "2024-01-01T05:00:00"
-    let date = new Date(tipInfo.date);
-    // "2024-01-01T05:00:00" => "v2024.01.01-05.00.00"
-    let version = date.toISOString();
-    // strip '.000Z'
-    version = version.replace(/\.\d+Z/, '');
-    version = version.replace(/[:\-]/g, '.');
-    version = version.replace(/T/, '-');
-    Object.assign(tipInfo, { version: `v${version}`, channel: '' });
-    if (commitInfos.length > 1) {
-      tipInfo.channel = 'beta';
-    }
-    commitInfos.push(tipInfo);
-  }
-
-  let releases = [];
-  for (let commitInfo of commitInfos) {
-    let version = commitInfo.version.replace(/^v/, '');
-
-    let date = new Date(commitInfo.date);
-    let isoDate = date.toISOString();
-    isoDate = isoDate.replace(/\.\d+Z/, '');
-
-    // tags and HEAD qualify for '--branch <branchish>'
-    let branch = commitInfo.commitish;
-
-    let rel = {
-      name: `${repoName}-v${version}`,
-      version: version,
-      git_tag: branch,
-      git_commit_hash: commitInfo.commit_id,
-      lts: false,
-      channel: commitInfo.channel,
-      date: isoDate,
-      os: '*',
-      arch: '*',
-      ext: 'git',
-      download: gitUrl,
-    };
-
-    releases.push(rel);
-  }
-  all.releases = releases;
-
-  return all;
-}
-
-module.exports = getDistributables;
-
-if (module === require.main) {
-  (async function main() {
-    let testRepos = [
-      // just a few tags, and a different HEAD
-      'https://github.com/tpope/vim-commentary.git',
-      // no tags, just HEAD
-      'https://github.com/ziglang/zig.vim.git',
-      // many, many tags
-      //'https://github.com/dense-analysis/ale.git',
-    ];
-    for (let url of testRepos) {
-      let all = await getDistributables(url);
-
-      all = require('../_webi/normalize.js')(all);
-      console.info(JSON.stringify(all, null, 2));
-    }
-  })()
-    .then(function () {
-      process.exit(0);
-    })
-    .catch(function (err) {
-      console.error(err);
-    });
-}

_common/gitea.js

@@ -1,47 +0,0 @@
-'use strict';
-
-var GitHubish = require('./githubish.js');
-
-/**
- * Lists Gitea Releases (w/ uploaded assets)
- *
- * @param {null} _ - deprecated
- * @param {String} owner
- * @param {String} repo
- * @param {String} baseurl
- * @param {String} [username]
- * @param {String} [token]
- */
-async function getDistributables(
-  _,
-  owner,
-  repo,
-  baseurl,
-  username = '',
-  token = '',
-) {
-  baseurl = `${baseurl}/api/v1`;
-  let all = await GitHubish.getDistributables({
-    owner,
-    repo,
-    baseurl,
-    username,
-    token,
-  });
-  return all;
-}
-
-module.exports = getDistributables;
-
-if (module === require.main) {
-  getDistributables(
-    null,
-    'root',
-    'pathman',
-    'https://git.rootprojects.org',
-    '',
-    '',
-  ).then(function (all) {
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

_common/github-source.js

@@ -1,40 +0,0 @@
-'use strict';
-
-require('dotenv').config({ path: '.env' });
-
-let GitHubSource = module.exports;
-
-let GitHubishSource = require('./githubish-source.js');
-
-/**
- * @param {Object} opts
- * @param {String} opts.owner
- * @param {String} opts.repo
- * @param {String} [opts.baseurl]
- * @param {String} [opts.username]
- * @param {String} [opts.token]
- */
-GitHubSource.getDistributables = async function ({
-  owner,
-  repo,
-  baseurl = 'https://api.github.com',
-  username = process.env.GITHUB_USERNAME || '',
-  token = process.env.GITHUB_TOKEN || '',
-}) {
-  let all = await GitHubishSource.getDistributables({
-    owner,
-    repo,
-    baseurl,
-    username,
-    token,
-  });
-  return all;
-};
-
-if (module === require.main) {
-  GitHubSource.getDistributables(null, 'BeyondCodeBootcamp', 'DuckDNS.sh').then(
-    function (all) {
-      console.info(JSON.stringify(all, null, 2));
-    },
-  );
-}

_common/github.js

@@ -1,42 +0,0 @@
-'use strict';
-
-require('dotenv').config({ path: '.env' });
-
-let GitHubish = require('./githubish.js');
-
-/**
- * Lists GitHub Releases (w/ uploaded assets)
- *
- * @param {null} _ - deprecated
- * @param {String} owner
- * @param {String} repo
- * @param {String} [baseurl]
- * @param {String} [username]
- * @param {String} [token]
- */
-module.exports = async function (
-  _,
-  owner,
-  repo,
-  baseurl = 'https://api.github.com',
-  username = process.env.GITHUB_USERNAME || '',
-  token = process.env.GITHUB_TOKEN || '',
-) {
-  let all = await GitHubish.getDistributables({
-    owner,
-    repo,
-    baseurl,
-    username,
-    token,
-  });
-  return all;
-};
-
-let GitHub = module.exports;
-GitHub.getDistributables = module.exports;
-
-if (module === require.main) {
-  GitHub.getDistributables(null, 'BurntSushi', 'ripgrep').then(function (all) {
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

_common/githubish-source.js

@@ -1,174 +0,0 @@
-'use strict';
-
-let Fetcher = require('../_common/fetcher.js');
-
-let GitHubishSource = module.exports;
-
-/**
- * Lists GitHub-Like Releases (source tarball & zip)
- *
- * @param {Object} opts
- * @param {String} opts.owner
- * @param {String} opts.repo
- * @param {String} opts.baseurl
- * @param {String} [opts.username]
- * @param {String} [opts.token]
- */
-GitHubishSource.getDistributables = async function ({
-  owner,
-  repo,
-  baseurl,
-  username = '',
-  token = '',
-}) {
-  if (!owner) {
-    throw new Error('missing owner for repo');
-  }
-  if (!repo) {
-    throw new Error('missing repo name');
-  }
-  if (!baseurl) {
-    throw new Error('missing baseurl');
-  }
-
-  let url = `${baseurl}/repos/${owner}/${repo}/releases`;
-  let opts = {
-    headers: {
-      'Content-Type': 'appplication/json',
-    },
-  };
-
-  if (token) {
-    let userpass = `${username}:${token}`;
-    let basicAuth = btoa(userpass);
-    Object.assign(opts.headers, {
-      Authorization: `Basic ${basicAuth}`,
-    });
-  }
-
-  let resp;
-  try {
-    resp = await Fetcher.fetch(url, opts);
-  } catch (e) {
-    /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error
-    let err = e;
-    if (err.code === 'E_FETCH_RELEASES') {
-      err.message = `failed to fetch '${baseurl}' (githubish-source, user '${username}) release data: ${err.response.status} ${err.response.body}`;
-    }
-    throw e;
-  }
-  let gHubResp = JSON.parse(resp.body);
-
-  let all = {
-    /** @type {Array<BuildInfo>} */
-    releases: [],
-    download: '',
-  };
-
-  for (let release of gHubResp) {
-    let dists = GitHubishSource.releaseToDistributables(release);
-    for (let dist of dists) {
-      let updates =
-        await GitHubishSource.followDistributableDownloadAttachment(dist);
-      Object.assign(dist, updates);
-      all.releases.push(dist);
-    }
-  }
-
-  return all;
-};
-
-/**
- * @typedef BuildInfo
- * @prop {String} [name] - name to use instead of filename for hash urls
- * @prop {String} version
- * @prop {String} [_version]
- * @prop {String} [arch]
- * @prop {String} channel
- * @prop {String} date
- * @prop {String} download
- * @prop {String} [ext]
- * @prop {String} [_filename]
- * @prop {String} [hash]
- * @prop {String} [libc]
- * @prop {Boolean} [_musl]
- * @prop {Boolean} [lts]
- * @prop {String} [size]
- * @prop {String} os
- */
-
-/**
- * @param {any} ghRelease - TODO
- * @returns {Array<BuildInfo>}
- */
-GitHubishSource.releaseToDistributables = function (ghRelease) {
-  let ghTag = ghRelease['tag_name']; // TODO tags aren't always semver / sensical
-  let lts = /(\b|_)(lts)(\b|_)/.test(ghRelease['tag_name']);
-  let channel = 'stable';
-  if (ghRelease['prerelease']) {
-    channel = 'beta';
-  }
-  let date = ghRelease['published_at'] || '';
-  date = date.replace(/T.*/, '');
-
-  let urls = [ghRelease.tarball_url, ghRelease.zipball_url];
-  /** @type {Array<BuildInfo>} */
-  let dists = [];
-  for (let url of urls) {
-    dists.push({
-      name: '',
-      version: ghTag,
-      lts: lts,
-      channel: channel,
-      date: date,
-      os: '*',
-      arch: '*',
-      libc: '',
-      ext: '',
-      download: url,
-    });
-  }
-
-  return dists;
-};
-
-/**
- * @param {BuildInfo} dist
- */
-GitHubishSource.followDistributableDownloadAttachment = async function (dist) {
-  let abortCtrl = new AbortController();
-  let resp = await fetch(dist.download, {
-    method: 'HEAD',
-    redirect: 'follow',
-    signal: abortCtrl.signal,
-  });
-  let headers = Object.fromEntries(resp.headers);
-
-  // Workaround for bug where METHOD changes to GET
-  abortCtrl.abort();
-  await resp.text().catch(function (err) {
-    if (err.name !== 'AbortError') {
-      throw err;
-    }
-  });
-
-  // ex: content-disposition: attachment; filename=BeyondCodeBootcamp-DuckDNS.sh-v1.0.1-0-ga2f4bde.zip
-  //  => BeyondCodeBootcamp-DuckDNS.sh-v1.0.1-0-ga2f4bde.zip
-  let name = headers['content-disposition'].replace(
-    /.*filename=([^;]+)(;|$)/,
-    '$1',
-  );
-  let download = resp.url;
-
-  return { name, download };
-};
-
-if (module === require.main) {
-  GitHubishSource.getDistributables({
-    owner: 'BeyondCodeBootcamp',
-    repo: 'DuckDNS.sh',
-    baseurl: 'https://api.github.com',
-  }).then(function (all) {
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

_common/githubish.js

@@ -1,137 +0,0 @@
-'use strict';
-
-let Fetcher = require('../_common/fetcher.js');
-
-/**
- * @typedef DistributableRaw
- * @prop {String} name
- * @prop {String} version
- * @prop {Boolean} lts
- * @prop {String} [channel]
- * @prop {String} date
- * @prop {String} os
- * @prop {String} arch
- * @prop {String} ext
- * @prop {String} download
- */
-
-let GitHubish = module.exports;
-
-/**
- * Lists GitHub-Like Releases (w/ uploaded assets)
- *
- * @param {Object} opts
- * @param {String} opts.owner
- * @param {String} opts.repo
- * @param {String} opts.baseurl
- * @param {String} [opts.username]
- * @param {String} [opts.token]
- */
-GitHubish.getDistributables = async function ({
-  owner,
-  repo,
-  baseurl,
-  username = '',
-  token = '',
-}) {
-  if (!owner) {
-    throw new Error('missing owner for repo');
-  }
-  if (!repo) {
-    throw new Error('missing repo name');
-  }
-  if (!baseurl) {
-    throw new Error('missing baseurl');
-  }
-
-  let url = `${baseurl}/repos/${owner}/${repo}/releases`;
-  let opts = {
-    headers: {
-      'Content-Type': 'appplication/json',
-    },
-  };
-
-  if (token) {
-    let userpass = `${username}:${token}`;
-    let basicAuth = btoa(userpass);
-    Object.assign(opts.headers, {
-      Authorization: `Basic ${basicAuth}`,
-    });
-  }
-
-  let resp;
-  try {
-    resp = await Fetcher.fetch(url, opts);
-  } catch (e) {
-    /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error
-    let err = e;
-    if (err.code === 'E_FETCH_RELEASES') {
-      err.message = `failed to fetch '${baseurl}' (githubish, user '${username}) release data: ${err.response.status} ${err.response.body}`;
-    }
-    throw e;
-  }
-  let gHubResp = JSON.parse(resp.body);
-
-  let all = {
-    /** @type {Array<DistributableRaw>} */
-    releases: [],
-    // todo make this ':baseurl' + ':releasename'
-    download: '',
-  };
-
-  try {
-    gHubResp.forEach(transformReleases);
-  } catch (e) {
-    /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error
-    let err = e;
-    console.error(err.message);
-    console.error('Error Headers:', resp.headers);
-    console.error('Error Body:', resp.body);
-    let msg = `failed to transform releases from '${baseurl}' with user '${username}'`;
-    throw new Error(msg);
-  }
-
-  /**
-   * @param {any} release - TODO
-   */
-  function transformReleases(release) {
-    for (let asset of release['assets']) {
-      let name = asset['name'];
-      let date = release['published_at']?.replace(/T.*/, '');
-      let download = asset['browser_download_url'];
-
-      // TODO tags aren't always semver / sensical
-      let version = release['tag_name'];
-      let channel;
-      if (release['prerelease']) {
-        // -rcX, -preview, -beta, etc will be checked in _webi/normalize.js
-        channel = 'beta';
-      }
-      let lts = /(\b|_)(lts)(\b|_)/.test(release['tag_name']);
-
-      all.releases.push({
-        name: name,
-        version: version,
-        lts: lts,
-        channel: channel,
-        date: date,
-        os: '', // will be guessed by download filename
-        arch: '', // will be guessed by download filename
-        ext: '', // will be normalized
-        download: download,
-      });
-    }
-  }
-
-  return all;
-};
-
-if (module === require.main) {
-  GitHubish.getDistributables({
-    owner: 'BurntSushi',
-    repo: 'ripgrep',
-    baseurl: 'https://api.github.com',
-  }).then(function (all) {
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

_example/releases.conf

@@ -0,0 +1,3 @@
+# Example releases.conf — uses ripgrep as a sample project.
+# Copy this file into your package directory and adjust.
+github_releases = BurntSushi/ripgrep

_example/releases.js

@@ -1,37 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'BurntSushi';
-var repo = 'ripgrep';
-
-/******************************************************************************/
-/** Note: Delete this Comment!                                               **/
-/**                                                                          **/
-/** Need a an example that filters out miscellaneous release files?          **/
-/**   See `deno`, `gitea`, or `caddy`                                        **/
-/**                                                                          **/
-/******************************************************************************/
-
-let Releases = module.exports;
-
-Releases.latest = async function () {
-  let all = await github(null, owner, repo);
-  return all;
-};
-
-Releases.sample = async function () {
-  let normalize = require('../_webi/normalize.js');
-  let all = await Releases.latest();
-  all = normalize(all);
-  // just select the first 5 for demonstration
-  all.releases = all.releases.slice(0, 5);
-  return all;
-};
-
-if (module === require.main) {
-  (async function () {
-    let samples = await Releases.sample();
-
-    console.info(JSON.stringify(samples, null, 2));
-  })();
-}

_skills/installer/SKILL.md

@@ -1,442 +0,0 @@
----
-name: installer
-description: >
-  Create or update install.sh and install.ps1 scripts for a webi package.
-  Use when adding a new package to webi-installers, or when an existing
-  install script needs to be updated to match a changed archive structure.
-  Covers discovering archive layout from GitHub releases, identifying the
-  right install pattern (A–I), and writing both the POSIX shell and
-  PowerShell scripts that the webi framework calls.
-  Note: this skill covers install scripts only — writing releases.js /
-  releases.conf (the release-fetcher config) is a separate concern.
-license: MIT
-compatibility: Requires git, curl, tar. GitHub API access needed for
-  discovery phase. Designed for Claude Code in the webi-installers repo.
-metadata:
-  author: AJ ONeal
-  version: "1.1"
----
-
-# Webi Installer Skill
-
-Write `install.sh` and `install.ps1` for a webi package. These scripts are
-called by the webi framework **after** it has already downloaded and verified
-the archive — your job is only to unpack and place the files.
-
-> **Scope:** This skill covers `install.sh` and `install.ps1` only. A
-> separate `releases.js` / `releases.conf` file is needed to tell webi where
-> to fetch releases from. That config must already exist (or be written
-> separately) before these install scripts are useful.
-
-## Quick overview
-
-1. [Discover the archive layout](#1-discover-the-archive-layout) — inspect
-   GitHub releases with `curl` + `tar -t` to understand what's inside.
-2. [Choose the install pattern](#2-choose-the-install-pattern) — nine
-   patterns (A–I) cover almost every real-world case.
-3. [Write `install.sh`](#3-write-installsh) — POSIX shell, ~20–40 lines.
-4. [Write `install.ps1`](#4-write-installps1) — PowerShell, ~40–60 lines.
-5. [Check for classification issues](#5-check-for-classification-issues) —
-   look for variant assets, non-standard OS/arch naming, or installer .exe
-   files that need special handling.
-
-Full reference: [`references/PATTERNS.md`](references/PATTERNS.md)
-Archive layout details: [`references/ARCHIVE-LAYOUTS.md`](references/ARCHIVE-LAYOUTS.md)
-Classification guide: [`references/CLASSIFICATION.md`](references/CLASSIFICATION.md)
-
----
-
-## 1. Discover the archive layout
-
-### Use the webi releases API (fastest, if the package already exists)
-
-```sh
-# JSON with all releases for a package
-curl -s https://webinstall.dev/api/releases/bat.json | jq '.releases[:3]'
-```
-
-Each entry has `name` (filename), `version`, `os`, `arch`, `ext`, `download`.
-
-### Or inspect GitHub releases directly
-
-```sh
-# List asset filenames for the latest release
-curl -s "https://api.github.com/repos/sharkdp/bat/releases?per_page=3" \
-  | jq '.[0].assets[] | .name'
-```
-
-### Inspect what's inside an archive
-
-Download one representative asset and list its contents **without extracting**:
-
-```sh
-# tar.gz / tar.xz
-curl -fsSL "$DOWNLOAD_URL" | tar -tz
-
-# tar.zst (modern systems — GNU tar / bsdtar both support this)
-curl -fsSL "$DOWNLOAD_URL" | tar --zstd -tz
-
-# zip
-curl -fsSL "$DOWNLOAD_URL" -o /tmp/pkg.zip && unzip -l /tmp/pkg.zip
-
-# bare binary (no archive extension, e.g. jq-linux-amd64)
-# The file IS the binary — no unpacking needed. Set WEBI_SINGLE=true.
-```
-
-Look for:
-- Is the binary at the top level or inside a subdirectory?
-- Does the subdirectory name include the version and/or triplet?
-- Are there completions (`completions/`, `autocomplete/`, `complete/`)?
-- Are there man pages (`*.1`, `doc/*.1`, `man/man1/`)?
-- Are there shared libraries (`.so`, `.dylib`, `.dll`) alongside the binary?
-- Is the binary name different from the package command name?
-
-See [`references/ARCHIVE-LAYOUTS.md`](references/ARCHIVE-LAYOUTS.md) for
-what each pattern looks like, with real examples.
-
----
-
-## 2. Choose the install pattern
-
-| Pattern | Description | Examples |
-|---------|-------------|---------|
-| **A** | Bare binary (or binary+docs) at archive root | caddy, fzf, k9s, terraform |
-| **B** | Binary inside a version/triplet-named subdirectory | delta, shellcheck, trip, xsv |
-| **C** | Like B, plus shell completions and/or man pages | bat, fd, rg, sd, watchexec, zoxide |
-| **D** | Binary + shared libraries (bundled) | ollama (Linux), psql, sass, syncthing |
-| **E** | FHS-like layout (`bin/`, `share/man/`) | gh, pandoc |
-| **F** | Renamed binary needing install-time rename | pathman, yq |
-| **G** | Full SDK/toolchain (many files) | go, node, zig, flutter, julia |
-| **H** | .NET runtime bundle | pwsh |
-| **I** | Multi-binary distribution | dashcore, mutagen |
-
-**Pattern A** is by far the most common (~28 packages). When in doubt,
-download the archive and `tar -tz` it before writing a single line of code.
-
----
-
-## 3. Write `install.sh`
-
-The framework (`_webi/package-install.tpl.sh`) handles: user-agent detection,
-version resolution, download, checksum verification, and PATH management.
-Your script is **injected into** the framework and provides the
-package-specific part: where to find the binary and how to move it.
-
-### Script structure
-
-Every `install.sh` wraps its definitions in an `__init_pkgname()` function
-and immediately calls it. This prevents variable leakage when the script is
-sourced by the framework:
-
-```sh
-#!/bin/sh
-
-__init_toolname() {
-    set -e
-    set -u
-
-    ####################
-    # Install toolname #
-    ####################
-
-    pkg_cmd_name="toolname"
-    WEBI_SINGLE=true   # if applicable — see below
-
-    pkg_dst_cmd="$HOME/.local/bin/toolname"
-    pkg_dst="$pkg_dst_cmd"
-
-    pkg_src_cmd="$HOME/.local/opt/toolname-v$WEBI_VERSION/bin/toolname"
-    pkg_src_dir="$HOME/.local/opt/toolname-v$WEBI_VERSION"
-    pkg_src="$pkg_src_cmd"
-
-    pkg_install() {
-        # ...
-    }
-
-    pkg_get_current_version() {
-        # ...
-    }
-
-}
-
-__init_toolname
-```
-
-### Variables
-
-| Variable | Description |
-|----------|-------------|
-| `pkg_cmd_name` | The command name that ends up on `$PATH` |
-| `pkg_dst_cmd` | Final destination: `~/.local/bin/<cmd>` (the symlink) |
-| `pkg_dst` | Same as `pkg_dst_cmd` for single-binary packages; `~/.local/opt/<cmd>` for SDKs |
-| `pkg_src_cmd` | Versioned binary: `~/.local/opt/<pkg>-v<ver>/bin/<cmd>` |
-| `pkg_src_dir` | Versioned install dir: `~/.local/opt/<pkg>-v<ver>` |
-| `pkg_src` | Same as `pkg_src_cmd` for single-binary packages; same as `pkg_src_dir` for SDKs |
-
-**Framework-derived (set by the framework before calling `pkg_install` — do not set manually):**
-- `pkg_src_bin` — `$(dirname "$pkg_src_cmd")` — the versioned `bin/` dir
-- `pkg_dst_bin` — `$(dirname "$pkg_dst_cmd")` — `~/.local/bin`
-
-### `WEBI_SINGLE`
-
-`WEBI_SINGLE=true` affects the default values the framework uses for
-`pkg_src` and `pkg_dst`, and how `webi_link()` creates the symlink:
-
-- **With `WEBI_SINGLE=true`**: links the binary file directly:
-  `~/.local/bin/cmd → ~/.local/opt/cmd-vX.Y.Z/bin/cmd`
-- **Without it (default)**: links the directory:
-  `~/.local/opt/cmd → ~/.local/opt/cmd-vX.Y.Z`
-
-Set `WEBI_SINGLE=true` when using the conventional Pattern A skeleton
-(where `pkg_src` and `pkg_dst` are not set to custom values). When you
-explicitly assign all six variables yourself (as in Patterns B–F),
-`WEBI_SINGLE` is not strictly required but can still be set for clarity.
-
-Pattern G (SDKs) and Pattern H (.NET bundles) do NOT use `WEBI_SINGLE` —
-they define `pkg_link()` manually because the whole directory tree must
-be linked, not just a single binary.
-
-### Required function: `pkg_install`
-
-Moves files from the extracted archive into the versioned opt directory.
-The framework has already extracted the archive into a temp directory and
-`cd`'d into it before calling `pkg_install`.
-
-```sh
-pkg_install() {
-    mkdir -p "$pkg_src_bin"
-    mv ./tool-*/tool "$pkg_src_cmd"
-    chmod a+x "$pkg_src_cmd"
-}
-```
-
-### Recommended function: `pkg_get_current_version`
-
-Used to detect whether the package is already installed at the right version:
-
-```sh
-pkg_get_current_version() {
-    # 'tool --version' output: "tool 1.2.3 (rev abc)"
-    # trim to just the version number
-    tool --version 2>/dev/null | head -n 1 | cut -d' ' -f2
-}
-```
-
-### Skeletons by pattern
-
-**Pattern A** — binary at archive root (`WEBI_SINGLE=true`):
-```sh
-WEBI_SINGLE=true
-pkg_install() {
-    mkdir -p "$pkg_src_bin"
-    mv ./"$pkg_cmd_name"* "$pkg_src_cmd"
-    chmod a+x "$pkg_src_cmd"
-}
-```
-Use `$pkg_cmd_name*` as the glob — it matches the binary and avoids
-accidentally moving LICENSE or README into the binary path.
-
-**Pattern B** — binary inside a `tool-{ver}-{triplet}/` subdirectory:
-```sh
-WEBI_SINGLE=true
-pkg_install() {
-    mkdir -p "$pkg_src_bin"
-    mv ./tool-*/tool "$pkg_src_cmd"
-    chmod a+x "$pkg_src_cmd"
-}
-```
-
-**Pattern C** — like B, plus completions and man pages.
-The completion directory and filename vary per package — always check
-`tar -tz` output first. Common variants: `completions/`, `autocomplete/`,
-`complete/`. See [`references/PATTERNS.md`](references/PATTERNS.md) for
-a full example with guards:
-```sh
-WEBI_SINGLE=true
-pkg_install() {
-    mkdir -p "$pkg_src_bin"
-    mv ./tool-*/tool "$pkg_src_cmd"
-    chmod a+x "$pkg_src_cmd"
-
-    # bash completion (directory name varies — check tar -tz)
-    if test -e ./tool-*/completions/tool.bash; then
-        mkdir -p "$pkg_src_dir/share/bash-completion/completions"
-        mv ./tool-*/completions/tool.bash \
-            "$pkg_src_dir/share/bash-completion/completions/tool"
-    fi
-    if test -e ./tool-*/completions/tool.fish; then
-        mkdir -p "$pkg_src_dir/share/fish/vendor_completions.d"
-        mv ./tool-*/completions/tool.fish \
-            "$pkg_src_dir/share/fish/vendor_completions.d/tool.fish"
-    fi
-    if test -e ./tool-*/completions/_tool; then
-        mkdir -p "$pkg_src_dir/share/zsh/site-functions"
-        mv ./tool-*/completions/_tool \
-            "$pkg_src_dir/share/zsh/site-functions/_tool"
-    fi
-    if test -e ./tool-*/tool.1; then
-        mkdir -p "$pkg_src_dir/share/man/man1"
-        mv ./tool-*/tool.1 "$pkg_src_dir/share/man/man1/tool.1"
-    fi
-}
-```
-
-**Pattern D** — binary + shared libraries. The entire directory structure
-must be preserved. See [`references/PATTERNS.md`](references/PATTERNS.md)
-for the ollama and psql examples.
-
-**Pattern E** — FHS layout (archive already has `bin/` and `share/`):
-```sh
-WEBI_SINGLE=true
-pkg_install() {
-    mkdir -p "$(dirname "$pkg_src_dir")"
-    mv ./tool-*/ "$pkg_src_dir"
-}
-```
-
-**Pattern F** — binary needs rename (archive name ≠ command name).
-Use when the binary in the archive cannot be matched by `$pkg_cmd_name*`
-— e.g., `yq_linux_amd64` for a command named `yq`:
-```sh
-WEBI_SINGLE=true
-pkg_install() {
-    mkdir -p "$pkg_src_bin"
-    mv ./yq_* "$pkg_src_cmd"
-    chmod a+x "$pkg_src_cmd"
-}
-```
-
-**Pattern G** — full SDK (do NOT set `WEBI_SINGLE`):
-```sh
-# pkg_src = directory, not a binary
-pkg_src="$pkg_src_dir"
-pkg_dst="$HOME/.local/opt/tool"
-
-pkg_install() {
-    mkdir -p "$(dirname "$pkg_src_dir")"
-    mv ./tool-*/ "$pkg_src_dir"
-}
-
-pkg_link() {
-    rm -f "$pkg_dst"
-    ln -s "$pkg_src" "$pkg_dst"
-}
-```
-
----
-
-## 4. Write `install.ps1`
-
-A PowerShell framework template exists (`_webi/package-install.tpl.ps1`)
-and injects the `install.ps1` script at the `# {{ installer }}` placeholder.
-The template provides: error handling, directory setup, `Invoke-DownloadUrl`
-helper, and PATH management via `webi_path_add`. However, unlike the shell
-side, the PS1 framework does **not** download or extract the archive — the
-package script must handle that itself. The same path conventions apply
-(opt/bin layout), but Windows uses `Copy-Item` instead of symlinks for
-the final `bin/` step.
-
-### Variable block (always at top)
-
-```powershell
-$pkg_cmd_name = "tool"
-
-$pkg_dst_cmd = "$Env:USERPROFILE\.local\bin\tool.exe"
-$pkg_dst_bin = "$Env:USERPROFILE\.local\bin"
-$pkg_dst = "$pkg_dst_cmd"
-
-$pkg_src_cmd = "$Env:USERPROFILE\.local\opt\tool-v$Env:WEBI_VERSION\bin\tool.exe"
-$pkg_src_bin = "$Env:USERPROFILE\.local\opt\tool-v$Env:WEBI_VERSION\bin"
-$pkg_src_dir = "$Env:USERPROFILE\.local\opt\tool-v$Env:WEBI_VERSION"
-$pkg_src = "$pkg_src_cmd"
-```
-
-### Standard body
-
-```powershell
-New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
-$pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
-
-# Fetch archive
-if (!(Test-Path -Path "$pkg_download")) {
-    Write-Output "Downloading tool from $Env:WEBI_PKG_URL to $pkg_download"
-    & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
-    & Move-Item "$pkg_download.part" "$pkg_download"
-}
-
-if (!(Test-Path -Path "$pkg_src_cmd")) {
-    Write-Output "Installing tool"
-
-    Push-Location .local\tmp
-        Remove-Item -Path ".\tool-v*" -Recurse -ErrorAction Ignore
-
-        # Unpack — Windows BSD-tar handles zip too
-        Write-Output "Unpacking $pkg_download"
-        & tar xf "$pkg_download"
-
-        # Move binary into place — adjust glob for your archive structure
-        Write-Output "Install Location: $pkg_src_cmd"
-        New-Item "$pkg_src_bin" -ItemType Directory -Force | Out-Null
-        Move-Item -Path ".\tool-*\tool.exe" -Destination "$pkg_src_bin"
-    Pop-Location
-}
-
-# Windows has no symlinks in the webi sense — copy to bin/
-Write-Output "Copying into '$pkg_dst_cmd' from '$pkg_src_cmd'"
-Remove-Item -Path "$pkg_dst_cmd" -Recurse -ErrorAction Ignore | Out-Null
-New-Item "$pkg_dst_bin" -ItemType Directory -Force | Out-Null
-Copy-Item -Path "$pkg_src" -Destination "$pkg_dst" -Recurse
-```
-
-For Pattern A (binary at archive root), change the `Move-Item` line to:
-```powershell
-Move-Item -Path ".\tool.exe" -Destination "$pkg_src_bin"
-```
-
----
-
-## 5. Check for classification issues
-
-Before writing any scripts, scan the asset list for red flags:
-
-### Non-standard OS/arch names in filenames
-
-The webi classifier recognises most patterns automatically. Watch for:
-- `darwin` vs `macos` — both recognised; output normalised to `macos`
-- `x86_64` vs `amd64` — both recognised; output normalised to `amd64`
-- `aarch64` vs `arm64` — both recognised; output normalised to `arm64`
-- `armv7` (missing trailing `l`) — normalised to `armv7l`
-
-These are handled automatically. Only flag them if the asset list contains
-something genuinely unusual that the classifier would not recognise.
-
-### Variant assets needing tags
-
-Flag if you see multiple assets for the same OS/arch that serve different
-hardware or runtime requirements:
-- **GPU variants**: `*-rocm*`, `*-cuda*`, `*-vulkan*` alongside a baseline build
-- **Windows installer**: `*Setup.exe` or `*Install.exe` alongside a bare `*.exe`
-- **Framework-dependent .NET**: `*-fxdependent*` vs self-contained
-- **AppImage**: `*.AppImage` — not supported by the webi installer
-- **Electron/GUI app**: `*.dmg` or `*.AppImage` that is a full GUI app, not a CLI
-
-If you find variants, see [`references/CLASSIFICATION.md`](references/CLASSIFICATION.md)
-for how to write a variant tagger.
-
-### Formats to drop
-
-These are automatically filtered by the framework — no action needed:
-- `.deb`, `.rpm`, `.snap`, `.AppImage`
-- Checksums (`*.sha256`, `*.sha512`, `*.asc`, `*.sig`)
-- Source archives (`*-src.tar.gz`, `*.tar.gz` with no OS in name)
-
----
-
-## Reference files
-
-- [`references/PATTERNS.md`](references/PATTERNS.md) — detailed pattern
-  descriptions with real package examples and complete install script snippets
-- [`references/ARCHIVE-LAYOUTS.md`](references/ARCHIVE-LAYOUTS.md) — actual
-  `tar -t` output for representative packages in each pattern
-- [`references/CLASSIFICATION.md`](references/CLASSIFICATION.md) — when and
-  how to write variant taggers; non-standard filename conventions

_skills/installer/references/ARCHIVE-LAYOUTS.md

@@ -1,289 +0,0 @@
-# Archive Layouts — Real Package Examples
-
-Actual `tar -t` / `unzip -l` output for representative packages.
-Use these to calibrate your eye for what each pattern looks like.
-
----
-
-## Pattern A — Flat archive (no subdirectory)
-
-### caddy 2.9.1 — linux/amd64 tar.gz
-```
-caddy
-LICENSE
-README.md
-```
-Binary `caddy` is at the top level. Set `WEBI_SINGLE=true`.
-
-### fzf 0.70.0 — linux/amd64 tar.gz
-```
-fzf
-```
-Minimal — just the binary.
-
-### terraform 1.9.8 — linux/amd64 zip
-```
-terraform
-LICENSE.txt
-```
-Zip archive but same flat layout.
-
-### k9s — linux/amd64 tar.gz
-```
-k9s
-LICENSE
-README.md
-```
-
----
-
-## Pattern B — Named subdirectory, binary only
-
-### delta 0.18.2 — linux/amd64 tar.gz
-```
-delta-0.18.2-x86_64-unknown-linux-musl/
-delta-0.18.2-x86_64-unknown-linux-musl/delta
-delta-0.18.2-x86_64-unknown-linux-musl/LICENSE
-delta-0.18.2-x86_64-unknown-linux-musl/README.md
-```
-Glob to move: `./delta-*/delta`
-
-### shellcheck 0.10.0 — linux/x86_64 tar.xz
-```
-shellcheck-v0.10.0/
-shellcheck-v0.10.0/shellcheck
-shellcheck-v0.10.0/LICENSE.txt
-shellcheck-v0.10.0/README.txt
-```
-Glob to move: `./shellcheck-*/shellcheck`
-
-### xsv 0.13.0 — linux/x86_64 tar.gz
-```
-xsv-0.13.0-x86_64-unknown-linux-musl/
-xsv-0.13.0-x86_64-unknown-linux-musl/xsv
-xsv-0.13.0-x86_64-unknown-linux-musl/UNLICENSE
-```
-
----
-
-## Pattern C — Subdirectory + completions + man pages
-
-### rg/ripgrep 14.1.1 — linux/amd64 tar.gz
-```
-ripgrep-14.1.1-x86_64-unknown-linux-musl/
-ripgrep-14.1.1-x86_64-unknown-linux-musl/rg
-ripgrep-14.1.1-x86_64-unknown-linux-musl/complete/
-ripgrep-14.1.1-x86_64-unknown-linux-musl/complete/_rg
-ripgrep-14.1.1-x86_64-unknown-linux-musl/complete/_rg.ps1
-ripgrep-14.1.1-x86_64-unknown-linux-musl/complete/rg.bash
-ripgrep-14.1.1-x86_64-unknown-linux-musl/complete/rg.fish
-ripgrep-14.1.1-x86_64-unknown-linux-musl/doc/
-ripgrep-14.1.1-x86_64-unknown-linux-musl/doc/rg.1
-ripgrep-14.1.1-x86_64-unknown-linux-musl/doc/FAQ.md
-ripgrep-14.1.1-x86_64-unknown-linux-musl/doc/GUIDE.md
-ripgrep-14.1.1-x86_64-unknown-linux-musl/CHANGELOG.md
-ripgrep-14.1.1-x86_64-unknown-linux-musl/LICENSE-MIT
-ripgrep-14.1.1-x86_64-unknown-linux-musl/README.md
-```
-Note: completions are in `complete/` (not `completions/`). Man page is `doc/rg.1`.
-
-### sd 1.1.0 — linux/x86_64 tar.gz
-```
-sd-v1.1.0-x86_64-unknown-linux-musl/
-sd-v1.1.0-x86_64-unknown-linux-musl/sd
-sd-v1.1.0-x86_64-unknown-linux-musl/sd.1
-sd-v1.1.0-x86_64-unknown-linux-musl/completions/
-sd-v1.1.0-x86_64-unknown-linux-musl/completions/sd.bash
-sd-v1.1.0-x86_64-unknown-linux-musl/completions/sd.elv
-sd-v1.1.0-x86_64-unknown-linux-musl/completions/sd.fish
-sd-v1.1.0-x86_64-unknown-linux-musl/completions/_sd
-sd-v1.1.0-x86_64-unknown-linux-musl/completions/_sd.ps1
-sd-v1.1.0-x86_64-unknown-linux-musl/CHANGELOG.md
-sd-v1.1.0-x86_64-unknown-linux-musl/LICENSE
-sd-v1.1.0-x86_64-unknown-linux-musl/README.md
-```
-Note: man page `sd.1` is at subdirectory root. Completions in `completions/`.
-
-### bat 0.26.1 — linux/amd64 tar.gz
-```
-bat-v0.26.1-x86_64-unknown-linux-musl/
-bat-v0.26.1-x86_64-unknown-linux-musl/bat
-bat-v0.26.1-x86_64-unknown-linux-musl/bat.1
-bat-v0.26.1-x86_64-unknown-linux-musl/autocomplete/
-bat-v0.26.1-x86_64-unknown-linux-musl/autocomplete/bat.bash
-bat-v0.26.1-x86_64-unknown-linux-musl/autocomplete/bat.fish
-bat-v0.26.1-x86_64-unknown-linux-musl/autocomplete/bat.zsh
-bat-v0.26.1-x86_64-unknown-linux-musl/LICENSE-APACHE
-bat-v0.26.1-x86_64-unknown-linux-musl/LICENSE-MIT
-bat-v0.26.1-x86_64-unknown-linux-musl/README.md
-```
-Note: completions in `autocomplete/` (not `completions/`). Zsh file is `bat.zsh` not `_bat`.
-
-### goreleaser — linux/amd64 tar.gz
-```
-goreleaser
-completions/
-completions/goreleaser.bash
-completions/goreleaser.fish
-completions/goreleaser.zsh
-manpages/
-manpages/goreleaser.1.gz
-LICENSE.md
-README.md
-```
-Note: goreleaser uses Pattern A layout (binary at root, no subdirectory)
-but includes completions and a gzipped man page. Set `WEBI_SINGLE=true`;
-move completions and man page after the binary.
-
----
-
-## Pattern D — Binary + shared libraries
-
-### ollama 0.17.7 — linux/amd64 tar.zst
-```
-bin/
-bin/ollama
-lib/
-lib/ollama/
-lib/ollama/libggml-base.so
-lib/ollama/libggml-cpu-alderlake.so
-lib/ollama/libggml-cpu-haswell.so
-lib/ollama/libggml-cpu-icelake.so
-lib/ollama/libggml-cpu-sandybridge.so
-lib/ollama/libggml-cpu-skylakex.so
-lib/ollama/libggml-cpu-sse42.so
-lib/ollama/libggml-cpu-x64.so
-lib/ollama/cuda_v12/
-lib/ollama/cuda_v12/libcublas.so.12
-lib/ollama/cuda_v12/libcublasLt.so.12
-lib/ollama/cuda_v12/libcudart.so.12
-lib/ollama/cuda_v12/libggml-cuda.so
-... (66 files total)
-```
-Extract bin/ and lib/ directories separately or together.
-
-### psql (postgres client) — linux/amd64 tar.gz
-```
-psql-17.2-linux-x86_64/
-psql-17.2-linux-x86_64/bin/
-psql-17.2-linux-x86_64/bin/psql
-psql-17.2-linux-x86_64/lib/
-psql-17.2-linux-x86_64/lib/libpq.so.5
-psql-17.2-linux-x86_64/lib/libz.so.1
-psql-17.2-linux-x86_64/lib/libzstd.so.1
-psql-17.2-linux-x86_64/lib/libssl.so.3
-psql-17.2-linux-x86_64/lib/libcrypto.so.3
-psql-17.2-linux-x86_64/include/
-... (75 files total)
-```
-Move the entire `psql-{ver}-{triplet}/` directory: `mv ./psql-*/ "$pkg_src_dir"`
-
----
-
-## Pattern E — FHS layout
-
-### gh 2.67.0 — linux/amd64 tar.gz
-```
-gh_2.67.0_linux_amd64/
-gh_2.67.0_linux_amd64/bin/
-gh_2.67.0_linux_amd64/bin/gh
-gh_2.67.0_linux_amd64/share/
-gh_2.67.0_linux_amd64/share/man/
-gh_2.67.0_linux_amd64/share/man/man1/
-gh_2.67.0_linux_amd64/share/man/man1/gh-actions-cache-delete.1
-gh_2.67.0_linux_amd64/share/man/man1/gh-actions-cache-list.1
-... (129 man pages)
-gh_2.67.0_linux_amd64/LICENSE
-```
-Move the entire `gh_*/` directory: `mv ./gh_*/ "$pkg_src_dir"`
-
----
-
-## Pattern F — Binary needs rename
-
-### yq — linux/amd64 tar.gz (WEBI_SINGLE=true)
-```
-yq_linux_amd64
-yq.1
-```
-Binary is `yq_linux_amd64` — must rename to `yq` during install.
-
-### pathman 0.6.0 — linux/amd64 tar.gz (WEBI_SINGLE=true)
-```
-pathman-v0.6.0-linux-amd64_v1
-```
-Binary name includes the full release tag. Rename to `pathman`.
-
----
-
-## Pattern G — Full SDK
-
-### node 24.14.0 — linux/amd64 tar.xz
-```
-node-v24.14.0-linux-x64/
-node-v24.14.0-linux-x64/bin/
-node-v24.14.0-linux-x64/bin/node
-node-v24.14.0-linux-x64/bin/npm     -> ../lib/node_modules/npm/bin/npm-cli.js
-node-v24.14.0-linux-x64/bin/npx     -> ../lib/node_modules/npm/bin/npx-cli.js
-node-v24.14.0-linux-x64/include/
-node-v24.14.0-linux-x64/lib/
-node-v24.14.0-linux-x64/lib/node_modules/
-node-v24.14.0-linux-x64/share/
-... (thousands of files)
-```
-Move entire directory: `mv ./node-*/ "$pkg_src_dir"`
-
-### go 1.24.1 — linux/amd64 tar.gz
-```
-go/
-go/bin/
-go/bin/go
-go/bin/gofmt
-go/src/
-go/pkg/
-... (thousands of files)
-```
-Note: go's archive root directory is literally `go/` with no version in the name.
-
----
-
-## Pattern H — .NET runtime bundle
-
-### pwsh 7.4.6 — linux/amd64 tar.gz
-```
-pwsh
-Accessibility.dll
-clrcompression.dll
-clrjit.dll
-createdump
-cs/
-cs/System.Private.CoreLib.resources.dll
-de/
-de/System.Private.CoreLib.resources.dll
-... (727 files, all in same flat directory)
-```
-No subdirectory. Move all files into `$pkg_src_bin/`.
-
----
-
-## Inspecting archives yourself
-
-```sh
-# tar.gz / tar.xz / tar.zst — list contents only (no extraction)
-curl -fsSL "$URL" | tar -tz | head -20
-
-# zip
-curl -fsSL "$URL" -o /tmp/pkg.zip
-unzip -l /tmp/pkg.zip | head -20
-
-# For a .zst file when tar doesn't support zstd natively:
-curl -fsSL "$URL" -o /tmp/pkg.tar.zst && zstd -dc /tmp/pkg.tar.zst | tar -tz | head -20
-```
-
-**What to look for**:
-1. Is there a top-level directory? (Pattern B/C/D/E/G) or no directory? (Pattern A/F/H)
-2. What is the directory named? Does it contain version? triplet?
-3. Are there `completions/`, `autocomplete/`, `complete/` subdirs? (Pattern C)
-4. Are there `.so`/`.dylib`/`.dll` files? (Pattern D or H)
-5. Does the binary name match the command you want on PATH? (Pattern F if not)
-6. Is there a `bin/` directory at the top level? (Pattern E or G)

_skills/installer/references/CLASSIFICATION.md

@@ -1,183 +0,0 @@
-# Classification Reference
-
-When to flag classification issues, what the webi classifier does automatically,
-and what needs manual annotation.
-
----
-
-## What the classifier handles automatically
-
-The webi classifier (`internal/classify/classify.go`) parses asset filenames
-using regex patterns and produces canonical `os`, `arch`, `libc`, and `ext`
-values. It handles the vast majority of packages with no configuration needed.
-
-### OS recognition
-Filenames containing these terms are classified automatically:
-- `darwin`, `macos`, `osx`, `apple` → `macos` in legacy cache
-- `linux` → `linux`
-- `windows`, `win`, `win32`, `win64` → `windows`
-- `freebsd`, `openbsd`, `netbsd`, `dragonfly` → respective values
-- `.deb`, `.rpm`, `.snap` → `linux` (but dropped from legacy cache)
-- `.dmg`, `.app.zip` → `macos`
-
-### Arch recognition
-Filenames containing these terms are classified automatically:
-- `x86_64`, `amd64`, `64bit`, `x64` → `amd64`
-- `aarch64`, `arm64` → `arm64`
-- `armv7`, `armv7l`, `armhf`, `gnueabihf` → `armv7l`
-- `armv6`, `armv6l` → `armv6l`
-- `i386`, `i686`, `386`, `x86` → `x86`
-- `universal`, `universal2` → `amd64` (fat binary; arm64 falls back to this)
-
-### Format recognition
-- `.tar.gz`, `.tar.xz`, `.tar.zst`, `.tar.bz2`, `.zip`, `.7z` → compressed archive
-- `.pkg`, `.msi`, `.dmg` → platform installer
-- `.exe` → either bare binary or GUI installer (see below)
-- No extension in filename → bare binary (ext = `exe` in cache)
-
-### Automatically dropped
-These asset types are recognised and excluded without any configuration:
-- Checksums: `*.sha256`, `*.sha512`, `*.md5`, `*.sha256sum`
-- Signatures: `*.asc`, `*.sig`, `*.cosign`, `*.sbom`
-- Source archives: files with `source`, `src` in the name but no OS
-- Package formats not supported by the Node installer: `.deb`, `.rpm`, `.snap`,
-  `.AppImage`, `.apk`
-
----
-
-## When you need to add configuration
-
-### Variant assets
-
-A **variant** is a secondary build that serves the same OS/arch as a baseline
-build but requires different hardware or runtime support. The Node.js installer
-can't choose between variants — it only knows OS, arch, and libc. Variants
-must be tagged and then excluded at export time.
-
-**Common variants and how to identify them**:
-
-| Variant | Filename pattern | Notes |
-|---------|-----------------|-------|
-| CUDA (GPU) | `*-cuda*`, `*cuda12*` | NVIDIA GPU support |
-| ROCm (GPU) | `*-rocm*` | AMD GPU support |
-| Vulkan | `*-vulkan*` | Cross-vendor GPU |
-| AppImage | `*.AppImage` | Linux sandboxed app |
-| .NET fxdependent | `*-fxdependent*` | Requires .NET runtime |
-| Windows installer | `*Setup.exe`, `*Install.exe` | GUI installer, not the binary |
-
-**Rule**: if there are multiple assets for the same OS/arch combination and
-they serve the same users differently, they need variant tags. The baseline
-(most widely compatible) build should be kept; variants should be tagged and
-excluded.
-
-**Example**: ollama publishes for linux/amd64:
-- `ollama-linux-amd64.tar.zst` — baseline (CPU + any GPU auto-detected)
-- `ollama-linux-amd64-rocm.tar.zst` — ROCm variant
-- `ollama-linux-amd64-jetpack6.tar.zst` — NVIDIA Jetson variant
-
-Only the baseline is useful via webi. The ROCm and Jetpack builds should be
-tagged as variants and excluded.
-
----
-
-### Windows .exe: bare binary vs GUI installer
-
-`.exe` assets are ambiguous — they could be:
-1. A bare binary (the tool itself, run from command line)
-2. A GUI installer (runs a setup wizard, not useful for webi)
-
-**How to tell**:
-- GUI installer: filename contains `Setup`, `Install`, `Installer`, `inno`, `nsis`
-- GUI installer: the tool also has a `.zip` or `.tar.gz` for Windows
-- Bare binary: filename matches the tool name with minimal decoration
-
-**When you see both**, the `.zip`/archive build is what webi uses. The `.exe`
-installer should be tagged as a variant (`installer`) so it's excluded.
-
-**When there's only a `.exe`** (no archive), it's probably the bare binary.
-Test by downloading and running it — a bare binary runs immediately.
-
----
-
-### Packages with no OS/arch in filenames
-
-Some packages (rare) release with minimal filename decoration. Examples:
-- `tool-v1.2.3.tar.gz` — no OS, no arch
-- `tool.tar.gz` — version not even in filename
-
-These are usually source archives (not compiled binaries) and should be
-dropped entirely from the release list. If they are compiled binaries for a
-specific OS, the releases.js config needs an `asset_filter` key to match the
-right file, plus OS/arch metadata added.
-
----
-
-### Non-standard OS naming in filenames
-
-A few upstreams use unusual OS names:
-- `sunos` — should map to `solaris` (the webi classifier does this)
-- `osx` or `macosx` — recognised as `macos`
-- `apple-darwin` (Rust triplet) — recognised as `macos`
-
-If a package uses a genuinely unknown OS string, the classifier will produce
-`os = ""` for that asset. Those entries are dropped from the legacy cache.
-
----
-
-### Asset filter configuration
-
-If GitHub releases for a package include multiple builds that would otherwise
-collide (e.g. `extended` vs non-extended for hugo, or specific project builds
-in a monorepo), add to the package's `releases.conf`:
-
-```ini
-# Only include assets containing "extended" in the name
-asset_filter = extended
-
-# Exclude assets containing "legacy" in the name
-asset_exclude = legacy
-```
-
-These filters run before classification.
-
----
-
-## Quick checklist when inspecting a new package
-
-1. **Look at the latest 2–3 releases** on GitHub. Note all asset filenames.
-2. **Find the "standard" builds** — the ones a normal user would download for
-   their OS. Usually there are ≤4 per OS (amd64, arm64, x86, armv7l).
-3. **Check for extras**:
-   - Are there GPU-specific builds for the same OS/arch? → variant
-   - Are there `.exe` installer files alongside a `.zip`? → variant
-   - Are there `.deb`/`.rpm`/`.AppImage`? → auto-dropped, no action needed
-   - Does the Windows build have no archive and only a bare `.exe`? → fine
-4. **Check OS/arch naming** — does the filename use standard terms, or
-   something unusual that might confuse the classifier?
-5. **Check format changes** — do old releases use a different archive type
-   or directory layout than recent ones? The install script may need to
-   handle both.
-
----
-
-## Canonical vocabulary reference
-
-All cache output must use exactly these values.
-
-**OS**: `macos`, `linux`, `windows`, `freebsd`, `openbsd`, `netbsd`,
-`dragonfly`, `aix`, `illumos`, `plan9`, `solaris`
-
-**Arch**:
-- `amd64` (not `x86_64`)
-- `arm64` (not `aarch64`)
-- `armv7l` (not `armv7` — the `l` stands for little-endian; `uname -m` reports `armv7l`)
-- `armv6l` (not `armv6`)
-- `x86` (not `i386`, `i686`, `386`)
-- `mipsle` (not `mipsel`)
-- `mips64le` (not `mips64el`)
-- Other: `arm`, `ppc64le`, `ppc64`, `loong64`, `riscv64`, `s390x`, `mips`, `mips64`
-
-**Libc**: `none` (static/Go/Zig — never empty), `gnu`, `musl`, `msvc`
-
-**Ext**: `tar.gz`, `tar.xz`, `zip`, `exe`, `7z`, `pkg`, `msi`
-(no leading dot; `exe` for bare binaries with no file extension)

_skills/installer/references/PATTERNS.md

@@ -1,388 +0,0 @@
-# Install Patterns Reference
-
-Nine patterns cover the full range of webi packages. Pattern A is by far
-the most common. Check `tar -tz $ARCHIVE` before writing any code.
-
----
-
-## Pattern A — Bare binary at archive root
-
-The archive extracts directly to the current directory with no wrapper
-subdirectory. Binary (and optional LICENSE/README) is at the top level.
-
-**Set `WEBI_SINGLE=true`** — tells the framework to link the binary file
-directly (`~/.local/bin/cmd → ~/.local/opt/cmd-vX/bin/cmd`) rather than
-linking the versioned directory.
-
-Representative packages: caddy, fzf, k9s, terraform, sttr, lf, monorel,
-awless, bun, cilium, curlie, dashmsg, dotenv, dotenv-linter, ffuf,
-gitdeploy, gprox, grype, hugo, keypairs, koji, ots, runzip, sclient,
-sqlc, sqlpkg, uuidv7, xcaddy, deno
-
-**install.sh**:
-```sh
-pkg_cmd_name="caddy"
-WEBI_SINGLE=true
-
-pkg_dst_cmd="$HOME/.local/bin/caddy"
-pkg_dst="$pkg_dst_cmd"
-pkg_src_cmd="$HOME/.local/opt/caddy-v$WEBI_VERSION/bin/caddy"
-pkg_src_dir="$HOME/.local/opt/caddy-v$WEBI_VERSION"
-pkg_src="$pkg_src_cmd"
-
-pkg_install() {
-    mkdir -p "$pkg_src_bin"
-    mv ./"$pkg_cmd_name"* "$pkg_src_cmd"
-    chmod a+x "$pkg_src_cmd"
-}
-
-pkg_get_current_version() {
-    caddy version 2>/dev/null | head -n 1 | cut -d' ' -f1 | sed 's:^v::'
-}
-```
-
-**install.ps1** key lines:
-```powershell
-# No subdirectory — binary is at the top level of the archive
-Move-Item -Path ".\caddy.exe" -Destination "$pkg_src_bin"
-```
-
----
-
-## Pattern B — Binary inside a version/triplet subdirectory
-
-Archive extracts to a single directory named with the version and/or
-platform triplet. Binary (and docs) live inside that directory.
-
-Representative packages: delta, hexyl, shellcheck, trip, xsv, kubectx, kubens
-
-**Subdirectory naming conventions seen in the wild**:
-- `tool-{ver}-{triplet}/` — most Rust tools (delta, shellcheck, xsv)
-- `tool-{ver}/` — simpler version-only dirs
-- flat (no dir) — kubectx/kubens use flat archives despite being "B-ish"
-
-**install.sh**:
-```sh
-pkg_cmd_name="delta"
-# WEBI_SINGLE not set (or false)
-
-pkg_dst_cmd="$HOME/.local/bin/delta"
-pkg_dst="$pkg_dst_cmd"
-pkg_src_cmd="$HOME/.local/opt/delta-v$WEBI_VERSION/bin/delta"
-pkg_src_dir="$HOME/.local/opt/delta-v$WEBI_VERSION"
-pkg_src="$pkg_src_cmd"
-
-pkg_install() {
-    mkdir -p "$pkg_src_bin"
-    mv ./delta-*/delta "$pkg_src_cmd"
-    chmod a+x "$pkg_src_cmd"
-}
-
-pkg_get_current_version() {
-    delta --version 2>/dev/null | head -n 1 | cut -d' ' -f2
-}
-```
-
-**install.ps1** key lines:
-```powershell
-Move-Item -Path ".\delta-*\delta.exe" -Destination "$pkg_src_bin"
-```
-
----
-
-## Pattern C — Subdirectory with binary + completions and/or man pages
-
-Same as B but the archive also contains shell completions and/or man pages
-worth installing.
-
-Representative packages: bat, fd, lsd, rg/ripgrep, sd, watchexec, zoxide
-
-Note: goreleaser has a flat archive (Pattern A layout) but with completions at
-the archive root. See the goreleaser entry in ARCHIVE-LAYOUTS.md.
-
-**Completion directory name varies by package**:
-- `completions/` — sd, watchexec, zoxide
-- `autocomplete/` — bat, fd, lsd
-- `complete/` — rg/ripgrep
-
-**Completion filename conventions**:
-- Bash: `tool.bash`, `tool.bash-completion`, `_tool.bash`
-- Fish: `tool.fish`
-- Zsh: `_tool`
-- PowerShell: `_tool.ps1`, `tool.ps1`
-
-**Man page location varies**:
-- `tool.1` at subdirectory root — sd, bat, fd, lsd
-- `doc/tool.1` — rg/ripgrep
-- `man/man1/tool.1` — zoxide (deepest path)
-
-**install.sh** (rg as example):
-```sh
-pkg_cmd_name="rg"
-
-pkg_dst_cmd="$HOME/.local/bin/rg"
-pkg_dst="$pkg_dst_cmd"
-pkg_src_cmd="$HOME/.local/opt/rg-v$WEBI_VERSION/bin/rg"
-pkg_src_dir="$HOME/.local/opt/rg-v$WEBI_VERSION"
-pkg_src="$pkg_src_cmd"
-
-pkg_install() {
-    mkdir -p "$pkg_src_bin"
-    mv ./ripgrep-*/rg "$pkg_src_cmd"
-    chmod a+x "$pkg_src_cmd"
-
-    # bash completion
-    if test -e ./ripgrep-*/complete/rg.bash; then
-        mkdir -p "$pkg_src_dir/share/bash-completion/completions"
-        mv ./ripgrep-*/complete/rg.bash \
-            "$pkg_src_dir/share/bash-completion/completions/rg"
-    fi
-    # fish completion
-    if test -e ./ripgrep-*/complete/rg.fish; then
-        mkdir -p "$pkg_src_dir/share/fish/vendor_completions.d"
-        mv ./ripgrep-*/complete/rg.fish \
-            "$pkg_src_dir/share/fish/vendor_completions.d/rg.fish"
-    fi
-    # zsh completion
-    if test -e ./ripgrep-*/complete/_rg; then
-        mkdir -p "$pkg_src_dir/share/zsh/site-functions"
-        mv ./ripgrep-*/complete/_rg \
-            "$pkg_src_dir/share/zsh/site-functions/_rg"
-    fi
-    # man page
-    if test -e ./ripgrep-*/doc/rg.1; then
-        mkdir -p "$pkg_src_dir/share/man/man1"
-        mv ./ripgrep-*/doc/rg.1 "$pkg_src_dir/share/man/man1/rg.1"
-    fi
-}
-
-pkg_get_current_version() {
-    rg --version 2>/dev/null | head -n 1 | cut -d' ' -f2
-}
-```
-
-**Note**: Completion paths in completions/man install are best-effort
-— use `if test -e ...` guards so the script still works on older releases
-that didn't include them.
-
----
-
-## Pattern D — Binary + shared libraries
-
-The package bundles shared libraries alongside the binary. The entire
-directory tree must be preserved.
-
-Representative packages: ollama (Linux), psql/postgres, sass (Dart VM),
-syncthing, xz
-
-**install.sh**:
-```sh
-pkg_cmd_name="ollama"
-
-pkg_dst_cmd="$HOME/.local/bin/ollama"
-pkg_dst="$pkg_dst_cmd"
-pkg_src_cmd="$HOME/.local/opt/ollama-v$WEBI_VERSION/bin/ollama"
-pkg_src_dir="$HOME/.local/opt/ollama-v$WEBI_VERSION"
-pkg_src="$pkg_src_cmd"
-
-pkg_install() {
-    mkdir -p "$(dirname "$pkg_src_dir")"
-    # Archive already has bin/ and lib/ layout
-    mv ./bin "$pkg_src_dir/bin"
-    mv ./lib "$pkg_src_dir/lib"
-}
-```
-
-For psql (archive has a `psql-{ver}-{triplet}/` wrapper dir):
-```sh
-pkg_install() {
-    mkdir -p "$(dirname "$pkg_src_dir")"
-    mv ./psql-*/ "$pkg_src_dir"
-}
-```
-
----
-
-## Pattern E — FHS-like layout
-
-Archive already follows `bin/`, `share/man/`, `share/doc/` hierarchy.
-Extract the whole thing directly into the versioned opt directory.
-
-Representative packages: gh (GitHub CLI), pandoc
-
-**install.sh**:
-```sh
-pkg_cmd_name="gh"
-
-pkg_dst_cmd="$HOME/.local/bin/gh"
-pkg_dst="$pkg_dst_cmd"
-pkg_src_cmd="$HOME/.local/opt/gh-v$WEBI_VERSION/bin/gh"
-pkg_src_dir="$HOME/.local/opt/gh-v$WEBI_VERSION"
-pkg_src="$pkg_src_cmd"
-
-pkg_install() {
-    mkdir -p "$(dirname "$pkg_src_dir")"
-    mv ./gh_*/ "$pkg_src_dir"
-}
-
-pkg_get_current_version() {
-    gh --version 2>/dev/null | head -n 1 | cut -d' ' -f3
-}
-```
-
-No `chmod` needed — binary is already executable inside the archive.
-
----
-
-## Pattern F — Binary needs rename
-
-Binary in the archive doesn't match the expected command name.
-
-Representative packages: pathman (`pathman-v0.6.0-linux-amd64_v1` → `pathman`),
-yq (`yq_linux_amd64` → `yq`)
-
-**install.sh**:
-```sh
-pkg_cmd_name="yq"
-WEBI_SINGLE=true
-
-pkg_dst_cmd="$HOME/.local/bin/yq"
-pkg_dst="$pkg_dst_cmd"
-pkg_src_cmd="$HOME/.local/opt/yq-v$WEBI_VERSION/bin/yq"
-pkg_src_dir="$HOME/.local/opt/yq-v$WEBI_VERSION"
-pkg_src="$pkg_src_cmd"
-
-pkg_install() {
-    mkdir -p "$pkg_src_bin"
-    # Binary is named yq_linux_amd64 (or yq_darwin_amd64 etc)
-    mv ./yq_* "$pkg_src_cmd"
-    chmod a+x "$pkg_src_cmd"
-}
-```
-
----
-
-## Pattern G — Full SDK / toolchain
-
-Archive contains a complete runtime or SDK (hundreds to thousands of files).
-The entire tree goes into opt; multiple binaries are linked from `bin/`.
-
-Representative packages: go, node, zig, flutter, julia, cmake, tinygo
-
-**install.sh** (node as example):
-```sh
-pkg_cmd_name="node"
-# NOTE: pkg_src points to the directory, not a binary
-
-pkg_dst_cmd="$HOME/.local/bin/node"
-pkg_dst="$HOME/.local/opt/node"   # versioned-dir symlink target
-
-pkg_src_cmd="$HOME/.local/opt/node-v$WEBI_VERSION/bin/node"
-pkg_src_dir="$HOME/.local/opt/node-v$WEBI_VERSION"
-pkg_src="$pkg_src_dir"            # pkg_src = the directory
-
-pkg_install() {
-    mkdir -p "$(dirname "$pkg_src")"
-    mv ./node-*/ "$pkg_src"
-}
-
-pkg_link() {
-    rm -f "$pkg_dst"
-    ln -s "$pkg_src" "$pkg_dst"
-}
-
-pkg_get_current_version() {
-    node --version 2>/dev/null | head -n 1 | sed 's:^v::'
-}
-```
-
----
-
-## Pattern H — .NET runtime bundle
-
-Flat directory with one binary and hundreds of `.dll` files. The entire
-directory must be preserved. Like Pattern G (SDK) in structure — the
-versioned directory is the package root, with the binary directly inside
-(no `bin/` subdirectory). A `pkg_link()` creates the unversioned symlink.
-
-Representative packages: pwsh (PowerShell Core)
-
-**install.sh**:
-```sh
-pkg_cmd_name="pwsh"
-
-# note: binary is at pkg_src_dir root, no bin/ subdirectory
-pkg_src_cmd="$HOME/.local/opt/pwsh-v$WEBI_VERSION/pwsh"
-pkg_src_dir="$HOME/.local/opt/pwsh-v$WEBI_VERSION"
-pkg_src="$pkg_src_dir"
-
-pkg_dst_cmd="$HOME/.local/opt/pwsh/pwsh"
-pkg_dst="$HOME/.local/opt/pwsh"
-
-pkg_install() {
-    # Archive extracts flat — move all contents into the versioned dir
-    mkdir -p "$pkg_src_dir"
-    mv ./* "$pkg_src_dir"
-    chmod a+x "$pkg_src_cmd"
-}
-
-pkg_link() {
-    rm -rf "$pkg_dst"
-    ln -s "$pkg_src" "$pkg_dst"
-}
-```
-
----
-
-## Pattern I — Multi-binary distribution
-
-Archive contains multiple related binaries. Install the primary one and
-link only that.
-
-Representative packages: dashcore (dashd + dash-cli + dash-qt + ...),
-mutagen (mutagen + mutagen-agents.tar.gz)
-
-**install.sh** (dashcore-style):
-```sh
-pkg_cmd_name="dashd"
-
-pkg_dst_cmd="$HOME/.local/bin/dashd"
-pkg_dst="$pkg_dst_cmd"
-pkg_src_cmd="$HOME/.local/opt/dashcore-v$WEBI_VERSION/bin/dashd"
-pkg_src_dir="$HOME/.local/opt/dashcore-v$WEBI_VERSION"
-pkg_src="$pkg_src_cmd"
-
-pkg_install() {
-    mkdir -p "$(dirname "$pkg_src_dir")"
-    mv ./dashcore-*/ "$pkg_src_dir"
-}
-```
-
----
-
-## Choosing between patterns
-
-```
-Archive root contains a single binary (or binary + docs)?
-  → Pattern A  (set WEBI_SINGLE=true)
-
-Archive has a named subdirectory wrapping the binary?
-  ├─ Binary only inside subdir?         → Pattern B
-  ├─ Binary + completions/man pages?    → Pattern C
-  └─ Binary + shared libraries (.so)?  → Pattern D
-
-Archive already has bin/ and share/ layout?
-  → Pattern E
-
-Binary name doesn't match the command name?
-  → Pattern F  (rename during install)
-
-Archive is a full SDK (compiler, runtime, stdlib)?
-  → Pattern G  (pkg_src = pkg_src_dir)
-
-Flat directory with many DLLs (.NET)?
-  → Pattern H
-
-Multiple binaries for a single distributed system?
-  → Pattern I
-```

_webi/builds-cacher-test.js

@@ -13,7 +13,6 @@ async function main() {
     caches: CACHE_DIR,
     installers: INSTALLERS_DIR,
   });
-  bc.freshenRandomPackage(600 * 1000);
 
   // let dirs = await bc.getProjectsByType();
   // let projNames = Object.keys(dirs.valid);

_webi/builds-cacher.js

@@ -3,13 +3,16 @@
 var BuildsCacher = module.exports;
 
 let Fs = require('node:fs/promises');
+let Os = require('node:os');
 let Path = require('node:path');
 
+let LEGACY_CACHE_DIR = Path.join(Os.homedir(), '.cache/webi/legacy');
+
 let HostTargets = require('./build-classifier/host-targets.js');
 let Lexver = require('./build-classifier/lexver.js');
 let Triplet = require('./build-classifier/triplet.js');
 
-var ALIAS_RE = /^alias: (\w+)$/m;
+var ALIAS_RE = /^alias: ([\w.-]+)$/m;
 
 var LEGACY_ARCH_MAP = {
   '*': 'ANYARCH',
@@ -126,61 +129,8 @@ async function readFirstBytes(path) {
   return str;
 }
 
-let promises = {};
-async function getLatestBuilds(Releases, installersDir, cacheDir, name, date) {
-  console.info(`[INFO] getLatestBuilds: ${name}`);
-
-  if (!Releases) {
-    Releases = require(`${installersDir}/${name}/releases.js`);
-  }
-  // TODO update all releases files with module.exports.xxxx = 'foo';
-  if (!Releases.latest) {
-    Releases.latest = Releases;
-  }
-
-  let id = `${cacheDir}/${name}`;
-  if (!promises[id]) {
-    promises[id] = Promise.resolve();
-  }
-
-  promises[id] = promises[id].then(async function () {
-    return await getLatestBuildsInner(Releases, cacheDir, name, date);
-  });
-
-  return await promises[id];
-}
-
-async function getLatestBuildsInner(Releases, cacheDir, name, date) {
-  let data = await Releases.latest();
-
-  if (!date) {
-    date = new Date();
-  }
-  let isoDate = date.toISOString();
-  let yearMonth = isoDate.slice(0, 7);
-
-  // TODO hash file
-  let dataFile = `${cacheDir}/${yearMonth}/${name}.json`;
-  // TODO fsstat releases.js vs require-ing time as well
-  let tsFile = `${cacheDir}/${yearMonth}/${name}.updated.txt`;
-
-  let dirPath = Path.dirname(dataFile);
-  await Fs.mkdir(dirPath, { recursive: true });
-
-  let json = JSON.stringify(data, null, 2);
-  await Fs.writeFile(dataFile, json, 'utf8');
-
-  let seconds = date.valueOf();
-  let ms = seconds / 1000;
-  let msStr = ms.toFixed(3);
-  await Fs.writeFile(tsFile, msStr, 'utf8');
-
-  return data;
-}
-
-BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
+BuildsCacher.create = function ({ ALL_TERMS, installers }) {
   let installersDir = installers;
-  let cacheDir = caches;
 
   if (!ALL_TERMS) {
     ALL_TERMS = Triplet.TERMS_PRIMARY_MAP;
@@ -195,9 +145,11 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
   bc._triplets = {};
   bc._targetsByBuildIdCache = {};
   bc._caches = {};
-  bc._staleAge = 15 * 60 * 1000;
   bc._allFormats = {};
   bc._allTriplets = {};
+  // Per-name lock: serializes cold-cache getPackages so concurrent
+  // callers can't corrupt bc._caches[name] via a transformAndUpdate race.
+  bc._inflight = {};
 
   for (let term of TERMS_META) {
     delete bc.orphanTerms[term];
@@ -216,19 +168,6 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     let entries = await Fs.readdir(installersDir, { withFileTypes: true });
     for (let entry of entries) {
       let meta = await bc.getProjectTypeByEntry(entry);
-      if (meta.type === 'not_found') {
-        let err = meta.detail;
-        console.error('');
-        console.error('PROBLEM');
-        console.error(`    ${err.message}`);
-        console.error('');
-        console.error('SOLUTION');
-        console.error('    npm clean-install');
-        console.error('');
-        throw new Error(
-          '[SANITY FAIL] should never have missing modules in prod',
-        );
-      }
       dirs[meta.type][entry.name] = meta.detail;
     }
 
@@ -297,19 +236,16 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
       return { type: 'alias', detail: link };
     }
 
-    let releasesPath = Path.join(path, 'releases.js');
-    try {
-      void require(releasesPath);
-    } catch (err) {
-      if (err.code !== 'MODULE_NOT_FOUND') {
-        return { type: 'errors', detail: err };
-      }
-
-      if (err.message.includes(`Cannot find module '${releasesPath}'`)) {
-        return { type: 'selfhosted', detail: true };
-      }
-
-      return { type: 'not_found', detail: err };
+    let cacheFile = `${LEGACY_CACHE_DIR}/${entry.name}.json`;
+    let hasCacheFile = await Fs.access(cacheFile)
+      .then(function () {
+        return true;
+      })
+      .catch(function () {
+        return false;
+      });
+    if (!hasCacheFile) {
+      return { type: 'selfhosted', detail: true };
     }
 
     return { type: 'valid', detail: true };
@@ -317,14 +253,26 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
 
   // Typically a package is organized by release (ex: go has 1.20, 1.21, etc),
   // but we will organize by the build (ex: go1.20-darwin-arm64.tar.gz, etc).
-  bc.getPackages = async function ({ Releases, name, date }) {
-    if (!date) {
-      date = new Date();
+  bc.getPackages = async function (args) {
+    let name = args.name;
+    let warm = bc._caches[name];
+    if (warm) {
+      return _doGetPackages(args);
     }
-    let isoDate = date.toISOString();
-    let yearMonth = isoDate.slice(0, 7);
-    let dataFile = `${cacheDir}/${yearMonth}/${name}.json`;
-    let tsFile = `${cacheDir}/${yearMonth}/${name}.updated.txt`;
+    let inflight = bc._inflight[name];
+    if (inflight) {
+      return inflight;
+    }
+    let p = _doGetPackages(args).finally(function () {
+      delete bc._inflight[name];
+    });
+    bc._inflight[name] = p;
+    return p;
+  };
+
+  async function _doGetPackages({ name }) {
+    let dataFile = `${LEGACY_CACHE_DIR}/${name}.json`;
+    let tsFile = `${LEGACY_CACHE_DIR}/${name}.updated.txt`;
 
     let tsDate;
     {
@@ -378,7 +326,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
       }
     }
     if (!projInfo) {
-      projInfo = await getLatestBuilds(Releases, installersDir, cacheDir, name);
+      return meta;
     }
     let latestProjInfo = await BuildsCacher.transformAndUpdate(
       name,
@@ -389,63 +337,8 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     );
     bc._caches[name] = latestProjInfo;
 
-    process.nextTick(async function () {
-      let now = date.valueOf();
-      let age = now - projInfo.updated;
-
-      let fresh = age < bc._staleAge;
-      if (fresh) {
-        return;
-      }
-
-      projInfo = await getLatestBuilds(Releases, installersDir, cacheDir, name);
-      let latestProjInfo = BuildsCacher.transformAndUpdate(
-        name,
-        projInfo,
-        meta,
-        date,
-        bc,
-      );
-      bc._caches[name] = latestProjInfo;
-    });
-
-    return projInfo;
-  };
-
-  // Makes sure that packages are updated once an hour, on average
-  bc._staleNames = [];
-  bc._freshenTimeout = null;
-  bc.freshenRandomPackage = async function (minDelay) {
-    if (!minDelay) {
-      minDelay = 15 * 1000;
-    }
-
-    if (bc._staleNames.length === 0) {
-      let dirs = await bc.getProjectsByType();
-      bc._staleNames = Object.keys(dirs.valid);
-      bc._staleNames.sort(function () {
-        return 0.5 - Math.random();
-      });
-    }
-
-    let name = bc._staleNames.pop();
-    void (await bc.getPackages({
-      //Releases: Releases,
-      name: name,
-      date: new Date(),
-    }));
-    console.info(`[INFO] freshenRandomPackage: ${name}`);
-
-    let hour = 60 * 60 * 1000;
-    let delay = minDelay;
-    let spread = hour / bc._staleNames.length;
-    let seed = Math.random();
-    delay += seed * spread;
-
-    clearTimeout(bc._freshenTimeout);
-    bc._freshenTimeout = setTimeout(bc.freshenRandomPackage, delay);
-    bc._freshenTimeout.unref();
-  };
+    return latestProjInfo;
+  }
 
   /**
    * Given a list of acceptable formats, get the sorted list of of formats.
@@ -537,7 +430,10 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
       exts.push('.gz');
       exts.push('.sh');
     }
-    exts.push('.git');
+    let hasGit = formats.includes('git') || formats.includes('.git');
+    if (hasGit) {
+      exts.push('.git');
+    }
 
     // Fallbacks
     // (we include everything to bubble an extract error over not found)
@@ -646,29 +542,19 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
       return null;
     }
 
-    for (let _triplet of triplets) {
-      let targetReleases = projInfo.releasesByTriplet[_triplet];
-      if (!targetReleases) {
-        continue;
-      }
+    // Version-first iteration, not triplet-first: take the newest
+    // version even when its only build lives in a fallback triplet
+    // (e.g. serviceman v1.0.1 only exists at posix_2017-ANYARCH-none).
+    for (let lexver of projInfo.lexvers) {
+      let ver = projInfo.lexversMap[lexver] || lexver;
 
-      let versions = Object.keys(targetReleases);
-      //console.log('dbg: targetRelease versions', versions);
-      let lexvers = [];
-      for (let version of versions) {
-        let lexPrefix = Lexver.parseVersion(version);
-        lexvers.push(lexPrefix);
-      }
-      lexvers.sort();
-      lexvers.reverse();
-      // TODO get the other matchInfo props
+      for (let _triplet of triplets) {
+        let targetReleases = projInfo.releasesByTriplet[_triplet];
+        if (!targetReleases) {
+          continue;
+        }
 
-      // Make sure that these releases are the expected version
-      // (ex: jq1.7 => darwin-arm64-libc, jq1.6 => darwin-x86_64-libc)
-      for (let matchver of lexvers) {
-        let ver = projInfo.lexversMap[matchver] || matchver;
         let packages = targetReleases[ver];
-        //console.log('dbg: packages', packages);
         if (!packages) {
           continue;
         }
@@ -719,22 +605,32 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
       return triplets;
     }
 
+    // Prefer platform-specific matches over ANYOS/ANYARCH fallbacks.
+    // This ensures e.g. darwin-aarch64-none matches before
+    // ANYOS-ANYARCH-none (.git source URLs from old releases).
     let oses = [];
     if (hostTarget.os === 'windows') {
-      oses = ['ANYOS', 'windows'];
+      oses = ['windows', 'ANYOS'];
     } else if (hostTarget.os === 'android') {
-      oses = ['ANYOS', 'posix_2017', 'posix_2024', 'android', 'linux'];
+      oses = ['android', 'linux', 'posix_2017', 'posix_2024', 'ANYOS'];
     } else {
-      oses = ['ANYOS', 'posix_2017', 'posix_2024', hostTarget.os];
+      oses = [hostTarget.os, 'posix_2017', 'posix_2024', 'ANYOS'];
     }
 
     let waterfall = HostTargets.WATERFALL[hostTarget.os] || {};
     let arches = waterfall[hostTarget.arch] ||
       HostTargets.WATERFALL.ANYOS[hostTarget.arch] || [hostTarget.arch];
-    arches = ['ANYARCH'].concat(arches);
+    arches = arches.concat(['ANYARCH']);
     let libcs = waterfall[hostTarget.libc] ||
       HostTargets.WATERFALL.ANYOS[hostTarget.libc] || [hostTarget.libc];
 
+    // Extend the glibc-host waterfall: the table only lists [none, libc]
+    // but Rust projects (bat, rg) and node ship libc='gnu' builds, and
+    // static musl builds also run on glibc hosts.
+    if (hostTarget.libc === 'libc' && !libcs.includes('gnu')) {
+      libcs = ['none', 'gnu', 'musl', 'libc'];
+    }
+
     for (let os of oses) {
       for (let arch of arches) {
         for (let libc of libcs) {
@@ -762,10 +658,17 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
 };
 
 BuildsCacher._classify = function (bc, projInfo, build) {
-  /* jshint maxcomplexity: 25 */
-  let maybeInstallable = Triplet.maybeInstallable(projInfo, build);
-  if (!maybeInstallable) {
-    return null;
+  /* jshint maxcomplexity: 30 */
+  // Cache entries arrive pre-classified (os/arch/libc/ext set). Skip
+  // maybeInstallable for those — it false-rejects names ending in a
+  // version tag (`serviceman-v1.0.1`, `v1.0.1.zip`).
+  let cacheClassified =
+    build.os && build.arch && build.libc && build.ext;
+  if (!cacheClassified) {
+    let maybeInstallable = Triplet.maybeInstallable(projInfo, build);
+    if (!maybeInstallable) {
+      return null;
+    }
   }
 
   if (LEGACY_OS_MAP[build.os]) {
@@ -829,16 +732,26 @@ BuildsCacher._classify = function (bc, projInfo, build) {
     bc.unknownTerms[term] = true;
   }
 
-  // {NAME}.windows.x86_64v2.musl.exe
-  //     windows-x86_64_v2-musl
+  // Skip termsToTarget for cache-classified entries: it false-flags
+  // e.g. .git URLs as os=ANYOS while the cache says os=posix_2017,
+  // and the mismatch check throws.
   target = { triplet: '' };
-  try {
-    void Triplet.termsToTarget(target, projInfo, build, terms);
-  } catch (e) {
-    console.error(`PACKAGE FORMAT CHANGE for '${projInfo.name}':`);
-    console.error(e.message);
-    console.error(build);
-    return null;
+  if (cacheClassified) {
+    target.os = build.os;
+    target.arch = build.arch;
+    target.libc = build.libc;
+    target.vendor = build.vendor || 'unknown';
+    target.android = false;
+    target.unknownTerms = [];
+  } else {
+    try {
+      void Triplet.termsToTarget(target, projInfo, build, terms);
+    } catch (e) {
+      console.error(`PACKAGE FORMAT CHANGE for '${projInfo.name}':`);
+      console.error(e.message);
+      console.error(build);
+      return null;
+    }
   }
 
   target.triplet = `${target.arch}-${target.vendor}-${target.os}-${target.libc}`;

_webi/builds.js

@@ -15,11 +15,8 @@ let bc = BuildsCacher.create({
   caches: CACHE_DIR,
   installers: INSTALLERS_DIR,
 });
-bc.freshenRandomPackage(600 * 1000);
 
 Builds.init = async function () {
-  bc.freshenRandomPackage(600 * 1000);
-
   let dirs = await bc.getProjectsByType();
   let projNames = Object.keys(dirs.valid);
 
@@ -27,7 +24,6 @@ Builds.init = async function () {
   await Parallel.run(parallel, projNames, getAll);
   async function getAll(name) {
     void (await bc.getPackages({
-      //Releases: Releases,
       name: name,
       date: new Date(),
     }));

_webi/classify-one.js

@@ -1,11 +1,14 @@
 'use strict';
 
+let Fs = require('node:fs/promises');
+let Os = require('node:os');
 let Path = require('node:path');
 
-// let Builds = require('./builds.js');
 let BuildsCacher = require('./builds-cacher.js');
 let Triplet = require('./build-classifier/triplet.js');
 
+let LEGACY_CACHE_DIR = Path.join(Os.homedir(), '.cache/webi/legacy');
+
 async function main() {
   let projName = process.argv[2];
   if (!projName) {
@@ -36,16 +39,11 @@ async function main() {
     arches: [],
     libcs: [],
     formats: [],
-    // TODO channels: [],
   };
 
-  let installersDir = Path.join(__dirname, '..');
-  let Releases = require(`${installersDir}/${projName}/releases.js`);
-  if (!Releases.latest) {
-    Releases.latest = Releases;
-  }
-
-  let projInfo = await Releases.latest();
+  let dataFile = Path.join(LEGACY_CACHE_DIR, `${projName}.json`);
+  let json = await Fs.readFile(dataFile, 'utf8');
+  let projInfo = JSON.parse(json);
 
   // let packages = await Builds.getPackage({ name: projName });
   // console.log(packages);
@@ -70,9 +68,11 @@ async function main() {
   console.log(`[DEBUG] transformed`);
   let sample = transformed.packages.slice(0, 20);
   console.log('packages:', sample, ':packages');
+  let firstTriplet = Object.keys(transformed.releasesByTriplet)[0];
+  let firstVersion = transformed.versions[0];
   console.log(
-    'releasesByTriplet:',
-    transformed.releasesByTriplet['linux-x86_64-none'][transformed.versions[0]],
+    `releasesByTriplet[${firstTriplet}][${firstVersion}]:`,
+    transformed.releasesByTriplet[firstTriplet]?.[firstVersion],
     ':releasesByTriplet',
   );
   console.log('versions:', transformed.versions, ':versions');

_webi/lint-builds.js

@@ -139,8 +139,6 @@ async function main() {
     console.info('');
   }
 
-  bc.freshenRandomPackage(600 * 1000);
-
   let rows = [];
   let triples = [];
   let valids = Object.keys(dirs.valid);

_webi/normalize.js

@@ -1,259 +0,0 @@
-'use strict';
-
-// this may need customizations between packages
-var osMap = {
-  macos: /(\b|_)(apple|os(\s_-)?x\b|mac|darwin|iPhone|iOS|iPad)/i,
-  linux: /(\b|_)(linux)/i,
-  freebsd: /(\b|_)(freebsd)/i,
-  windows: /(\b|_)(win|microsoft|msft)/i,
-  sunos: /(\b|_)(sun)/i,
-  aix: /(\b|_)(aix)/i,
-};
-
-var maps = {
-  oses: {},
-  arches: {},
-  libcs: {},
-  formats: {},
-};
-
-Object.keys(osMap).forEach(function (name) {
-  maps.oses[name] = true;
-});
-
-var formats = ['zip', 'xz', 'tar', 'pkg', 'msi', 'git', 'exe', 'dmg', 'git'];
-formats.forEach(function (name) {
-  maps.formats[name] = true;
-});
-
-// evaluation order matters
-// (i.e. otherwise x86 and x64 can cross match)
-var arches = [
-  // arm64/aarch64 has very high specificity, so it comes first
-  'arm64',
-  // arm 7 is also generic aarch/arm/arm32
-  'armv7l',
-  // arm6 can run on armv7
-  'armv6l',
-  // amd64 is more likely and less often specified than arm64
-  'amd64',
-  'x86',
-  'ppc64le',
-  'ppc64',
-  's390x',
-];
-// Used for detecting system arch from package download url, for example:
-//
-// https://git.com/org/foo/releases/v0.7.9/foo-aarch64-linux-musl.tar.gz
-// https://git.com/org/foo/releases/v0.7.9/foo-arm-linux-musleabihf.tar.gz
-// https://git.com/org/foo/releases/v0.7.9/foo-armv7-linux-musleabihf.tar.gz
-// https://git.com/org/foo/releases/v0.7.9/foo-x86_64-linux-musl.tar.gz
-//
-var archMap = {
-  arm64: /(\b|_)(aarch64|arm64)/i,
-  armv7l: /(\b|_)(arm32|arm[_\-]?v?7l?)/i,
-  armv6l: /(\b|_)(arm|aarch32|arm[_\-]?v?6l?)(\b|_)/i,
-  //amd64: /(amd.?64|x64|[_\-]64)/i,
-  amd64:
-    /(\b|_|amd|(dar)?win(dows)?|mac(os)?|linux|osx|x)64([_\-]?bit)?(\b|_)/i,
-  //x86: /(86)(\b|_)/i,
-  x86: /(\b|_|amd|(dar)?win(dows)?|mac(os)?|linux|osx|x)(86|32|i?386)([_\-]?bit)?(\b|_)/i,
-  ppc64le: /(\b|_)(ppc64le)/i,
-  ppc64: /(\b|_)(ppc64)(\b|_)/i,
-  s390x: /(\b|_)(s390x)/i,
-};
-arches.forEach(function (name) {
-  maps.arches[name] = true;
-});
-
-var libcs = ['none', 'musl', 'gnu', 'msvc', 'libc'];
-libcs.forEach(function (name) {
-  maps.libcs[name] = true;
-});
-
-function normalize(all) {
-  /* jshint maxcomplexity:50 */
-  /* jshint maxdepth:10 */
-  var supported = {
-    oses: {},
-    arches: {},
-    libcs: {},
-    formats: {},
-  };
-
-  for (let rel of all.releases) {
-    rel.version = rel.version.replace(/^v/i, '');
-
-    if (!rel.name) {
-      rel.name = rel.download.replace(/.*\//, '');
-    }
-
-    if (!rel.os) {
-      rel.os = 'unknown';
-
-      let osNames = Object.keys(osMap);
-      for (let osName of osNames) {
-        let relName = rel.name || rel.download;
-        let osRegExp = osMap[osName];
-        let matches = osRegExp.test(relName);
-        if (matches) {
-          rel.os = osName;
-          break;
-        }
-      }
-    }
-    supported.oses[rel.os] = true;
-
-    if (!rel.arch) {
-      for (let arch of arches) {
-        let name = rel.name || rel.download;
-        let isArch = name.match(archMap[arch]);
-        if (isArch) {
-          rel.arch = arch;
-          break;
-        }
-      }
-    }
-    if (!rel.arch) {
-      if ('macos' === rel.os) {
-        rel.arch = 'amd64';
-      }
-    }
-    supported.arches[rel.arch] = true;
-
-    // note: depends on rel.os
-    if (!rel.libc) {
-      let isMusl;
-      let isMsvc;
-      let isStatic;
-      let isGnu;
-
-      // extra blocks to prevent copy pasta errors
-
-      {
-        let muslRe = /(\b|_)(musl)(\b|_)/i;
-        isMusl = muslRe.test(rel.download) || muslRe.test(rel.name);
-      }
-
-      {
-        let msvcRe = /(\b|_)(msvc)(\b|_)/i;
-        isMsvc = msvcRe.test(rel.download) || msvcRe.test(rel.name);
-      }
-
-      {
-        let staticRe = /(\b|_)(static)(\b|_)/i;
-        isStatic = staticRe.test(rel.download) || staticRe.test(rel.name);
-      }
-
-      {
-        let gnuRe = /(\b|_)(gnu|glibc|libc)(\b|_)/i;
-        isGnu = gnuRe.test(rel.download) || gnuRe.test(rel.name);
-      }
-
-      if (isMusl) {
-        // we specifically tag things that need musl++ in their own releases
-        rel.libc = 'none';
-      } else if (isStatic) {
-        rel.libc = 'none';
-      } else if (isGnu) {
-        rel.libc = 'gnu';
-        if (rel.os === 'windows') {
-          // windows gnu is static
-          rel.libc = 'none';
-        } else if (rel.os === 'darwin') {
-          // if glibc is required on macos, it'll be static
-          rel.libc = 'none';
-        }
-      } else if (isMsvc) {
-        rel.libc = 'msvc';
-      } else {
-        // The default is no requirement for any particular libc
-        // (Go, Zig, POSIX Shell, JS, etc)
-        // and hopefully we never have to worry about mingw and friends
-        rel.libc = 'none';
-      }
-    }
-    supported.libcs[rel.libc] = true;
-
-    var tarExt;
-    if (!rel.ext) {
-      // pkg-v1.0.tar.gz => ['gz', 'tar', '0', 'pkg-v1']
-      // pkg-v1.0.tar => ['tar', '0' ,'pkg-v1']
-      // pkg-v1.0.zip => ['zip', '0', 'pkg-v1']
-      var exts = (rel.name || rel.download).split('.');
-      if (1 === exts.length) {
-        // for bare releases in the format of foo-linux-amd64
-        rel.ext = 'exe';
-      }
-      exts = exts.reverse().slice(0, 2);
-      if ('tar' === exts[1]) {
-        rel.ext = exts.reverse().join('.');
-        tarExt = 'tar';
-      } else if ('tgz' === exts[0]) {
-        rel.ext = 'tar.gz';
-        tarExt = 'tar';
-      } else {
-        rel.ext = exts[0];
-      }
-      if (/\-|linux|mac|os[_\-]?x|arm|amd|86|64|mip/i.test(rel.ext)) {
-        // for bare releases in the format of foo.linux-amd64
-        rel.ext = 'exe';
-      }
-    }
-    supported.formats[tarExt || rel.ext] = true;
-
-    if (!rel.channel) {
-      // basically like this: (+.-_)(beta|rc)(0-9)(+.-_)
-      // matches:
-      //   - v1.0-beta
-      //   - v1.0-beta1.1
-      //   - v1.0-beta-11
-      // won't match:
-      //   - v1.0beta
-      //   - v1.0-beta1b
-      let isBetaRe =
-        /(\b|_)(alpha|beta|dev|developer|prev|preview|rc)(\d+)(\b|_)/;
-      let isBeta = isBetaRe.test(rel.name);
-      if (isBeta) {
-        rel.channel = 'beta';
-      } else {
-        rel.channel = 'stable';
-      }
-    }
-
-    if (all.download) {
-      rel.download = all.download.replace(/{{ download }}/, rel.download);
-    }
-  }
-
-  all.oses = Object.keys(supported.oses).filter(function (name) {
-    return maps.oses[name];
-  });
-  all.arches = Object.keys(supported.arches).filter(function (name) {
-    return maps.arches[name];
-  });
-  all.libcs = Object.keys(supported.libcs).filter(function (name) {
-    return maps.libcs[name];
-  });
-  all.formats = Object.keys(supported.formats).filter(function (name) {
-    return maps.formats[name];
-  });
-
-  return all;
-}
-
-module.exports = normalize;
-module.exports._debug = function (all) {
-  all = normalize(all);
-  all.releases = all.releases
-    .filter(function (r) {
-      return ['windows', 'macos', 'linux'].includes(r.os) && 'amd64' === r.arch;
-    })
-    .slice(0, 10);
-  return all;
-};
-// NOT in order of priority (which would be tar, xz, zip, ...)
-module.exports.formats = formats;
-module.exports.arches = arches;
-module.exports.libcs = libcs;
-module.exports.formatsMap = maps.formats;

_webi/test.js

@@ -49,7 +49,7 @@ var baseurl = 'https://webinstall.dev';
 var maxLen = 0;
 console.info('');
 console.info('Has the necessary files?');
-['README.md', 'releases.js', 'install.sh', 'install.ps1']
+['README.md', 'install.sh', 'install.ps1']
   .map(function (node) {
     maxLen = Math.max(maxLen, node.length);
     return node;

_webi/transform-releases.js

@@ -2,75 +2,30 @@
 
 var Releases = module.exports;
 
+var Fs = require('node:fs/promises');
+var Os = require('node:os');
 var path = require('path');
-var _normalize = require('./normalize.js');
-
 var cache = {};
-//var staleAge = 5 * 1000;
-//var expiredAge = 15 * 1000;
-var staleAge = 5 * 60 * 1000;
-var expiredAge = 15 * 60 * 1000;
 
-let installerDir = path.join(__dirname, '..');
+var LEGACY_CACHE_DIR = path.join(Os.homedir(), '.cache/webi/legacy');
 
-Releases.get = async function (pkgdir) {
-  let get;
-  try {
-    get = require(`${pkgdir}/releases.js`);
-    // TODO update all releases files with module.exports.xxxx = 'foo';
-    if (!get.latest) {
-      get.latest = get;
-    }
-  } catch (e) {
-    let err = new Error('no releases.js for', pkgdir.split(/[\/\\]+/).pop());
-    err.code = 'E_NO_RELEASE';
-    throw err;
-  }
-
-  let all = await get.latest();
-
-  return _normalize(all);
-};
-
-// TODO needs a proper test, and more accurate (though perhaps far less simple) code
+// Sort releases by ext preference and libc within the same version.
+// The cache is already sorted by version (stable before beta, newest first),
+// so we only re-order within the same version string.
 function createFormatsSorter(formats) {
-  return function sortByVerExt(a, b) {
-    function lexver(semver) {
-      // v1.20.156 => 00001.00020.00156.zzzzz
-      // TODO BUG: v1.20.156-rc2 => 00001.00020.00156.rc2zz
-      var parts = semver.split(/[+\.\-]/g);
-      while (parts.length < 4) {
-        parts.push('');
-      }
-      return parts
-        .map(function (num, i) {
-          if (3 === i) {
-            return num.toString().padEnd(10, 'z');
-          }
-          return num.toString().padStart(10, '0');
-        })
-        .join('.');
-    }
-
-    var aver = lexver(a.version);
-    var bver = lexver(b.version);
-    if (aver > bver) {
-      //console.log(aver, '>', bver);
-      return -1;
-    }
-    if (aver < bver) {
-      //console.log(aver, '<', bver);
-      return 1;
+  return function sortByExtLibc(a, b) {
+    if (a.version !== b.version) {
+      // Array.sort is stable (V8, ES2019), so returning 0 across
+      // versions preserves the cache's pre-sorted version-desc order.
+      return 0;
     }
 
     var aExtPri = formats.indexOf(a.ext.replace(/tar\..*/, 'tar'));
     var bExtPri = formats.indexOf(b.ext.replace(/tar\..*/, 'tar'));
     if (aExtPri > bExtPri) {
-      //console.log(a.ext, aExtPri, '>', b.ext, bExtPri);
       return -1;
     }
     if (aExtPri < bExtPri) {
-      //console.log(a.ext, aExtPri, '<', b.ext, bExtPri);
       return 1;
     }
 
@@ -87,99 +42,39 @@ function createFormatsSorter(formats) {
 }
 
 async function getCachedReleases(pkg) {
-  // returns { download: '<template string>', releases: [{ version, date, os, arch, lts, channel, download}] }
+  // returns { download: '', releases: [{ version, date, os, arch, lts, channel, download}] }
 
-  async function chainCachePromise(fn) {
-    cache[pkg].promise = cache[pkg].promise.then(fn);
-    return cache[pkg].promise;
+  if (cache[pkg]) {
+    return cache[pkg];
   }
 
-  async function sleep(ms) {
-    return await new Promise(function (resolve, reject) {
-      setTimeout(resolve, ms);
-    });
-  }
+  let dataFile = `${LEGACY_CACHE_DIR}/${pkg}.json`;
 
-  async function putCache() {
-    var age = Date.now() - cache[pkg].updatedAt;
-    if (age < staleAge) {
-      //console.debug('NOT STALE ANYMORE - updated in previous promise');
-      return cache[pkg].all;
+  let json = await Fs.readFile(dataFile, 'utf8').catch(function (err) {
+    if (err.code === 'ENOENT') {
+      return null;
     }
+    throw err;
+  });
 
-    //console.debug('DOWNLOADING NEW "%s" releases', pkg);
-    var pkgdir = path.join(installerDir, pkg);
-
-    // workaround for request timeout seeming to not work
-    let complete = false;
-    await Promise.race([
-      Releases.get(pkgdir)
-        .catch(function (err) {
-          if ('E_NO_RELEASE' === err.code) {
-            let all = { _error: 'E_NO_RELEASE', download: '', releases: [] };
-            return all;
-          }
-
-          throw err;
-        })
-        .catch(function (err) {
-          let hasReleases = cache[pkg].all?.releases?.length > 1;
-          if (!hasReleases) {
-            throw err;
-          }
-
-          console.error(`Error: the BOOGEYMAN got us!`);
-          console.error(err.stack);
-
-          return cache[pkg].all;
-        })
-        .then(function (all) {
-          // Note: it is possible for slightly older data
-          // to replace slightly newer data, but this is better
-          // than being in a cycle where release updates _always_
-          // take longer than expected.
-          //console.debug('DOWNLOADED NEW "%s" releases', pkg);
-          cache[pkg].updatedAt = Date.now();
-          cache[pkg].all = all;
-          complete = true;
-        }),
-      sleep(15000).then(function () {
-        if (complete) {
-          return;
-        }
-        console.error(`request timeout waiting for '${pkg}' release info`);
-      }),
-    ]);
-
-    return cache[pkg].all;
+  if (!json) {
+    let empty = { download: '', releases: [] };
+    cache[pkg] = empty;
+    return empty;
   }
 
-  if (!cache[pkg]) {
-    cache[pkg] = {
-      updatedAt: 0,
-      all: { download: '', releases: [] },
-      promise: Promise.resolve(),
-    };
+  let all;
+  try {
+    all = JSON.parse(json);
+  } catch (e) {
+    console.error(`error: ${dataFile}:\n\t${e.message}`);
+    let empty = { download: '', releases: [] };
+    cache[pkg] = empty;
+    return empty;
   }
 
-  var bgRenewal;
-  var age = Date.now() - cache[pkg].updatedAt;
-  var fresh = age < staleAge;
-  if (!fresh) {
-    bgRenewal = chainCachePromise(putCache);
-  }
-
-  var tooStale = age > expiredAge;
-  if (!tooStale) {
-    return await cache[pkg].all;
-  }
-
-  return await Promise.race([
-    bgRenewal,
-    sleep(5000).then(function () {
-      return cache[pkg].all;
-    }),
-  ]);
+  cache[pkg] = all;
+  return all;
 }
 
 async function filterReleases(
@@ -190,15 +85,22 @@ async function filterReleases(
   // sort the most compatible format first
   // (i.e. so that we don't do .pkg on linux except on purpose)
   var rformats = formats.slice(0).reverse();
-  var sortByVerExt = createFormatsSorter(rformats);
+  var sortByExtLibc = createFormatsSorter(rformats);
   var reVer = new RegExp('^' + ver + '\\b');
 
   function selectMatches(rel) {
+    /* jshint maxcomplexity: 25 */
     if (os) {
-      if (rel.os !== '*') {
-        if (rel.os !== os) {
-          return false;
-        }
+      // '*' = any OS (matches anything, including windows).
+      // 'posix' / 'posix_20xx' = any POSIX OS (matches linux, macos,
+      // freebsd, etc., but NOT windows).
+      let isPosix = rel.os === 'posix' || rel.os.startsWith('posix_20');
+      let osMatches =
+        rel.os === '*' ||
+        rel.os === os ||
+        (isPosix && os !== 'windows');
+      if (!osMatches) {
+        return false;
       }
     }
 
@@ -210,7 +112,10 @@ async function filterReleases(
       }
     }
 
-    if (rel.libc !== 'none') {
+    // libc='libc' is serve-releases.js's default when the caller
+    // didn't pin one — treat it as 'no preference', not a filter.
+    let isMeaningfulLibc = libc && libc !== 'libc' && rel.libc !== 'none';
+    if (isMeaningfulLibc) {
       let releaseRequiresMusl = rel.libc === 'musl';
       // goal: handle non-glibc (Alpine / Docker / musl)
       let osHasMusl = libc === 'musl';
@@ -257,7 +162,7 @@ async function filterReleases(
     return true;
   }
 
-  var sortedRels = all.releases.filter(selectMatches).sort(sortByVerExt);
+  var sortedRels = all.releases.filter(selectMatches).sort(sortByExtLibc);
   //console.log(sortedRels.slice(0, 4));
 
   return sortedRels.slice(0, limit || 1000);
@@ -416,8 +321,8 @@ Releases.getReleases = function ({
 };
 
 if (require.main === module) {
-  return module
-    .exports({
+  return Releases
+    .getReleases({
       pkg: 'node',
       ver: '',
       os: 'macos',

aliasman/releases.conf

@@ -0,0 +1,2 @@
+github_sources = BeyondCodeBootcamp/aliasman
+git_url = https://github.com/BeyondCodeBootcamp/aliasman.git

aliasman/releases.js

@@ -1,22 +0,0 @@
-'use strict';
-
-let Releases = module.exports;
-
-let GitHubSource = require('../_common/github-source.js');
-let owner = 'BeyondCodeBootcamp';
-let repo = 'aliasman';
-
-Releases.latest = async function () {
-  let all = await GitHubSource.getDistributables({ owner, repo });
-  for (let pkg of all.releases) {
-    pkg.os = 'posix_2017';
-  }
-  return all;
-};
-
-if (module === require.main) {
-  Releases.latest().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

arc/releases.conf

@@ -0,0 +1 @@
+github_releases = mholt/archiver

arc/releases.js

@@ -1,21 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'mholt';
-var repo = 'archiver';
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    all._names = ['archiver', 'arc'];
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    // just select the first 5 for demonstration
-    all.releases = all.releases.slice(0, 5);
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

atomicparsley/releases.conf

@@ -0,0 +1 @@
+github_releases = wez/atomicparsley

atomicparsley/releases.js

@@ -1,79 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'wez';
-var repo = 'atomicparsley';
-
-let targets = {
-  x86win: {
-    os: 'windows',
-    arch: 'x86',
-    libc: 'msvc',
-  },
-  x64win: {
-    os: 'windows',
-    arch: 'amd64',
-    // https://github.com/wez/atomicparsley/issues/6#issuecomment-1364523028
-    libc: 'msvc',
-  },
-  x64mac: {
-    os: 'macos',
-    arch: 'amd64',
-  },
-  x64lin: {
-    os: 'linux',
-    arch: 'amd64',
-    libc: 'gnu',
-  },
-  x64musl: {
-    os: 'linux',
-    arch: 'amd64',
-    libc: 'musl',
-  },
-};
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    for (let rel of all.releases) {
-      let windows32 = rel.name.includes('WindowsX86.');
-      if (windows32) {
-        Object.assign(rel, targets.x86win);
-        continue;
-      }
-
-      let windows64 = rel.name.includes('Windows.');
-      if (windows64) {
-        Object.assign(rel, targets.x64win);
-        continue;
-      }
-
-      let macos64 = rel.name.includes('MacOS');
-      if (macos64) {
-        Object.assign(rel, targets.x64mac);
-        continue;
-      }
-
-      let musl64 = rel.name.includes('Alpine');
-      if (musl64) {
-        Object.assign(rel, targets.x64musl);
-        continue;
-      }
-
-      let lin64 = rel.name.includes('Linux.');
-      if (lin64) {
-        Object.assign(rel, targets.x64lin);
-        continue;
-      }
-    }
-    all._names = ['AtomicParsley', 'atomicparsley'];
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    console.info(JSON.stringify(all));
-    //console.info(JSON.stringify(all, null, 2));
-  });
-}

awless/releases.conf

@@ -0,0 +1 @@
+github_releases = wallix/awless

awless/releases.js

@@ -1,22 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'wallix';
-var repo = 'awless';
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    // remove checksums and .deb
-    all.releases = all.releases.filter(function (rel) {
-      return !/(\.txt)|(\.deb)$/i.test(rel.name);
-    });
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    console.info(JSON.stringify(all));
-  });
-}

bat/releases.conf

@@ -0,0 +1 @@
+github_releases = sharkdp/bat

bat/releases.js

@@ -1,20 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'sharkdp';
-var repo = 'bat';
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    all.releases = all.releases.slice(0, 10);
-    //console.info(JSON.stringify(all));
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

bun/releases.conf

@@ -0,0 +1,5 @@
+github_releases = oven-sh/bun
+tag_prefix = bun-
+default_x86_64 = x86_64_v3
+x86_64_v2 = baseline
+variants = profile

bun/releases.js

@@ -1,61 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'oven-sh';
-var repo = 'bun';
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    // collect baseline asset names so we can prefer them over non-baseline
-    // (baseline builds avoid SIGILL on older/container CPUs)
-    let baselineNames = new Set();
-    all.releases.forEach(function (r) {
-      if (r.name.includes('-baseline')) {
-        baselineNames.add(r.name.replace('-baseline', ''));
-      }
-    });
-
-    all.releases = all.releases
-      .filter(function (r) {
-        if (r.name.includes('-profile')) {
-          return false;
-        }
-
-        if (r.name.endsWith('.txt') || r.name.endsWith('.asc')) {
-          return false;
-        }
-
-        // drop the non-baseline asset when a baseline twin exists
-        if (!r.name.includes('-baseline') && baselineNames.has(r.name)) {
-          return false;
-        }
-
-        let isMusl = r.name.includes('-musl');
-        if (isMusl) {
-          r._musl = true;
-          r.libc = 'musl';
-        } else if (r.name.includes('-linux-')) {
-          r.libc = 'gnu';
-        }
-
-        return true;
-      })
-      .map(function (r) {
-        // bun-linux-x64-baseline.zip => bun-linux-x64
-        r.name = r.name.replace('-baseline', '').replace(/\.zip$/, '');
-        // bun-v0.5.1 => v0.5.1
-        r.version = r.version.replace(/bun-/g, '');
-        return r;
-      });
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    // just select the first 5 for demonstration
-    all.releases = all.releases.slice(0, 5);
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

caddy/releases.conf

@@ -0,0 +1 @@
+github_releases = caddyserver/caddy

caddy/releases.js

@@ -1,27 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'caddyserver';
-var repo = 'caddy';
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    // remove checksums and .deb
-    all.releases = all.releases.filter(function (rel) {
-      let isOneOffAsset = rel.download.includes('buildable-artifact');
-      if (isOneOffAsset) {
-        return false;
-      }
-
-      return true;
-    });
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    console.info(JSON.stringify(all));
-  });
-}

chromedriver/releases.conf

@@ -0,0 +1 @@
+source = chromedist

chromedriver/releases.js

@@ -1,101 +0,0 @@
-'use strict';
-
-let Fetcher = require('../_common/fetcher.js');
-
-// See <https://googlechromelabs.github.io/chrome-for-testing/>
-const releaseApiUrl =
-  'https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json';
-
-// {
-//   "timestamp": "2023-11-15T21:08:56.730Z",
-//   "versions": [
-//     {
-//       "version": "121.0.6120.0",
-//       "revision": "1222902",
-//       "downloads": {
-//         "chrome": [],
-//         "chromedriver": [
-//           {
-//             "platform": "linux64",
-//             "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/121.0.6120.0/linux64/chromedriver-linux64.zip"
-//           },
-//           {
-//             "platform": "mac-arm64",
-//             "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/121.0.6120.0/mac-arm64/chromedriver-mac-arm64.zip"
-//           },
-//           {
-//             "platform": "mac-x64",
-//             "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/121.0.6120.0/mac-x64/chromedriver-mac-x64.zip"
-//           },
-//           {
-//             "platform": "win32",
-//             "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/121.0.6120.0/win32/chromedriver-win32.zip"
-//           },
-//           {
-//             "platform": "win64",
-//             "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/121.0.6120.0/win64/chromedriver-win64.zip"
-//           }
-//         ],
-//         "chrome-headless-shell": []
-//       }
-//     }
-//   ]
-// }
-
-module.exports = async function () {
-  let resp;
-  try {
-    resp = await Fetcher.fetch(releaseApiUrl, {
-      headers: { Accept: 'application/json' },
-    });
-  } catch (e) {
-    /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error
-    let err = e;
-    if (err.code === 'E_FETCH_RELEASES') {
-      err.message = `failed to fetch 'chromedriver' release data: ${err.response.status} ${err.response.body}`;
-    }
-    throw e;
-  }
-  let data = JSON.parse(resp.body);
-
-  let builds = [];
-  for (let release of data.versions) {
-    if (!release.downloads.chromedriver) {
-      continue;
-    }
-
-    let version = release.version;
-    for (let asset of release.downloads.chromedriver) {
-      let build = {
-        version: version,
-        download: asset.url,
-        // I' not sure that this is actually statically built but it
-        // seems to be and at worst we'll just get bug reports for Alpine
-        libc: 'none',
-      };
-
-      builds.push(build);
-    }
-  }
-
-  let all = {
-    download: '',
-    releases: builds,
-  };
-
-  return all;
-};
-
-if (module === require.main) {
-  module
-    .exports()
-    .then(function (all) {
-      all = require('../_webi/normalize.js')(all);
-      // just select the latest 20 for demonstration
-      all.releases = all.releases.slice(-20);
-      console.info(JSON.stringify(all, null, 2));
-    })
-    .catch(function (err) {
-      console.error('Error:', err);
-    });
-}

cilium/releases.conf

@@ -0,0 +1 @@
+github_releases = cilium/cilium-cli

cilium/releases.js

@@ -1,21 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'cilium';
-var repo = 'cilium-cli';
-
-module.exports = async function () {
-  let all = await github(null, owner, repo);
-  return all;
-};
-
-if (module === require.main) {
-  (async function () {
-    let normalize = require('../_webi/normalize.js');
-    let all = await module.exports();
-    all = normalize(all);
-    // just select the first 5 for demonstration
-    all.releases = all.releases.slice(0, 5);
-    console.info(JSON.stringify(all, null, 2));
-  })();
-}

cmake/releases.conf

@@ -0,0 +1 @@
+github_releases = Kitware/CMake

cmake/releases.js

@@ -1,53 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'Kitware';
-var repo = 'CMake';
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    for (let rel of all.releases) {
-      if (rel.version.startsWith('v')) {
-        rel._version = rel.version.slice(1);
-      }
-
-      {
-        let linuxRe = /(\b|_)(linux|gnu)(\b|_)/i;
-        let isLinux = linuxRe.test(rel.download) || linuxRe.test(rel.name);
-
-        if (isLinux) {
-          let muslRe = /(\b|_)(musl|alpine)(\b|_)/i;
-          let isMusl = muslRe.test(rel.download) || muslRe.test(rel.name);
-          if (isMusl) {
-            rel.libc = 'musl';
-          } else {
-            rel.libc = 'gnu';
-          }
-          continue;
-        }
-      }
-
-      {
-        let windowsRe = /(\b|_)(win\d*|windows\d*)(\b|_)/i;
-        let isWindows =
-          windowsRe.test(rel.download) || windowsRe.test(rel.name);
-
-        if (isWindows) {
-          rel.libc = 'msvc';
-          continue;
-        }
-      }
-    }
-
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    // just select the first 5 for demonstration
-    all.releases = all.releases.slice(0, 5);
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

cmd/comparecache/main.go

@@ -0,0 +1,914 @@
+// Command comparecache compares Go-generated cache output against the
+// Node.js LIVE_cache. It identifies categorical differences in asset
+// selection — which filenames appear in one cache but not the other.
+//
+// The comparison is done at the filename level (not OS/arch/ext fields)
+// because the Node.js cache leaves those empty (normalize.js fills them
+// at serve time), while the Go pipeline classifies at write time.
+//
+// Usage:
+//
+//	go run ./cmd/comparecache -live ./LIVE_cache -go ./_cache
+//	go run ./cmd/comparecache -live ./LIVE_cache -go ./_cache bat jq
+//	go run ./cmd/comparecache -live ./LIVE_cache -go ./_cache -summary
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"math/rand/v2"
+	"os"
+	"path/filepath"
+	"slices"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/webinstall/webi-installers/internal/classify"
+	"github.com/webinstall/webi-installers/internal/lexver"
+)
+
+type cacheEntry struct {
+	Releases []cacheRelease `json:"releases"`
+}
+
+type cacheRelease struct {
+	Name     string `json:"name"`
+	Filename string `json:"_filename"` // Node.js uses _filename for some sources
+	Version  string `json:"version"`
+	Download string `json:"download"`
+	Channel  string `json:"channel"`
+	OS       string `json:"os"`
+	Arch     string `json:"arch"`
+	Libc     string `json:"libc"`
+	Ext      string `json:"ext"`
+}
+
+// fieldDiff records a field-level difference for an asset that exists
+// in both caches (same filename) but has different classification.
+type fieldDiff struct {
+	Filename string
+	Field    string // "os", "arch", "libc", "ext", "channel"
+	Live     string
+	Go       string
+	BothSet  bool // true when both live and go have non-empty values
+}
+
+type packageDiff struct {
+	Name         string
+	LiveCount    int
+	GoCount      int
+	OnlyInLive   []string // filenames only in Node.js cache
+	OnlyInGo     []string // filenames only in Go cache
+	FieldDiffs   []fieldDiff // classification differences on shared assets
+	VersionsLive []string    // unique versions in live
+	VersionsGo   []string    // unique versions in go
+	GoMissing    bool        // true if Go didn't produce output for this package
+	LiveMissing  bool        // true if no live cache for this package
+	Categories   []string    // categorical difference labels
+}
+
+func main() {
+	liveDir := flag.String("live", "./LIVE_cache", "path to Node.js LIVE_cache directory")
+	goDir := flag.String("go", "./_cache", "path to Go cache directory")
+	summary := flag.Bool("summary", false, "only print summary, not per-package details")
+	diffsOnly := flag.Bool("diffs", false, "only show packages with asset differences (skip matches)")
+	latest := flag.Bool("latest", false, "only compare latest version in each cache")
+	windowed := flag.Bool("windowed", false, "limit Go versions to the Node.js version range (2nd to 2nd-to-last)")
+	sample := flag.Int("sample", 0, "for each package diff, show N randomly sampled assets (implies -windowed -diffs)")
+	flag.Parse()
+	filterPkgs := flag.Args()
+
+	// -sample implies -windowed and -diffs so we focus on real classification
+	// differences, not version-depth noise.
+	if *sample > 0 {
+		*windowed = true
+		*diffsOnly = true
+	}
+
+	totalStart := time.Now()
+
+	// Find the most recent month directory in each cache.
+	liveMonth := findLatestMonth(*liveDir)
+	goMonth := findLatestMonth(*goDir)
+	if liveMonth == "" {
+		log.Fatalf("no month directories found in %s", *liveDir)
+	}
+
+	livePath := filepath.Join(*liveDir, liveMonth)
+	goPath := ""
+	if goMonth != "" {
+		goPath = filepath.Join(*goDir, goMonth)
+	}
+
+	// Discover all packages across both caches.
+	discoverStart := time.Now()
+	allPkgs := discoverPackages(livePath, goPath)
+	if len(filterPkgs) > 0 {
+		nameSet := make(map[string]bool, len(filterPkgs))
+		for _, n := range filterPkgs {
+			nameSet[n] = true
+		}
+		var filtered []string
+		for _, p := range allPkgs {
+			if nameSet[p] {
+				filtered = append(filtered, p)
+			}
+		}
+		allPkgs = filtered
+	}
+	log.Printf("discovered %d packages in %s", len(allPkgs), time.Since(discoverStart))
+
+	compareStart := time.Now()
+	var diffs []packageDiff
+	for _, pkg := range allPkgs {
+		d := compare(livePath, goPath, pkg, *latest, *windowed)
+		categorize(&d)
+		diffs = append(diffs, d)
+	}
+	log.Printf("compared %d packages in %s", len(diffs), time.Since(compareStart))
+
+	if *summary {
+		printSummary(diffs)
+	} else {
+		printDetails(diffs, *diffsOnly, *sample)
+	}
+
+	log.Printf("total: %s", time.Since(totalStart))
+}
+
+func findLatestMonth(dir string) string {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return ""
+	}
+	var months []string
+	for _, e := range entries {
+		if e.IsDir() && len(e.Name()) == 7 && e.Name()[4] == '-' {
+			months = append(months, e.Name())
+		}
+	}
+	if len(months) == 0 {
+		return ""
+	}
+	sort.Strings(months)
+	return months[len(months)-1]
+}
+
+func discoverPackages(livePath, goPath string) []string {
+	seen := make(map[string]bool)
+	for _, dir := range []string{livePath, goPath} {
+		if dir == "" {
+			continue
+		}
+		entries, err := os.ReadDir(dir)
+		if err != nil {
+			continue
+		}
+		for _, e := range entries {
+			name := e.Name()
+			if strings.HasSuffix(name, ".json") && !strings.HasSuffix(name, ".updated.txt") {
+				pkg := strings.TrimSuffix(name, ".json")
+				seen[pkg] = true
+			}
+		}
+	}
+	var pkgs []string
+	for p := range seen {
+		pkgs = append(pkgs, p)
+	}
+	sort.Strings(pkgs)
+	return pkgs
+}
+
+func loadCache(dir, pkg string) *cacheEntry {
+	if dir == "" {
+		return nil
+	}
+	data, err := os.ReadFile(filepath.Join(dir, pkg+".json"))
+	if err != nil {
+		return nil
+	}
+	var entry cacheEntry
+	if err := json.Unmarshal(data, &entry); err != nil {
+		return nil
+	}
+	return &entry
+}
+
+// effectiveName returns the best available filename for a release entry.
+// Node.js sometimes uses _filename (a path) instead of name.
+func effectiveName(name, filename, download string) string {
+	if name != "" {
+		return name
+	}
+	if filename != "" {
+		// _filename may be a path like "stable/macos/flutter_macos_3.41.4.zip"
+		if i := strings.LastIndex(filename, "/"); i >= 0 {
+			return filename[i+1:]
+		}
+		return filename
+	}
+	// Last resort: basename of download URL.
+	if download != "" {
+		if i := strings.LastIndex(download, "/"); i >= 0 {
+			return download[i+1:]
+		}
+	}
+	return ""
+}
+
+// versionWindow returns the 2nd and 2nd-to-last versions from a sorted
+// version list. This trims the edges where Node.js may have a newer fetch
+// or Go may have deeper history, focusing on the overlapping middle.
+func versionWindow(versions []string) (low, high string) {
+	if len(versions) <= 2 {
+		// Too few versions to window — use all.
+		if len(versions) > 0 {
+			return versions[0], versions[len(versions)-1]
+		}
+		return "", ""
+	}
+	// 2nd version (skip oldest) and 2nd-to-last (skip newest).
+	return versions[1], versions[len(versions)-2]
+}
+
+// filterVersionRange returns only the versions in sorted order that fall
+// within [low, high] inclusive (by lexver comparison).
+func filterVersionRange(vf map[string]map[string]bool, versions []string, low, high string) (map[string]bool, []string) {
+	lowV := lexver.Parse(low)
+	highV := lexver.Parse(high)
+
+	files := make(map[string]bool)
+	var kept []string
+	for _, v := range versions {
+		pv := lexver.Parse(v)
+		if lexver.Compare(pv, lowV) >= 0 && lexver.Compare(pv, highV) <= 0 {
+			kept = append(kept, v)
+			for f := range vf[v] {
+				files[f] = true
+			}
+		}
+	}
+	return files, kept
+}
+
+func compare(livePath, goPath, pkg string, latestOnly, windowed bool) packageDiff {
+	live := loadCache(livePath, pkg)
+	goCache := loadCache(goPath, pkg)
+
+	d := packageDiff{Name: pkg}
+
+	if live == nil {
+		d.LiveMissing = true
+	}
+	if goCache == nil {
+		d.GoMissing = true
+	}
+	if d.LiveMissing && d.GoMissing {
+		return d
+	}
+
+	normVersion := normalizeVersionFunc(pkg)
+
+	// Collect filenames by version. If filter is non-nil, skip filenames it rejects.
+	extractVersionFiles := func(ce *cacheEntry, filter func(string) bool) (map[string]map[string]bool, []string) {
+		vf := make(map[string]map[string]bool)
+		for _, r := range ce.Releases {
+			name := effectiveName(r.Name, r.Filename, r.Download)
+			if filter != nil && !filter(name) {
+				continue
+			}
+			ver := normVersion(r.Version)
+			if vf[ver] == nil {
+				vf[ver] = make(map[string]bool)
+			}
+			vf[ver][name] = true
+		}
+		var versions []string
+		for v := range vf {
+			versions = append(versions, v)
+		}
+		slices.SortFunc(versions, func(a, b string) int {
+			return lexver.Compare(lexver.Parse(a), lexver.Parse(b))
+		})
+		return vf, versions
+	}
+	notNoise := func(name string) bool { return !isLiveNoise(name) }
+
+	var liveFiles, goFiles map[string]bool
+
+	// Parse live cache.
+	var liveVF map[string]map[string]bool
+	var liveVersions []string
+	if live != nil {
+		liveVF, liveVersions = extractVersionFiles(live, notNoise)
+		d.VersionsLive = liveVersions
+		d.LiveCount = len(live.Releases)
+	}
+
+	// Parse Go cache.
+	var goVF map[string]map[string]bool
+	var goVersions []string
+	if goCache != nil {
+		goVF, goVersions = extractVersionFiles(goCache, notNoise)
+		d.VersionsGo = goVersions
+		d.GoCount = len(goCache.Releases)
+	}
+
+	// Determine which files to compare based on mode.
+	if latestOnly {
+		// Compare only the latest version from each cache.
+		if live != nil && len(liveVersions) > 0 {
+			liveFiles = liveVF[liveVersions[len(liveVersions)-1]]
+		}
+		if goCache != nil && len(goVersions) > 0 {
+			goFiles = goVF[goVersions[len(goVersions)-1]]
+		}
+	} else if windowed && live != nil && len(liveVersions) > 0 {
+		// Use the Node.js version range (2nd to 2nd-to-last) to establish
+		// the window. Include ALL Node.js versions in the window (so missing
+		// Go versions are visible), but exclude Go-only versions (those are
+		// just deeper history, not real gaps).
+		low, high := versionWindow(liveVersions)
+		lowV := lexver.Parse(low)
+		highV := lexver.Parse(high)
+
+		// Collect all live files in the window.
+		liveFiles = make(map[string]bool)
+		liveInWindow := make(map[string]bool)
+		for _, v := range liveVersions {
+			pv := lexver.Parse(v)
+			if lexver.Compare(pv, lowV) >= 0 && lexver.Compare(pv, highV) <= 0 {
+				liveInWindow[v] = true
+				for f := range liveVF[v] {
+					liveFiles[f] = true
+				}
+			}
+		}
+
+		// For Go, only include versions that Node.js also has in the window.
+		// Go-only versions are hidden (deeper history, not gaps).
+		goFiles = make(map[string]bool)
+		for _, v := range goVersions {
+			if !liveInWindow[v] {
+				continue
+			}
+			for f := range goVF[v] {
+				goFiles[f] = true
+			}
+		}
+	} else {
+		// Compare all versions — use pre-filtered version maps.
+		if live != nil {
+			liveFiles = make(map[string]bool)
+			for _, files := range liveVF {
+				for f := range files {
+					liveFiles[f] = true
+				}
+			}
+		}
+		if goCache != nil {
+			goFiles = make(map[string]bool)
+			for _, files := range goVF {
+				for f := range files {
+					goFiles[f] = true
+				}
+			}
+		}
+	}
+
+	if liveFiles == nil {
+		liveFiles = make(map[string]bool)
+	}
+	if goFiles == nil {
+		goFiles = make(map[string]bool)
+	}
+
+	for f := range liveFiles {
+		if !goFiles[f] {
+			d.OnlyInLive = append(d.OnlyInLive, f)
+		}
+	}
+	for f := range goFiles {
+		if !liveFiles[f] {
+			d.OnlyInGo = append(d.OnlyInGo, f)
+		}
+	}
+	sort.Strings(d.OnlyInLive)
+	sort.Strings(d.OnlyInGo)
+
+	// Field-level comparison on assets that exist in both caches.
+	// Build version+filename → fields maps from each cache.
+	if live != nil && goCache != nil {
+		type assetKey struct {
+			version  string
+			filename string
+		}
+		liveByKey := make(map[assetKey]cacheRelease)
+		for _, r := range live.Releases {
+			name := effectiveName(r.Name, r.Filename, r.Download)
+			ver := normVersion(r.Version)
+			liveByKey[assetKey{ver, name}] = r
+		}
+
+		for _, r := range goCache.Releases {
+			name := effectiveName(r.Name, r.Filename, r.Download)
+			ver := normVersion(r.Version)
+			lr, ok := liveByKey[assetKey{ver, name}]
+			if !ok {
+				continue
+			}
+
+			// Compare classification fields.
+			// Use equivalence checks for os/arch/ext so naming
+			// convention differences don't mask real classification bugs.
+			for _, cmp := range []struct {
+				field string
+				live  string
+				go_   string
+				equiv bool
+			}{
+				{"os", lr.OS, r.OS, equivOS(lr.OS, r.OS)},
+				{"arch", lr.Arch, r.Arch, equivArch(lr.Arch, r.Arch)},
+				{"libc", lr.Libc, r.Libc, lr.Libc == r.Libc},
+				{"ext", lr.Ext, r.Ext, equivExt(lr.Ext, r.Ext)},
+				{"channel", lr.Channel, r.Channel, lr.Channel == r.Channel},
+			} {
+				if cmp.equiv {
+					continue
+				}
+				d.FieldDiffs = append(d.FieldDiffs, fieldDiff{
+					Filename: name,
+					Field:    cmp.field,
+					Live:     cmp.live,
+					Go:       cmp.go_,
+					BothSet:  cmp.live != "" && cmp.go_ != "",
+				})
+			}
+		}
+		sort.Slice(d.FieldDiffs, func(i, j int) bool {
+			if d.FieldDiffs[i].Field != d.FieldDiffs[j].Field {
+				return d.FieldDiffs[i].Field < d.FieldDiffs[j].Field
+			}
+			return d.FieldDiffs[i].Filename < d.FieldDiffs[j].Filename
+		})
+	}
+
+	return d
+}
+
+// equivOS returns true if two OS values are equivalent across naming conventions.
+func equivOS(a, b string) bool {
+	return a == b || canonicalOS(a) == canonicalOS(b)
+}
+
+func canonicalOS(s string) string {
+	switch strings.ToLower(s) {
+	case "darwin", "macos", "mac", "osx":
+		return "darwin"
+	case "win", "windows":
+		return "windows"
+	default:
+		return strings.ToLower(s)
+	}
+}
+
+// equivArch returns true if two arch values are equivalent.
+func equivArch(a, b string) bool {
+	return a == b || canonicalArch(a) == canonicalArch(b)
+}
+
+func canonicalArch(s string) string {
+	switch strings.ToLower(s) {
+	case "x86_64", "amd64", "x64":
+		return "x86_64"
+	case "aarch64", "arm64":
+		return "aarch64"
+	case "armv7", "armv7l":
+		return "armv7"
+	case "armv6", "armv6l":
+		return "armv6"
+	case "x86", "i386", "i686", "386":
+		return "x86"
+	default:
+		return strings.ToLower(s)
+	}
+}
+
+// equivExt returns true if two extension values are equivalent.
+func equivExt(a, b string) bool {
+	// Normalize: strip leading dot, handle common aliases.
+	return a == b || canonicalExt(a) == canonicalExt(b)
+}
+
+func canonicalExt(s string) string {
+	s = strings.TrimPrefix(s, ".")
+	switch s {
+	case "tgz":
+		return "tar.gz"
+	default:
+		return s
+	}
+}
+
+func categorize(d *packageDiff) {
+	if d.GoMissing {
+		d.Categories = append(d.Categories, "go-missing")
+		return
+	}
+	if d.LiveMissing {
+		d.Categories = append(d.Categories, "live-missing")
+		return
+	}
+
+	if len(d.OnlyInLive) == 0 && len(d.OnlyInGo) == 0 && len(d.FieldDiffs) == 0 {
+		d.Categories = append(d.Categories, "match")
+		return
+	}
+	if len(d.OnlyInLive) == 0 && len(d.OnlyInGo) == 0 && len(d.FieldDiffs) > 0 {
+		d.Categories = append(d.Categories, "fields-only")
+	}
+
+	// Check if differences are only version depth (Go has more history).
+	liveVersionSet := make(map[string]bool, len(d.VersionsLive))
+	for _, v := range d.VersionsLive {
+		liveVersionSet[v] = true
+	}
+	goVersionSet := make(map[string]bool, len(d.VersionsGo))
+	for _, v := range d.VersionsGo {
+		goVersionSet[v] = true
+	}
+
+	goExtraVersions := 0
+	for _, v := range d.VersionsGo {
+		if !liveVersionSet[v] {
+			goExtraVersions++
+		}
+	}
+	liveExtraVersions := 0
+	for _, v := range d.VersionsLive {
+		if !goVersionSet[v] {
+			liveExtraVersions++
+		}
+	}
+
+	if goExtraVersions > 0 {
+		d.Categories = append(d.Categories, fmt.Sprintf("go-extra-versions(%d)", goExtraVersions))
+	}
+	if liveExtraVersions > 0 {
+		d.Categories = append(d.Categories, fmt.Sprintf("live-extra-versions(%d)", liveExtraVersions))
+	}
+
+	// Check for meta-asset filtering differences.
+	metaOnlyInLive := 0
+	nonMetaOnlyInLive := 0
+	for _, f := range d.OnlyInLive {
+		if classify.IsMetaAsset(f) {
+			metaOnlyInLive++
+		} else {
+			nonMetaOnlyInLive++
+		}
+	}
+	metaOnlyInGo := 0
+	nonMetaOnlyInGo := 0
+	for _, f := range d.OnlyInGo {
+		if classify.IsMetaAsset(f) {
+			metaOnlyInGo++
+		} else {
+			nonMetaOnlyInGo++
+		}
+	}
+
+	if metaOnlyInLive > 0 {
+		d.Categories = append(d.Categories, fmt.Sprintf("live-has-meta(%d)", metaOnlyInLive))
+	}
+	if metaOnlyInGo > 0 {
+		d.Categories = append(d.Categories, fmt.Sprintf("go-has-meta(%d)", metaOnlyInGo))
+	}
+
+	// Check for source tarball differences.
+	srcOnlyInGo := 0
+	for _, f := range d.OnlyInGo {
+		if strings.HasSuffix(f, ".tar.gz") || strings.HasSuffix(f, ".zip") {
+			if strings.HasPrefix(f, "v") || strings.HasPrefix(f, "refs/") {
+				srcOnlyInGo++
+			}
+		}
+	}
+	if srcOnlyInGo > 0 {
+		d.Categories = append(d.Categories, fmt.Sprintf("go-has-source-tarballs(%d)", srcOnlyInGo))
+	}
+
+	if nonMetaOnlyInLive > 0 {
+		d.Categories = append(d.Categories, fmt.Sprintf("live-extra-assets(%d)", nonMetaOnlyInLive))
+	}
+	if nonMetaOnlyInGo > 0 {
+		d.Categories = append(d.Categories, fmt.Sprintf("go-extra-assets(%d)", nonMetaOnlyInGo))
+	}
+
+	// Count field diffs by field name, separating real disagreements
+	// from expected "live empty, Go classified" differences.
+	type fieldCount struct {
+		bothSet  int // both caches have a value but they disagree
+		oneEmpty int // one side is empty (typically live — normalize.js fills at serve time)
+	}
+	fieldCounts := make(map[string]fieldCount)
+	for _, fd := range d.FieldDiffs {
+		fc := fieldCounts[fd.Field]
+		if fd.BothSet {
+			fc.bothSet++
+		} else {
+			fc.oneEmpty++
+		}
+		fieldCounts[fd.Field] = fc
+	}
+	for _, field := range []string{"os", "arch", "libc", "ext", "channel"} {
+		fc := fieldCounts[field]
+		if fc.bothSet > 0 {
+			d.Categories = append(d.Categories, fmt.Sprintf("diff-%s(%d)", field, fc.bothSet))
+		}
+		if fc.oneEmpty > 0 {
+			d.Categories = append(d.Categories, fmt.Sprintf("fill-%s(%d)", field, fc.oneEmpty))
+		}
+	}
+}
+
+// isLiveNoise returns true for filenames that the Node.js cache keeps
+// but Go intentionally filters out. Pre-filtering these from the live
+// side prevents them from appearing as live-extra-assets noise.
+//
+// This includes everything classify.IsMetaAsset catches plus formats
+// that Go's legacy export strips (.deb, .rpm, etc.).
+func isLiveNoise(name string) bool {
+	if classify.IsMetaAsset(name) {
+		return true
+	}
+
+	lower := strings.ToLower(name)
+
+	// Formats Go filters from legacy export but Node.js keeps.
+	for _, suffix := range []string{
+		".deb", ".rpm", ".gpg",
+	} {
+		if strings.HasSuffix(lower, suffix) {
+			return true
+		}
+	}
+
+	// Source tarballs (e.g. gitea-src-1.25.4.tar.gz, caddy_2.10.0_src.tar.gz, go1.26.1.src.tar.gz).
+	if strings.Contains(lower, "-src-") || strings.Contains(lower, "_src.") || strings.Contains(lower, ".src.") || strings.HasPrefix(lower, "src-") {
+		return true
+	}
+
+	// Docs tarballs (e.g. gitea-docs-1.22.3.tar.gz).
+	if strings.Contains(lower, "-docs-") {
+		return true
+	}
+
+	// Bare executables without any extension — typically legacy shell scripts
+	// uploaded alongside proper archives (e.g. kubectx, kubens).
+	if !strings.Contains(name, ".") {
+		return true
+	}
+
+	// GPU accelerator / hardware variants that Go tags as variants
+	// but Node.js keeps with special arch names.
+	for _, v := range []string{"-rocm", "-jetpack"} {
+		if strings.Contains(lower, v) {
+			return true
+		}
+	}
+
+	// Linux binaries for packages where Node.js only kept macOS .app.zip.
+	// Go correctly includes these as installable on Linux.
+	if strings.HasPrefix(lower, "fish-") && strings.Contains(lower, "-linux-") {
+		return true
+	}
+
+	return false
+}
+
+// normalizeVersionFunc returns a version normalizer for a given package.
+// Most packages return the identity function. Some (like git) need
+// version string normalization to match across Go and Node.js caches.
+func normalizeVersionFunc(pkg string) func(string) string {
+	switch pkg {
+	case "git":
+		return func(v string) string {
+			// Git for Windows: v2.53.0.windows.1 → v2.53.0
+			//                  v2.53.0.windows.2 → v2.53.0.2
+			idx := strings.Index(v, ".windows.")
+			if idx < 0 {
+				return v
+			}
+			suffix := v[idx+len(".windows."):]
+			base := v[:idx]
+			if suffix == "1" {
+				return base
+			}
+			return base + "." + suffix
+		}
+	case "lf":
+		return func(v string) string {
+			// lf: r21 → 0.21.0
+			if strings.HasPrefix(v, "r") {
+				return "0." + v[1:] + ".0"
+			}
+			return v
+		}
+	case "bun":
+		return func(v string) string {
+			// bun: bun-v1.3.9 → v1.3.9
+			return strings.TrimPrefix(v, "bun-")
+		}
+	case "watchexec":
+		return func(v string) string {
+			// watchexec monorepo: cli-v1.20.5 → v1.20.5
+			return strings.TrimPrefix(v, "cli-")
+		}
+	case "go":
+		return func(v string) string {
+			// Go: go1.10 → 1.10.0 (pad to 3 parts)
+			v = strings.TrimPrefix(v, "go")
+			parts := strings.SplitN(v, ".", 3)
+			for len(parts) < 3 {
+				parts = append(parts, "0")
+			}
+			return strings.Join(parts, ".")
+		}
+	default:
+		return func(v string) string { return v }
+	}
+}
+
+func printSummary(diffs []packageDiff) {
+	// Count by category.
+	categoryCounts := make(map[string]int)
+	for _, d := range diffs {
+		for _, c := range d.Categories {
+			// Strip the count suffix for grouping.
+			base := c
+			if idx := strings.Index(c, "("); idx != -1 {
+				base = c[:idx]
+			}
+			categoryCounts[base]++
+		}
+	}
+
+	fmt.Println("=== COMPARISON SUMMARY ===")
+	fmt.Printf("Total packages: %d\n\n", len(diffs))
+
+	var cats []string
+	for c := range categoryCounts {
+		cats = append(cats, c)
+	}
+	sort.Strings(cats)
+	for _, c := range cats {
+		fmt.Printf("  %-30s %d\n", c, categoryCounts[c])
+	}
+
+	fmt.Println("\n=== PER-PACKAGE CATEGORIES ===")
+	for _, d := range diffs {
+		fmt.Printf("%-25s %s\n", d.Name, strings.Join(d.Categories, ", "))
+	}
+}
+
+func printDetails(diffs []packageDiff, diffsOnly bool, sampleN int) {
+	for _, d := range diffs {
+		if diffsOnly && len(d.OnlyInLive) == 0 && len(d.OnlyInGo) == 0 && len(d.FieldDiffs) == 0 {
+			continue
+		}
+
+		fmt.Printf("=== %s ===\n", d.Name)
+		fmt.Printf("  Categories: %s\n", strings.Join(d.Categories, ", "))
+		fmt.Printf("  Live: %d assets, %d versions  |  Go: %d assets, %d versions\n",
+			d.LiveCount, len(d.VersionsLive), d.GoCount, len(d.VersionsGo))
+
+		printAssetList("Only in LIVE", d.OnlyInLive, sampleN)
+		printAssetList("Only in Go", d.OnlyInGo, sampleN)
+		printFieldDiffs(d.FieldDiffs, sampleN)
+
+		fmt.Println()
+	}
+}
+
+// printFieldDiffs shows classification differences on shared assets.
+// Shows "real" diffs (both sides non-empty) first, then "fill" diffs
+// (one side empty) as a summary count only.
+func printFieldDiffs(diffs []fieldDiff, sampleN int) {
+	if len(diffs) == 0 {
+		return
+	}
+
+	// Separate real disagreements from fill diffs.
+	var real, fill []fieldDiff
+	for _, fd := range diffs {
+		if fd.BothSet {
+			real = append(real, fd)
+		} else {
+			fill = append(fill, fd)
+		}
+	}
+
+	// Show real disagreements in detail.
+	if len(real) > 0 {
+		byField := make(map[string][]fieldDiff)
+		for _, fd := range real {
+			byField[fd.Field] = append(byField[fd.Field], fd)
+		}
+
+		for _, field := range []string{"os", "arch", "libc", "ext", "channel"} {
+			fds := byField[field]
+			if len(fds) == 0 {
+				continue
+			}
+
+			fmt.Printf("  DISAGREE %s (%d):\n", field, len(fds))
+			printFieldDiffItems(fds, sampleN)
+		}
+	}
+
+	// Summarize fill diffs (live empty, Go classified) as counts.
+	if len(fill) > 0 {
+		byField := make(map[string]int)
+		for _, fd := range fill {
+			byField[fd.Field]++
+		}
+		var parts []string
+		for _, field := range []string{"os", "arch", "libc", "ext", "channel"} {
+			if n := byField[field]; n > 0 {
+				parts = append(parts, fmt.Sprintf("%s(%d)", field, n))
+			}
+		}
+		if len(parts) > 0 {
+			fmt.Printf("  Go fills empty: %s\n", strings.Join(parts, ", "))
+		}
+	}
+}
+
+func printFieldDiffItems(fds []fieldDiff, sampleN int) {
+	items := fds
+	if sampleN > 0 && len(items) > sampleN {
+		sampled := make([]fieldDiff, len(items))
+		copy(sampled, items)
+		rand.Shuffle(len(sampled), func(i, j int) {
+			sampled[i], sampled[j] = sampled[j], sampled[i]
+		})
+		items = sampled[:sampleN]
+		sort.Slice(items, func(i, j int) bool {
+			return items[i].Filename < items[j].Filename
+		})
+	}
+
+	limit := 20
+	for i, fd := range items {
+		if sampleN == 0 && i >= limit {
+			fmt.Printf("    ... and %d more\n", len(fds)-limit)
+			break
+		}
+		fmt.Printf("    - %s: live=%q go=%q\n", fd.Filename, fd.Live, fd.Go)
+	}
+	if sampleN > 0 && len(fds) > sampleN {
+		fmt.Printf("    ... sampled %d of %d\n", sampleN, len(fds))
+	}
+}
+
+// printAssetList prints a list of asset filenames, optionally sampling N at
+// random. When sampleN > 0 and the list is longer, it picks N random items
+// so you can spot classification bugs across the full range instead of only
+// seeing the first alphabetical entries.
+func printAssetList(label string, items []string, sampleN int) {
+	if len(items) == 0 {
+		return
+	}
+
+	fmt.Printf("  %s (%d):\n", label, len(items))
+
+	if sampleN > 0 && len(items) > sampleN {
+		// Shuffle a copy, take first N, then sort for readable output.
+		sampled := make([]string, len(items))
+		copy(sampled, items)
+		rand.Shuffle(len(sampled), func(i, j int) {
+			sampled[i], sampled[j] = sampled[j], sampled[i]
+		})
+		picked := sampled[:sampleN]
+		sort.Strings(picked)
+		for _, f := range picked {
+			fmt.Printf("    - %s\n", f)
+		}
+		fmt.Printf("    ... sampled %d of %d (run again for different sample)\n", sampleN, len(items))
+		return
+	}
+
+	limit := 20
+	for i, f := range items {
+		if i >= limit {
+			fmt.Printf("    ... and %d more\n", len(items)-limit)
+			break
+		}
+		fmt.Printf("    - %s\n", f)
+	}
+}

cmd/e2etest/main.go

@@ -0,0 +1,846 @@
+// Command e2etest runs the full release pipeline for selected packages
+// and compares results against the live webi.sh API.
+//
+// It fetches from upstream, classifies assets, resolves the best match
+// for a set of test queries, then fetches the same queries from the live
+// API and reports any differences.
+//
+// Usage:
+//
+//	go run ./cmd/e2etest
+//	go run ./cmd/e2etest -packages goreleaser,ollama,node
+//	go run ./cmd/e2etest -cache ./_cache/raw    # reuse existing cache
+package main
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/webinstall/webi-installers/internal/buildmeta"
+	"github.com/webinstall/webi-installers/internal/installerconf"
+	"github.com/webinstall/webi-installers/internal/lexver"
+	"github.com/webinstall/webi-installers/internal/rawcache"
+	"github.com/webinstall/webi-installers/internal/releases/github"
+	"github.com/webinstall/webi-installers/internal/releases/githubish"
+	"github.com/webinstall/webi-installers/internal/releases/nodedist"
+	"github.com/webinstall/webi-installers/internal/resolve"
+)
+
+// testCase is one query to resolve and compare against the live API.
+type testCase struct {
+	Name    string
+	Package string
+	OS      buildmeta.OS
+	Arch    buildmeta.Arch
+	Libc    buildmeta.Libc
+	Formats []string
+	UA      string // User-Agent for live API query
+}
+
+// liveResult holds parsed fields from the live webi API response.
+type liveResult struct {
+	Version  string
+	OS       string
+	Arch     string
+	Libc     string
+	Ext      string
+	PkgURL   string
+	PkgFile  string
+	Channel  string
+	Stable   string
+	Latest   string
+	Oses     string
+	Arches   string
+	Libcs    string
+	Formats  string
+}
+
+// UA format from webi.sh bootstrap: "curl {uname -s}/{uname -r} {uname -m}/unknown {libc}"
+// libc is "gnu", "musl", or "libc" (for darwin/other)
+var cases = []testCase{
+	{
+		Name: "goreleaser/linux/x86_64", Package: "goreleaser",
+		OS: buildmeta.OSLinux, Arch: buildmeta.ArchAMD64, Libc: buildmeta.LibcGNU,
+		Formats: []string{".tar.gz", ".tar.xz", ".zip"},
+		UA:      "curl Linux/6.6.123 x86_64/unknown gnu",
+	},
+	{
+		Name: "goreleaser/darwin/arm64", Package: "goreleaser",
+		OS: buildmeta.OSDarwin, Arch: buildmeta.ArchARM64, Libc: "",
+		Formats: []string{".tar.gz", ".tar.xz", ".zip"},
+		UA:      "curl Darwin/25.2.0 arm64/unknown libc",
+	},
+	{
+		Name: "goreleaser/windows/x86_64", Package: "goreleaser",
+		OS: buildmeta.OSWindows, Arch: buildmeta.ArchAMD64, Libc: "",
+		Formats: []string{".zip", ".exe"},
+		UA:      "PowerShell/7.0 Windows/10.0 x86_64/unknown msvc",
+	},
+	{
+		Name: "ollama/linux/x86_64", Package: "ollama",
+		OS: buildmeta.OSLinux, Arch: buildmeta.ArchAMD64, Libc: buildmeta.LibcGNU,
+		Formats: []string{".tar.gz", ".tar.xz", ".tar.zst", ".zip"},
+		UA:      "curl Linux/6.6.123 x86_64/unknown gnu",
+	},
+	{
+		Name: "ollama/darwin/arm64", Package: "ollama",
+		OS: buildmeta.OSDarwin, Arch: buildmeta.ArchARM64, Libc: "",
+		Formats: []string{".tar.gz", ".tar.xz", ".tar.zst", ".zip", ".dmg"},
+		UA:      "curl Darwin/25.2.0 arm64/unknown libc",
+	},
+	{
+		Name: "ollama/linux/arm64", Package: "ollama",
+		OS: buildmeta.OSLinux, Arch: buildmeta.ArchARM64, Libc: buildmeta.LibcGNU,
+		Formats: []string{".tar.gz", ".tar.xz", ".tar.zst", ".zip"},
+		UA:      "curl Linux/6.6.123 aarch64/unknown gnu",
+	},
+	{
+		Name: "node/linux/x86_64", Package: "node",
+		OS: buildmeta.OSLinux, Arch: buildmeta.ArchAMD64, Libc: buildmeta.LibcGNU,
+		Formats: []string{".tar.xz", ".tar.gz", ".zip"},
+		UA:      "curl Linux/6.6.123 x86_64/unknown gnu",
+	},
+	{
+		Name: "node/darwin/arm64", Package: "node",
+		OS: buildmeta.OSDarwin, Arch: buildmeta.ArchARM64, Libc: "",
+		Formats: []string{".tar.xz", ".tar.gz", ".zip"},
+		UA:      "curl Darwin/25.2.0 arm64/unknown libc",
+	},
+	{
+		Name: "node/linux/arm64", Package: "node",
+		OS: buildmeta.OSLinux, Arch: buildmeta.ArchARM64, Libc: buildmeta.LibcGNU,
+		Formats: []string{".tar.xz", ".tar.gz", ".zip"},
+		UA:      "curl Linux/6.6.123 aarch64/unknown gnu",
+	},
+}
+
+func main() {
+	cacheDir := flag.String("cache", "_cache/raw", "root directory for raw cache")
+	confDir := flag.String("conf", ".", "root directory containing {pkg}/releases.conf files")
+	token := flag.String("token", os.Getenv("GITHUB_TOKEN"), "GitHub API token")
+	skipFetch := flag.Bool("skip-fetch", false, "skip fetching, use existing cache")
+	skipLive := flag.Bool("skip-live", false, "skip live API comparison")
+	packages := flag.String("packages", "goreleaser,ollama,node", "comma-separated packages to test")
+	flag.Parse()
+
+	pkgList := strings.Split(*packages, ",")
+	pkgSet := make(map[string]bool, len(pkgList))
+	for _, p := range pkgList {
+		pkgSet[strings.TrimSpace(p)] = true
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	defer cancel()
+
+	client := &http.Client{Timeout: 30 * time.Second}
+	var auth *githubish.Auth
+	if *token != "" {
+		auth = &githubish.Auth{Token: *token}
+	}
+
+	// Step 1: Fetch raw releases.
+	if !*skipFetch {
+		log.Println("=== Step 1: Fetching releases ===")
+		for _, pkg := range pkgList {
+			if err := fetchPackage(ctx, client, *cacheDir, *confDir, pkg, auth); err != nil {
+				log.Fatalf("fetch %s: %v", pkg, err)
+			}
+		}
+	} else {
+		log.Println("=== Step 1: Skipping fetch (using cache) ===")
+	}
+
+	// Step 2: Classify releases.
+	log.Println("=== Step 2: Classifying releases ===")
+	allDists := make(map[string][]resolve.Dist)
+	for _, pkg := range pkgList {
+		conf, err := installerconf.Read(filepath.Join(*confDir, pkg, "releases.conf"))
+		if err != nil {
+			log.Fatalf("read conf %s: %v", pkg, err)
+		}
+		d, err := rawcache.Open(filepath.Join(*cacheDir, pkg))
+		if err != nil {
+			log.Fatalf("open cache %s: %v", pkg, err)
+		}
+		dists, err := classifyFromCache(pkg, conf, d)
+		if err != nil {
+			log.Fatalf("classify %s: %v", pkg, err)
+		}
+		allDists[pkg] = dists
+		log.Printf("  %s: %d distributables", pkg, len(dists))
+
+		// Show catalog.
+		cat := resolve.Survey(dists)
+		log.Printf("    oses=%v arches=%v libcs=%v formats=%v", cat.OSes, cat.Arches, cat.Libcs, cat.Formats)
+		log.Printf("    latest=%s stable=%s", cat.Latest, cat.Stable)
+	}
+
+	// Step 3: Resolve best match for each test case.
+	log.Println("=== Step 3: Resolving best matches ===")
+	type result struct {
+		tc    testCase
+		match *resolve.Match
+		live  *liveResult
+	}
+	var results []result
+	for _, tc := range cases {
+		if !pkgSet[tc.Package] {
+			continue
+		}
+		dists := allDists[tc.Package]
+		q := resolve.Query{
+			OS:      tc.OS,
+			Arch:    tc.Arch,
+			Libc:    tc.Libc,
+			Formats: tc.Formats,
+			Channel: "stable",
+		}
+		m := resolve.Best(dists, q)
+		results = append(results, result{tc: tc, match: m})
+	}
+
+	// Step 4: Compare with live API.
+	if !*skipLive {
+		log.Println("=== Step 4: Comparing with live API ===")
+		for i := range results {
+			tc := results[i].tc
+			live, err := queryLiveAPI(client, tc)
+			if err != nil {
+				log.Printf("  %s: live API error: %v", tc.Name, err)
+				continue
+			}
+			results[i].live = live
+		}
+	}
+
+	// Step 5: Report.
+	log.Println("")
+	log.Println("=== Results ===")
+	log.Println("")
+
+	pass, fail, warn := 0, 0, 0
+	for _, r := range results {
+		tc := r.tc
+		m := r.match
+		live := r.live
+
+		if m == nil {
+			log.Printf("FAIL %s: no match found", tc.Name)
+			fail++
+			continue
+		}
+
+		log.Printf("--- %s ---", tc.Name)
+		log.Printf("  Go:   version=%s file=%s ext=%s url=%s", m.Version, m.Filename, m.Format, m.Download)
+
+		if live != nil {
+			log.Printf("  Live: version=%s file=%s ext=%s url=%s", live.Version, live.PkgFile, live.Ext, live.PkgURL)
+
+			if live.Version == "0.0.0" {
+				log.Printf("  WARN: live API returned error (no match)")
+				warn++
+			} else if m.Version == live.Version && m.Filename == live.PkgFile {
+				log.Printf("  PASS: exact match")
+				pass++
+			} else if m.Version == live.Version && m.Download == live.PkgURL {
+				log.Printf("  PASS: same URL (filename display differs: go=%s live=%s)", m.Filename, live.PkgFile)
+				pass++
+			} else if m.Version == live.Version {
+				log.Printf("  WARN: same version, different file (go=%s live=%s)", m.Filename, live.PkgFile)
+				warn++
+			} else {
+				log.Printf("  DIFF: version mismatch (go=%s live=%s)", m.Version, live.Version)
+				fail++
+			}
+		} else {
+			log.Printf("  (no live comparison)")
+			pass++
+		}
+	}
+
+	log.Println("")
+	log.Printf("Summary: %d pass, %d fail, %d warn (live API errors)", pass, fail, warn)
+
+	if fail > 0 {
+		os.Exit(1)
+	}
+}
+
+// fetchPackage fetches raw releases for one package.
+func fetchPackage(ctx context.Context, client *http.Client, cacheRoot, confDir, pkg string, auth *githubish.Auth) error {
+	conf, err := installerconf.Read(filepath.Join(confDir, pkg, "releases.conf"))
+	if err != nil {
+		return fmt.Errorf("read conf: %w", err)
+	}
+
+	source := conf.Source
+	log.Printf("  %s: source=%s", pkg, source)
+
+	switch source {
+	case "github":
+		return fetchGitHub(ctx, client, cacheRoot, pkg, conf, auth)
+	case "nodedist":
+		return fetchNodeDist(ctx, client, cacheRoot, pkg, conf)
+	default:
+		return fmt.Errorf("unsupported source %q (only github and nodedist for e2e test)", source)
+	}
+}
+
+func fetchGitHub(ctx context.Context, client *http.Client, cacheRoot, pkg string, conf *installerconf.Conf, auth *githubish.Auth) error {
+	owner := conf.Owner
+	repo := conf.Repo
+	tagPrefix := conf.TagPrefix
+
+	d, err := rawcache.Open(filepath.Join(cacheRoot, pkg))
+	if err != nil {
+		return err
+	}
+
+	var added, changed, skipped int
+	var latest string
+	for batch, err := range github.Fetch(ctx, client, owner, repo, auth) {
+		if err != nil {
+			return fmt.Errorf("github %s/%s: %w", owner, repo, err)
+		}
+		for _, rel := range batch {
+			if rel.Draft {
+				continue
+			}
+
+			tag := rel.TagName
+			if tagPrefix != "" {
+				if !strings.HasPrefix(tag, tagPrefix) {
+					continue
+				}
+				tag = strings.TrimPrefix(tag, tagPrefix)
+			}
+
+			data, err := json.Marshal(rel)
+			if err != nil {
+				return fmt.Errorf("marshal %s: %w", tag, err)
+			}
+
+			action, err := d.Merge(tag, data)
+			if err != nil {
+				return err
+			}
+			switch action {
+			case "added":
+				added++
+			case "changed":
+				changed++
+			default:
+				skipped++
+			}
+
+			if latest == "" && !rel.Prerelease {
+				latest = tag
+			}
+		}
+	}
+
+	if latest != "" {
+		current := d.Latest()
+		if current == "" || lexver.Compare(lexver.Parse(latest), lexver.Parse(current)) > 0 {
+			d.SetLatest(latest)
+		}
+	}
+
+	log.Printf("    +%d ~%d =%d latest=%s", added, changed, skipped, d.Latest())
+	return nil
+}
+
+func fetchNodeDist(ctx context.Context, client *http.Client, cacheRoot, pkg string, conf *installerconf.Conf) error {
+	baseURL := conf.BaseURL
+	d, err := rawcache.Open(filepath.Join(cacheRoot, pkg))
+	if err != nil {
+		return err
+	}
+
+	var added, changed, skipped int
+	var latest string
+	for batch, err := range nodedist.Fetch(ctx, client, baseURL) {
+		if err != nil {
+			return fmt.Errorf("nodedist: %w", err)
+		}
+		for _, entry := range batch {
+			tag := entry.Version
+			data, err := json.Marshal(entry)
+			if err != nil {
+				return fmt.Errorf("marshal %s: %w", tag, err)
+			}
+
+			action, err := d.Merge(tag, data)
+			if err != nil {
+				return err
+			}
+			switch action {
+			case "added":
+				added++
+			case "changed":
+				changed++
+			default:
+				skipped++
+			}
+
+			if latest == "" {
+				latest = tag
+			}
+		}
+	}
+
+	if latest != "" {
+		current := d.Latest()
+		if current == "" || lexver.Compare(lexver.Parse(latest), lexver.Parse(current)) > 0 {
+			d.SetLatest(latest)
+		}
+	}
+
+	log.Printf("    +%d ~%d =%d latest=%s", added, changed, skipped, d.Latest())
+	return nil
+}
+
+// classifyFromCache reads the raw cache and produces classified dists.
+func classifyFromCache(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]resolve.Dist, error) {
+	source := conf.Source
+	switch source {
+	case "github":
+		return classifyGitHub(pkg, conf, d)
+	case "nodedist":
+		return classifyNodeDist(pkg, conf, d)
+	default:
+		return nil, fmt.Errorf("unsupported source %q", source)
+	}
+}
+
+func classifyGitHub(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]resolve.Dist, error) {
+	tagPrefix := conf.TagPrefix
+	releases, err := readAllReleases(d)
+	if err != nil {
+		return nil, err
+	}
+
+	var dists []resolve.Dist
+	for _, data := range releases {
+		var rel struct {
+			TagName     string `json:"tag_name"`
+			Prerelease  bool   `json:"prerelease"`
+			Draft       bool   `json:"draft"`
+			PublishedAt string `json:"published_at"`
+			Assets      []struct {
+				Name               string `json:"name"`
+				BrowserDownloadURL string `json:"browser_download_url"`
+				Size               int64  `json:"size"`
+			} `json:"assets"`
+		}
+		if err := json.Unmarshal(data, &rel); err != nil {
+			continue
+		}
+		if rel.Draft {
+			continue
+		}
+
+		version := rel.TagName
+		if tagPrefix != "" {
+			version = strings.TrimPrefix(version, tagPrefix)
+		}
+		// Strip leading "v" for version normalization.
+		version = strings.TrimPrefix(version, "v")
+
+		channel := "stable"
+		if rel.Prerelease {
+			channel = "beta"
+		}
+
+		date := ""
+		if len(rel.PublishedAt) >= 10 {
+			date = rel.PublishedAt[:10]
+		}
+
+		for _, asset := range rel.Assets {
+			if isMetaAsset(asset.Name) {
+				continue
+			}
+
+			r := classifyFilename(asset.Name)
+			extra := detectExtra(asset.Name)
+			dists = append(dists, resolve.Dist{
+				Package:  pkg,
+				Version:  version,
+				Channel:  channel,
+				OS:       r.os,
+				Arch:     r.arch,
+				Libc:     r.libc,
+				Format:   r.format,
+				Download: asset.BrowserDownloadURL,
+				Filename: asset.Name,
+				Size:     asset.Size,
+				Date:     date,
+				Extra:    extra,
+			})
+		}
+	}
+	return dists, nil
+}
+
+func classifyNodeDist(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]resolve.Dist, error) {
+	baseURL := conf.BaseURL
+	releases, err := readAllReleases(d)
+	if err != nil {
+		return nil, err
+	}
+
+	var dists []resolve.Dist
+	for _, data := range releases {
+		var entry struct {
+			Version  string          `json:"version"`
+			Date     string          `json:"date"`
+			Files    []string        `json:"files"`
+			LTS      json.RawMessage `json:"lts"`
+			Security bool            `json:"security"`
+		}
+		if err := json.Unmarshal(data, &entry); err != nil {
+			continue
+		}
+
+		lts := string(entry.LTS) != "false" && string(entry.LTS) != ""
+		version := strings.TrimPrefix(entry.Version, "v")
+
+		// Webi treats even major versions as "stable" (LTS-eligible).
+		channel := "stable"
+		parts := strings.SplitN(version, ".", 2)
+		if len(parts) > 0 {
+			var major int
+			fmt.Sscanf(parts[0], "%d", &major)
+			if major%2 != 0 {
+				channel = "beta"
+			}
+		}
+
+		for _, file := range entry.Files {
+			if file == "src" || file == "headers" {
+				continue
+			}
+			fileDists := expandNodeFile(pkg, entry.Version, version, channel, entry.Date, lts, baseURL, file)
+			dists = append(dists, fileDists...)
+		}
+	}
+	return dists, nil
+}
+
+func expandNodeFile(pkg, rawVersion, version, channel, date string, lts bool, baseURL, file string) []resolve.Dist {
+	parts := strings.Split(file, "-")
+	if len(parts) < 2 {
+		return nil
+	}
+
+	osMap := map[string]string{
+		"osx": "darwin", "linux": "linux", "win": "windows",
+		"sunos": "sunos", "aix": "aix",
+	}
+	archMap := map[string]string{
+		"x64": "x86_64", "x86": "x86", "arm64": "aarch64",
+		"armv7l": "armv7", "armv6l": "armv6",
+		"ppc64": "ppc64", "ppc64le": "ppc64le", "s390x": "s390x",
+		"loong64": "loong64", "riscv64": "riscv64",
+	}
+
+	os_ := osMap[parts[0]]
+	arch := archMap[parts[1]]
+	if os_ == "" || arch == "" {
+		return nil
+	}
+
+	libc := ""
+	pkgType := ""
+	if len(parts) > 2 {
+		pkgType = parts[2]
+	}
+
+	var formats []string
+	switch pkgType {
+	case "musl":
+		libc = "musl"
+		formats = []string{".tar.gz", ".tar.xz"}
+	case "tar":
+		formats = []string{".tar.gz", ".tar.xz"}
+	case "zip":
+		formats = []string{".zip"}
+	case "7z":
+		formats = []string{".7z"}
+	case "pkg":
+		formats = []string{".pkg"}
+	case "msi":
+		formats = []string{".msi"}
+	case "exe":
+		formats = []string{".exe"}
+	case "":
+		formats = []string{".tar.gz", ".tar.xz"}
+	default:
+		return nil
+	}
+
+	if libc == "" && os_ == "linux" {
+		libc = "gnu"
+	}
+
+	osPart := parts[0]
+	if osPart == "osx" {
+		osPart = "darwin"
+	}
+	archPart := parts[1]
+	muslExtra := ""
+	if libc == "musl" {
+		muslExtra = "-musl"
+	}
+
+	var dists []resolve.Dist
+	for _, format := range formats {
+		var filename string
+		if format == ".msi" {
+			filename = fmt.Sprintf("node-%s-%s%s%s", rawVersion, archPart, muslExtra, format)
+		} else {
+			filename = fmt.Sprintf("node-%s-%s-%s%s%s", rawVersion, osPart, archPart, muslExtra, format)
+		}
+
+		dists = append(dists, resolve.Dist{
+			Package:  pkg,
+			Version:  version,
+			Channel:  channel,
+			OS:       os_,
+			Arch:     arch,
+			Libc:     libc,
+			Format:   format,
+			Download: fmt.Sprintf("%s/%s/%s", baseURL, rawVersion, filename),
+			Filename: filename,
+			LTS:      lts,
+			Date:     date,
+		})
+	}
+	return dists
+}
+
+// queryLiveAPI queries the live webi.sh API and parses the response header.
+func queryLiveAPI(client *http.Client, tc testCase) (*liveResult, error) {
+	// Build format string matching what the webi.sh bootstrap sends.
+	// Order: tar,exe,zip,xz,dmg,git (least to most favorable in bootstrap,
+	// but the API doesn't care about order).
+	fmtParam := "tar,exe,zip,xz,dmg"
+
+	url := fmt.Sprintf("https://webi.sh/api/installers/%s@stable.sh?formats=%s", tc.Package, fmtParam)
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", tc.UA)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return parseLiveResponse(string(body)), nil
+}
+
+// parseLiveResponse extracts WEBI_* and PKG_* variables from the shell script.
+func parseLiveResponse(body string) *liveResult {
+	vars := make(map[string]string)
+	scanner := bufio.NewScanner(strings.NewReader(body))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		for _, prefix := range []string{"WEBI_", "PKG_"} {
+			if strings.HasPrefix(line, prefix) {
+				if eq := strings.IndexByte(line, '='); eq > 0 {
+					key := line[:eq]
+					val := line[eq+1:]
+					val = strings.Trim(val, "'\"")
+					vars[key] = val
+				}
+			}
+		}
+	}
+
+	return &liveResult{
+		Version: vars["WEBI_VERSION"],
+		OS:      vars["WEBI_OS"],
+		Arch:    vars["WEBI_ARCH"],
+		Libc:    vars["WEBI_LIBC"],
+		Ext:     vars["WEBI_EXT"],
+		PkgURL:  vars["WEBI_PKG_URL"],
+		PkgFile: vars["WEBI_PKG_FILE"],
+		Channel: vars["WEBI_CHANNEL"],
+		Stable:  vars["PKG_STABLE"],
+		Latest:  vars["PKG_LATEST"],
+		Oses:    vars["PKG_OSES"],
+		Arches:  vars["PKG_ARCHES"],
+		Libcs:   vars["PKG_LIBCS"],
+		Formats: vars["PKG_FORMATS"],
+	}
+}
+
+// readAllReleases reads all cached release files.
+func readAllReleases(d *rawcache.Dir) (map[string][]byte, error) {
+	active, err := d.ActivePath()
+	if err != nil {
+		return nil, err
+	}
+	entries, err := os.ReadDir(active)
+	if err != nil {
+		return nil, err
+	}
+	result := make(map[string][]byte, len(entries))
+	for _, e := range entries {
+		if e.IsDir() || strings.HasPrefix(e.Name(), "_") {
+			continue
+		}
+		data, err := os.ReadFile(filepath.Join(active, e.Name()))
+		if err != nil {
+			return nil, err
+		}
+		result[e.Name()] = data
+	}
+	return result, nil
+}
+
+type classResult struct {
+	os, arch, libc, format string
+}
+
+func classifyFilename(name string) classResult {
+	// Use the classify package.
+	// Import it indirectly to avoid circular deps — inline the logic
+	// we need for the e2e test.
+	lower := strings.ToLower(name)
+
+	var r classResult
+	r.format = detectFormat(name)
+
+	// OS detection
+	switch {
+	case strings.Contains(lower, "linux"):
+		r.os = "linux"
+	case strings.Contains(lower, "darwin") || strings.Contains(lower, "macos") || strings.Contains(lower, "apple"):
+		r.os = "darwin"
+	case strings.Contains(lower, "windows") || strings.Contains(lower, "win64") || strings.Contains(lower, "win32"):
+		r.os = "windows"
+	case strings.HasSuffix(lower, ".dmg") || strings.HasSuffix(lower, ".app.zip"):
+		r.os = "darwin"
+	case strings.HasSuffix(lower, ".exe") || strings.HasSuffix(lower, ".msi"):
+		r.os = "windows"
+	case strings.Contains(lower, "freebsd"):
+		r.os = "freebsd"
+	}
+
+	// Arch detection
+	switch {
+	case strings.Contains(lower, "x86_64") || strings.Contains(lower, "amd64") || strings.Contains(lower, "x64"):
+		r.arch = "x86_64"
+	case strings.Contains(lower, "aarch64") || strings.Contains(lower, "arm64"):
+		r.arch = "aarch64"
+	case strings.Contains(lower, "armv7") || strings.Contains(lower, "armhf"):
+		r.arch = "armv7"
+	case strings.Contains(lower, "armv6"):
+		r.arch = "armv6"
+	case strings.Contains(lower, "i686") || strings.Contains(lower, "i386") || strings.Contains(lower, "x86") || strings.Contains(lower, "386"):
+		r.arch = "x86"
+	case strings.Contains(lower, "ppc64le") || strings.Contains(lower, "powerpc64le"):
+		r.arch = "ppc64le"
+	case strings.Contains(lower, "ppc64") || strings.Contains(lower, "powerpc64"):
+		r.arch = "ppc64"
+	case strings.Contains(lower, "riscv64"):
+		r.arch = "riscv64"
+	case strings.Contains(lower, "s390x"):
+		r.arch = "s390x"
+	case strings.Contains(lower, "loong64"):
+		r.arch = "loong64"
+	}
+
+	// Libc detection
+	switch {
+	case strings.Contains(lower, "musl"):
+		r.libc = "musl"
+	case strings.Contains(lower, "gnu"):
+		r.libc = "gnu"
+	case strings.Contains(lower, "msvc"):
+		r.libc = "msvc"
+	}
+
+	return r
+}
+
+func detectFormat(name string) string {
+	lower := strings.ToLower(name)
+	for _, ext := range []string{".tar.gz", ".tar.xz", ".tar.bz2", ".tar.zst", ".exe.xz", ".app.zip"} {
+		if strings.HasSuffix(lower, ext) {
+			return ext
+		}
+	}
+	// .tgz is a common alias for .tar.gz
+	if strings.HasSuffix(lower, ".tgz") {
+		return ".tar.gz"
+	}
+	return filepath.Ext(lower)
+}
+
+// detectExtra identifies GPU/vendor-specific variant suffixes in filenames
+// like "ollama-linux-amd64-rocm.tar.zst" or "ollama-linux-arm64-jetpack5.tar.zst".
+func detectExtra(name string) string {
+	lower := strings.ToLower(name)
+	for _, variant := range []string{
+		"-rocm", "-jetpack", "-cuda", "-vulkan", "-metal",
+		"-extended", "-static", "-debug", "-nightly",
+	} {
+		if strings.Contains(lower, variant) {
+			return strings.TrimPrefix(variant, "-")
+		}
+	}
+	return ""
+}
+
+func isMetaAsset(name string) bool {
+	lower := strings.ToLower(name)
+	for _, suffix := range []string{
+		".sha256", ".sha256sum", ".sha512", ".sha512sum",
+		".md5", ".md5sum", ".sig", ".asc", ".pem",
+		"checksums.txt", "sha256sums", "sha512sums",
+		".sbom", ".spdx", ".json.sig", ".sigstore",
+		".d.ts", ".pub",
+	} {
+		if strings.HasSuffix(lower, suffix) {
+			return true
+		}
+	}
+	for _, contains := range []string{
+		"checksums", "sha256sum", "sha512sum",
+		"buildable-artifact",
+	} {
+		if strings.Contains(lower, contains) {
+			return true
+		}
+	}
+	for _, exact := range []string{
+		"install.sh", "install.ps1", "compat.json",
+	} {
+		if lower == exact {
+			return true
+		}
+	}
+	return false
+}

cmd/fetchraw/main.go

@@ -0,0 +1,862 @@
+// Command fetchraw fetches release histories from upstream APIs and
+// merges them into rawcache. Safe to run repeatedly — unchanged releases
+// are skipped, new/changed ones are recorded in the audit log.
+//
+// Reads releases.conf files from package directories to discover what
+// to fetch. Adding a new package is just creating a conf file.
+//
+// Usage:
+//
+//	go run ./cmd/fetchraw -cache ./_cache/raw
+//	go run ./cmd/fetchraw -cache ./_cache/raw hugo caddy
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/webinstall/webi-installers/internal/installerconf"
+	"github.com/webinstall/webi-installers/internal/lexver"
+	"github.com/webinstall/webi-installers/internal/rawcache"
+	"github.com/webinstall/webi-installers/internal/releases/chromedist"
+	"github.com/webinstall/webi-installers/internal/releases/flutterdist"
+	"github.com/webinstall/webi-installers/internal/releases/gitea"
+	"github.com/webinstall/webi-installers/internal/releases/github"
+	"github.com/webinstall/webi-installers/internal/releases/githubish"
+	"github.com/webinstall/webi-installers/internal/releases/gittag"
+	"github.com/webinstall/webi-installers/internal/releases/golang"
+	"github.com/webinstall/webi-installers/internal/releases/gpgdist"
+	"github.com/webinstall/webi-installers/internal/releases/hashicorp"
+	"github.com/webinstall/webi-installers/internal/releases/iterm2dist"
+	"github.com/webinstall/webi-installers/internal/releases/juliadist"
+	"github.com/webinstall/webi-installers/internal/releases/mariadbdist"
+	"github.com/webinstall/webi-installers/internal/releases/nodedist"
+	"github.com/webinstall/webi-installers/internal/releases/zigdist"
+)
+
+func main() {
+	cacheDir := flag.String("cache", "_cache/raw", "root directory for raw cache")
+	confDir := flag.String("conf", ".", "root directory containing {pkg}/releases.conf files")
+	token := flag.String("token", os.Getenv("GITHUB_TOKEN"), "GitHub API token")
+	flag.Parse()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+
+	client := &http.Client{Timeout: 30 * time.Second}
+	var auth *githubish.Auth
+	if *token != "" {
+		auth = &githubish.Auth{Token: *token}
+	}
+
+	// Discover packages from releases.conf files.
+	packages, err := discover(*confDir)
+	if err != nil {
+		log.Fatalf("discover: %v", err)
+	}
+
+	// Filter to requested packages if args given.
+	args := flag.Args()
+	if len(args) > 0 {
+		nameSet := make(map[string]bool, len(args))
+		for _, a := range args {
+			nameSet[a] = true
+		}
+		var filtered []pkgConf
+		for _, p := range packages {
+			if nameSet[p.name] {
+				filtered = append(filtered, p)
+			}
+		}
+		packages = filtered
+	}
+
+	log.Printf("found %d packages", len(packages))
+
+	for _, pkg := range packages {
+		// Aliases share cache with their target — skip fetching.
+		if alias := pkg.conf.Extra["alias_of"]; alias != "" {
+			log.Printf("  %s: alias of %s, skipping", pkg.name, alias)
+			continue
+		}
+
+		log.Printf("fetching %s...", pkg.name)
+		var err error
+		switch pkg.conf.Source {
+		case "github":
+			err = fetchGitHub(ctx, client, *cacheDir, pkg.name, pkg.conf, auth)
+		case "nodedist":
+			err = fetchNodeDist(ctx, client, *cacheDir, pkg.name, pkg.conf)
+		case "golang":
+			err = fetchGolang(ctx, client, *cacheDir, pkg.name)
+		case "zigdist":
+			err = fetchZig(ctx, client, *cacheDir, pkg.name)
+		case "flutterdist":
+			err = fetchFlutter(ctx, client, *cacheDir, pkg.name)
+		case "iterm2dist":
+			err = fetchITerm2(ctx, client, *cacheDir, pkg.name)
+		case "hashicorp":
+			err = fetchHashiCorp(ctx, client, *cacheDir, pkg.name, pkg.conf)
+		case "juliadist":
+			err = fetchJulia(ctx, client, *cacheDir, pkg.name)
+		case "gittag":
+			err = fetchGitTag(ctx, *cacheDir, pkg.name, pkg.conf)
+		case "gitea":
+			err = fetchGitea(ctx, client, *cacheDir, pkg.name, pkg.conf)
+		case "chromedist":
+			err = fetchChrome(ctx, client, *cacheDir, pkg.name)
+		case "gpgdist":
+			err = fetchGPG(ctx, client, *cacheDir, pkg.name)
+		case "mariadbdist":
+			err = fetchMariaDB(ctx, client, *cacheDir, pkg.name)
+		default:
+			log.Printf("  %s: unknown source %q, skipping", pkg.name, pkg.conf.Source)
+			continue
+		}
+		if err != nil {
+			log.Printf("  ERROR: %s: %v", pkg.name, err)
+		}
+	}
+}
+
+type pkgConf struct {
+	name string
+	conf *installerconf.Conf
+}
+
+// discover finds all {dir}/*/releases.conf files and returns them sorted.
+func discover(dir string) ([]pkgConf, error) {
+	pattern := filepath.Join(dir, "*", "releases.conf")
+	matches, err := filepath.Glob(pattern)
+	if err != nil {
+		return nil, err
+	}
+
+	var packages []pkgConf
+	for _, path := range matches {
+		name := filepath.Base(filepath.Dir(path))
+		// Skip infrastructure dirs (_example, _webi, _common, etc.)
+		if strings.HasPrefix(name, "_") {
+			continue
+		}
+		conf, err := installerconf.Read(path)
+		if err != nil {
+			log.Printf("warning: %s: %v", path, err)
+			continue
+		}
+		packages = append(packages, pkgConf{name: name, conf: conf})
+	}
+
+	sort.Slice(packages, func(i, j int) bool {
+		return packages[i].name < packages[j].name
+	})
+
+	return packages, nil
+}
+
+func fetchNodeDist(ctx context.Context, client *http.Client, cacheRoot, pkgName string, conf *installerconf.Conf) error {
+	baseURL := conf.BaseURL
+	if baseURL == "" {
+		return fmt.Errorf("missing url in releases.conf")
+	}
+
+	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
+	if err != nil {
+		return err
+	}
+
+	var added, changed, skipped int
+	var latest string
+	for batch, err := range nodedist.Fetch(ctx, client, baseURL) {
+		if err != nil {
+			return fmt.Errorf("%s fetch: %w", pkgName, err)
+		}
+		for _, entry := range batch {
+			tag := entry.Version
+			data, err := json.Marshal(entry)
+			if err != nil {
+				return fmt.Errorf("%s marshal %s: %w", pkgName, tag, err)
+			}
+
+			action, err := d.Merge(tag, data)
+			if err != nil {
+				return err
+			}
+			switch action {
+			case "added":
+				added++
+			case "changed":
+				changed++
+			default:
+				skipped++
+			}
+
+			if latest == "" {
+				latest = tag
+			}
+		}
+	}
+
+	if err := updateLatest(d, latest); err != nil {
+		return err
+	}
+
+	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
+	return nil
+}
+
+func fetchGitHub(ctx context.Context, client *http.Client, cacheRoot, pkgName string, conf *installerconf.Conf, auth *githubish.Auth) error {
+	owner := conf.Owner
+	repo := conf.Repo
+	tagPrefix := conf.TagPrefix
+
+	if owner == "" || repo == "" {
+		return fmt.Errorf("missing owner or repo in releases.conf")
+	}
+
+	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
+	if err != nil {
+		return err
+	}
+
+	var added, changed, skipped int
+	var latest string
+	for batch, err := range github.Fetch(ctx, client, owner, repo, auth) {
+		if err != nil {
+			return fmt.Errorf("github %s/%s: %w", owner, repo, err)
+		}
+		for _, rel := range batch {
+			if rel.Draft {
+				continue
+			}
+
+			tag := rel.TagName
+
+			if tagPrefix != "" {
+				if !strings.HasPrefix(tag, tagPrefix) {
+					continue
+				}
+				tag = strings.TrimPrefix(tag, tagPrefix)
+			}
+
+			data, err := json.Marshal(rel)
+			if err != nil {
+				return fmt.Errorf("marshal %s: %w", tag, err)
+			}
+
+			action, err := d.Merge(tag, data)
+			if err != nil {
+				return err
+			}
+			switch action {
+			case "added":
+				added++
+			case "changed":
+				changed++
+			default:
+				skipped++
+			}
+
+			if latest == "" && !rel.Prerelease {
+				latest = tag
+			}
+		}
+	}
+
+	if err := updateLatest(d, latest); err != nil {
+		return err
+	}
+
+	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
+	return nil
+}
+
+func updateLatest(d *rawcache.Dir, candidate string) error {
+	if candidate == "" {
+		return nil
+	}
+	current := d.Latest()
+	if current == "" || lexver.Compare(lexver.Parse(candidate), lexver.Parse(current)) > 0 {
+		return d.SetLatest(candidate)
+	}
+	return nil
+}
+
+func fetchGolang(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
+	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
+	if err != nil {
+		return err
+	}
+
+	var added, changed, skipped int
+	var latest string
+	for batch, err := range golang.Fetch(ctx, client) {
+		if err != nil {
+			return fmt.Errorf("golang: %w", err)
+		}
+		for _, rel := range batch {
+			tag := rel.Version // "go1.24.1"
+			data, err := json.Marshal(rel)
+			if err != nil {
+				return fmt.Errorf("golang marshal %s: %w", tag, err)
+			}
+
+			action, err := d.Merge(tag, data)
+			if err != nil {
+				return err
+			}
+			switch action {
+			case "added":
+				added++
+			case "changed":
+				changed++
+			default:
+				skipped++
+			}
+
+			if latest == "" && rel.Stable {
+				latest = tag
+			}
+		}
+	}
+
+	if err := updateLatest(d, latest); err != nil {
+		return err
+	}
+
+	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
+	return nil
+}
+
+func fetchZig(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
+	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
+	if err != nil {
+		return err
+	}
+
+	var added, changed, skipped int
+	var latest string
+	for batch, err := range zigdist.Fetch(ctx, client) {
+		if err != nil {
+			return fmt.Errorf("zigdist: %w", err)
+		}
+		for _, rel := range batch {
+			tag := rel.Version
+			data, err := json.Marshal(rel)
+			if err != nil {
+				return fmt.Errorf("zigdist marshal %s: %w", tag, err)
+			}
+
+			action, err := d.Merge(tag, data)
+			if err != nil {
+				return err
+			}
+			switch action {
+			case "added":
+				added++
+			case "changed":
+				changed++
+			default:
+				skipped++
+			}
+
+			// Stable versions have dots and no dev/pre markers.
+			isStable := strings.Contains(tag, ".") && !strings.ContainsAny(tag, "+-")
+			if isStable {
+				if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
+					latest = tag
+				}
+			}
+		}
+	}
+
+	if err := updateLatest(d, latest); err != nil {
+		return err
+	}
+
+	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
+	return nil
+}
+
+func fetchFlutter(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
+	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
+	if err != nil {
+		return err
+	}
+
+	var added, changed, skipped int
+	var latest string
+	for batch, err := range flutterdist.Fetch(ctx, client) {
+		if err != nil {
+			return fmt.Errorf("flutterdist: %w", err)
+		}
+		for _, rel := range batch {
+			// Use version+channel+os+arch as the tag. The arch is embedded
+			// in the archive path (e.g. flutter_macos_arm64_3.0.0-stable.zip
+			// vs flutter_macos_3.0.0-stable.zip for universal/x64).
+			arch := ""
+			base := filepath.Base(rel.Archive)
+			prefix := "flutter_" + rel.OS + "_"
+			if after, ok := strings.CutPrefix(base, prefix); ok {
+				if !strings.HasPrefix(after, rel.Version) {
+					// There's an arch segment between OS and version.
+					if idx := strings.Index(after, "_"); idx > 0 {
+						arch = after[:idx]
+					}
+				}
+			}
+			tag := fmt.Sprintf("%s-%s-%s", rel.Version, rel.Channel, rel.OS)
+			if arch != "" {
+				tag += "-" + arch
+			}
+			data, err := json.Marshal(rel)
+			if err != nil {
+				return fmt.Errorf("flutterdist marshal %s: %w", tag, err)
+			}
+
+			action, err := d.Merge(tag, data)
+			if err != nil {
+				return err
+			}
+			switch action {
+			case "added":
+				added++
+			case "changed":
+				changed++
+			default:
+				skipped++
+			}
+
+			if latest == "" && rel.Channel == "stable" {
+				latest = tag
+			}
+		}
+	}
+
+	if err := updateLatest(d, latest); err != nil {
+		return err
+	}
+
+	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
+	return nil
+}
+
+func fetchITerm2(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
+	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
+	if err != nil {
+		return err
+	}
+
+	var added, changed, skipped int
+	var latest string
+	for batch, err := range iterm2dist.Fetch(ctx, client) {
+		if err != nil {
+			return fmt.Errorf("iterm2dist: %w", err)
+		}
+		for _, entry := range batch {
+			tag := entry.Version
+			if tag == "" {
+				continue
+			}
+			data, err := json.Marshal(entry)
+			if err != nil {
+				return fmt.Errorf("iterm2dist marshal %s: %w", tag, err)
+			}
+
+			action, err := d.Merge(tag, data)
+			if err != nil {
+				return err
+			}
+			switch action {
+			case "added":
+				added++
+			case "changed":
+				changed++
+			default:
+				skipped++
+			}
+
+			if latest == "" && entry.Channel == "stable" {
+				latest = tag
+			}
+		}
+	}
+
+	if err := updateLatest(d, latest); err != nil {
+		return err
+	}
+
+	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
+	return nil
+}
+
+func fetchHashiCorp(ctx context.Context, client *http.Client, cacheRoot, pkgName string, conf *installerconf.Conf) error {
+	product := conf.Extra["product"]
+	if product == "" {
+		return fmt.Errorf("missing product in releases.conf")
+	}
+
+	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
+	if err != nil {
+		return err
+	}
+
+	var added, changed, skipped int
+	var latest string
+	for idx, err := range hashicorp.Fetch(ctx, client, product) {
+		if err != nil {
+			return fmt.Errorf("hashicorp %s: %w", product, err)
+		}
+		for tag, ver := range idx.Versions {
+			data, err := json.Marshal(ver)
+			if err != nil {
+				return fmt.Errorf("hashicorp marshal %s: %w", tag, err)
+			}
+
+			action, err := d.Merge(tag, data)
+			if err != nil {
+				return err
+			}
+			switch action {
+			case "added":
+				added++
+			case "changed":
+				changed++
+			default:
+				skipped++
+			}
+
+			// Stable = no prerelease markers. Compare all to find highest.
+			isStable := !strings.ContainsAny(tag, "-+")
+			if isStable {
+				if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
+					latest = tag
+				}
+			}
+		}
+	}
+
+	if err := updateLatest(d, latest); err != nil {
+		return err
+	}
+
+	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
+	return nil
+}
+
+func fetchJulia(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
+	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
+	if err != nil {
+		return err
+	}
+
+	var added, changed, skipped int
+	var latest string
+	for batch, err := range juliadist.Fetch(ctx, client) {
+		if err != nil {
+			return fmt.Errorf("juliadist: %w", err)
+		}
+		for _, rel := range batch {
+			tag := rel.Version
+			data, err := json.Marshal(rel)
+			if err != nil {
+				return fmt.Errorf("juliadist marshal %s: %w", tag, err)
+			}
+
+			action, err := d.Merge(tag, data)
+			if err != nil {
+				return err
+			}
+			switch action {
+			case "added":
+				added++
+			case "changed":
+				changed++
+			default:
+				skipped++
+			}
+
+			if rel.Stable {
+				if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
+					latest = tag
+				}
+			}
+		}
+	}
+
+	if err := updateLatest(d, latest); err != nil {
+		return err
+	}
+
+	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
+	return nil
+}
+
+func fetchGitTag(ctx context.Context, cacheRoot, pkgName string, conf *installerconf.Conf) error {
+	gitURL := conf.BaseURL
+	if gitURL == "" {
+		return fmt.Errorf("missing url in releases.conf")
+	}
+
+	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
+	if err != nil {
+		return err
+	}
+
+	repoDir := filepath.Join(cacheRoot, "_repos")
+	if err := os.MkdirAll(repoDir, 0o755); err != nil {
+		return err
+	}
+
+	var added, changed, skipped int
+	var latest string
+	for batch, err := range gittag.Fetch(ctx, gitURL, repoDir) {
+		if err != nil {
+			return fmt.Errorf("gittag %s: %w", pkgName, err)
+		}
+		for _, entry := range batch {
+			tag := entry.Version
+			if tag == "" {
+				tag = "HEAD-" + entry.CommitHash
+			}
+			data, err := json.Marshal(entry)
+			if err != nil {
+				return fmt.Errorf("gittag marshal %s: %w", tag, err)
+			}
+
+			action, err := d.Merge(tag, data)
+			if err != nil {
+				return err
+			}
+			switch action {
+			case "added":
+				added++
+			case "changed":
+				changed++
+			default:
+				skipped++
+			}
+
+			if entry.GitTag != "" && entry.GitTag != "HEAD" {
+				if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
+					latest = tag
+				}
+			}
+		}
+	}
+
+	if err := updateLatest(d, latest); err != nil {
+		return err
+	}
+
+	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
+	return nil
+}
+
+func fetchGitea(ctx context.Context, client *http.Client, cacheRoot, pkgName string, conf *installerconf.Conf) error {
+	baseURL := conf.BaseURL
+	owner := conf.Owner
+	repo := conf.Repo
+
+	if baseURL == "" || owner == "" || repo == "" {
+		return fmt.Errorf("missing base_url, owner, or repo in releases.conf")
+	}
+
+	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
+	if err != nil {
+		return err
+	}
+
+	var added, changed, skipped int
+	var latest string
+	for batch, err := range gitea.Fetch(ctx, client, baseURL, owner, repo, nil) {
+		if err != nil {
+			return fmt.Errorf("gitea %s/%s: %w", owner, repo, err)
+		}
+		for _, rel := range batch {
+			if rel.Draft {
+				continue
+			}
+
+			tag := rel.TagName
+			data, err := json.Marshal(rel)
+			if err != nil {
+				return fmt.Errorf("gitea marshal %s: %w", tag, err)
+			}
+
+			action, err := d.Merge(tag, data)
+			if err != nil {
+				return err
+			}
+			switch action {
+			case "added":
+				added++
+			case "changed":
+				changed++
+			default:
+				skipped++
+			}
+
+			if latest == "" && !rel.Prerelease {
+				latest = tag
+			}
+		}
+	}
+
+	if err := updateLatest(d, latest); err != nil {
+		return err
+	}
+
+	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
+	return nil
+}
+
+func fetchChrome(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
+	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
+	if err != nil {
+		return err
+	}
+
+	var added, changed, skipped int
+	var latest string
+	for batch, err := range chromedist.Fetch(ctx, client) {
+		if err != nil {
+			return fmt.Errorf("chromedist: %w", err)
+		}
+		for _, ver := range batch {
+			tag := ver.Version
+			data, err := json.Marshal(ver)
+			if err != nil {
+				return fmt.Errorf("chromedist marshal %s: %w", tag, err)
+			}
+
+			action, err := d.Merge(tag, data)
+			if err != nil {
+				return err
+			}
+			switch action {
+			case "added":
+				added++
+			case "changed":
+				changed++
+			default:
+				skipped++
+			}
+
+			if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
+				latest = tag
+			}
+		}
+	}
+
+	if err := updateLatest(d, latest); err != nil {
+		return err
+	}
+
+	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
+	return nil
+}
+
+func fetchGPG(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
+	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
+	if err != nil {
+		return err
+	}
+
+	var added, changed, skipped int
+	var latest string
+	for batch, err := range gpgdist.Fetch(ctx, client) {
+		if err != nil {
+			return fmt.Errorf("gpgdist: %w", err)
+		}
+		for _, entry := range batch {
+			tag := entry.Version
+			data, err := json.Marshal(entry)
+			if err != nil {
+				return fmt.Errorf("gpgdist marshal %s: %w", tag, err)
+			}
+
+			action, err := d.Merge(tag, data)
+			if err != nil {
+				return err
+			}
+			switch action {
+			case "added":
+				added++
+			case "changed":
+				changed++
+			default:
+				skipped++
+			}
+
+			if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
+				latest = tag
+			}
+		}
+	}
+
+	if err := updateLatest(d, latest); err != nil {
+		return err
+	}
+
+	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
+	return nil
+}
+
+func fetchMariaDB(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
+	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
+	if err != nil {
+		return err
+	}
+
+	var added, changed, skipped int
+	var latest string
+	for batch, err := range mariadbdist.Fetch(ctx, client) {
+		if err != nil {
+			return fmt.Errorf("mariadbdist: %w", err)
+		}
+		for _, rel := range batch {
+			tag := rel.ReleaseID
+			data, err := json.Marshal(rel)
+			if err != nil {
+				return fmt.Errorf("mariadbdist marshal %s: %w", tag, err)
+			}
+
+			action, err := d.Merge(tag, data)
+			if err != nil {
+				return err
+			}
+			switch action {
+			case "added":
+				added++
+			case "changed":
+				changed++
+			default:
+				skipped++
+			}
+
+			isStable := rel.MajorStatus == "Stable"
+			if isStable {
+				if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
+					latest = tag
+				}
+			}
+		}
+	}
+
+	if err := updateLatest(d, latest); err != nil {
+		return err
+	}
+
+	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
+	return nil
+}

cmd/inspect/main.go

@@ -0,0 +1,625 @@
+// Command inspect downloads release archives, unpacks them, and reports
+// their internal structure. This helps discover how packages are laid out
+// and whether the layout changes across versions.
+//
+// Usage:
+//
+//	go run ./cmd/inspect -csv distributables.csv -cache ./_cache/downloads ollama sd
+package main
+
+import (
+	"context"
+	"encoding/csv"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"mime"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/webinstall/webi-installers/internal/httpclient"
+)
+
+// Row is one CSV row from distributables.csv.
+type Row struct {
+	Package  string
+	Version  string
+	Channel  string
+	Date     string
+	OS       string
+	Arch     string
+	Libc     string
+	Format   string
+	Download string
+	Filename string
+	Extra    string
+}
+
+// archiveFormats are the formats we download and unpack.
+var archiveFormats = map[string]bool{
+	".tar.gz":  true,
+	".tar.xz":  true,
+	".tar.bz2": true,
+	".tar.zst": true,
+	".zip":     true,
+	".dmg":     true,
+	".gz":      true,
+	".xz":      true,
+}
+
+// inspectOSes are the OSes we inspect.
+var inspectOSes = map[string]bool{
+	"linux":   true,
+	"darwin":  true,
+	"windows": true,
+	"":        true, // source-only packages
+}
+
+// preferredArch picks one arch per OS to download.
+func preferredArch(os_ string) string {
+	switch os_ {
+	case "darwin":
+		return "aarch64"
+	default:
+		return "x86_64"
+	}
+}
+
+func main() {
+	csvFile := flag.String("csv", "distributables.csv", "path to distributables CSV")
+	cacheDir := flag.String("cache", "_cache/downloads", "download cache directory")
+	flag.Parse()
+
+	packages := flag.Args()
+	if len(packages) == 0 {
+		log.Fatal("usage: inspect [-csv FILE] [-cache DIR] PACKAGE [PACKAGE...]")
+	}
+
+	rows, err := readCSV(*csvFile)
+	if err != nil {
+		log.Fatalf("read csv: %v", err)
+	}
+
+	client := httpclient.New()
+	// Override timeout for large downloads.
+	client.Timeout = 10 * time.Minute
+
+	for _, pkg := range packages {
+		log.Printf("=== %s ===", pkg)
+		if err := inspectPackage(client, rows, pkg, *cacheDir); err != nil {
+			log.Printf("ERROR: %s: %v", pkg, err)
+		}
+	}
+}
+
+func readCSV(path string) ([]Row, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	r := csv.NewReader(f)
+	header, err := r.Read()
+	if err != nil {
+		return nil, err
+	}
+
+	// Build column index.
+	idx := make(map[string]int, len(header))
+	for i, col := range header {
+		idx[col] = i
+	}
+
+	var rows []Row
+	for {
+		record, err := r.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		get := func(col string) string {
+			if i, ok := idx[col]; ok && i < len(record) {
+				return record[i]
+			}
+			return ""
+		}
+
+		rows = append(rows, Row{
+			Package:  get("package"),
+			Version:  get("version"),
+			Channel:  get("channel"),
+			Date:     get("date"),
+			OS:       get("os"),
+			Arch:     get("arch"),
+			Libc:     get("libc"),
+			Format:   get("format"),
+			Download: get("download"),
+			Filename: get("filename"),
+			Extra:    get("extra"),
+		})
+	}
+	return rows, nil
+}
+
+func inspectPackage(client *http.Client, allRows []Row, pkg, cacheDir string) error {
+	// Filter rows for this package.
+	var pkgRows []Row
+	for _, r := range allRows {
+		if r.Package == pkg {
+			pkgRows = append(pkgRows, r)
+		}
+	}
+	if len(pkgRows) == 0 {
+		return fmt.Errorf("no rows found")
+	}
+
+	// Find latest stable version, fall back to any version.
+	versions := findVersionsByDate(pkgRows)
+	if len(versions) == 0 {
+		return fmt.Errorf("no versions found")
+	}
+
+	latestVer := versions[0]
+	log.Printf("  latest version: %s", latestVer)
+
+	// Check if latest has assets uploaded (more than just source tarballs).
+	latestRows := filterVersion(pkgRows, latestVer)
+	hasRealAssets := false
+	for _, r := range latestRows {
+		if r.Extra != "source" && archiveFormats[r.Format] {
+			hasRealAssets = true
+			break
+		}
+	}
+
+	// If latest looks empty, step back one version.
+	if !hasRealAssets && len(versions) > 1 {
+		latestVer = versions[1]
+		latestRows = filterVersion(pkgRows, latestVer)
+		log.Printf("  latest has no assets, using: %s", latestVer)
+	}
+
+	// Inspect the latest version.
+	if err := inspectVersion(client, pkg, latestVer, latestRows, cacheDir); err != nil {
+		return err
+	}
+
+	// Find versions roughly a year apart going back.
+	yearVersions := findYearlyVersions(pkgRows, latestVer)
+	for _, v := range yearVersions {
+		log.Printf("  --- checking %s ---", v)
+		vRows := filterVersion(pkgRows, v)
+		if err := inspectVersion(client, pkg, v, vRows, cacheDir); err != nil {
+			log.Printf("    ERROR: %v", err)
+		}
+	}
+
+	return nil
+}
+
+// findVersionsByDate returns versions sorted newest first, preferring stable.
+func findVersionsByDate(rows []Row) []string {
+	type vInfo struct {
+		version string
+		date    string
+		stable  bool
+	}
+	seen := map[string]*vInfo{}
+	for _, r := range rows {
+		if _, ok := seen[r.Version]; !ok {
+			seen[r.Version] = &vInfo{
+				version: r.Version,
+				date:    r.Date,
+				stable:  r.Channel == "stable",
+			}
+		}
+	}
+
+	var vs []*vInfo
+	for _, v := range seen {
+		vs = append(vs, v)
+	}
+
+	// Sort: stable first, then by date descending, then version descending.
+	sort.Slice(vs, func(i, j int) bool {
+		if vs[i].stable != vs[j].stable {
+			return vs[i].stable
+		}
+		if vs[i].date != vs[j].date {
+			return vs[i].date > vs[j].date
+		}
+		return vs[i].version > vs[j].version
+	})
+
+	result := make([]string, len(vs))
+	for i, v := range vs {
+		result[i] = v.version
+	}
+	return result
+}
+
+// findYearlyVersions picks versions roughly a year apart before the given version.
+func findYearlyVersions(rows []Row, latestVer string) []string {
+	// Find the date of latest.
+	var latestDate string
+	for _, r := range rows {
+		if r.Version == latestVer && r.Date != "" {
+			latestDate = r.Date
+			break
+		}
+	}
+	if latestDate == "" {
+		return nil
+	}
+
+	latestTime, err := time.Parse("2006-01-02", latestDate)
+	if err != nil {
+		return nil
+	}
+
+	// Collect all stable versions with dates.
+	type vd struct {
+		version string
+		date    time.Time
+	}
+	seen := map[string]bool{}
+	var all []vd
+	for _, r := range rows {
+		if seen[r.Version] || r.Date == "" || r.Channel != "stable" {
+			continue
+		}
+		seen[r.Version] = true
+		t, err := time.Parse("2006-01-02", r.Date)
+		if err != nil {
+			continue
+		}
+		if t.Before(latestTime) {
+			all = append(all, vd{r.Version, t})
+		}
+	}
+
+	sort.Slice(all, func(i, j int) bool {
+		return all[i].date.After(all[j].date)
+	})
+
+	// Pick versions roughly a year apart.
+	var result []string
+	nextTarget := latestTime.AddDate(-1, 0, 0)
+	for _, v := range all {
+		if v.date.Before(nextTarget) || v.date.Equal(nextTarget) {
+			result = append(result, v.version)
+			nextTarget = v.date.AddDate(-1, 0, 0)
+		}
+	}
+
+	return result
+}
+
+func filterVersion(rows []Row, version string) []Row {
+	var result []Row
+	for _, r := range rows {
+		if r.Version == version {
+			result = append(result, r)
+		}
+	}
+	return result
+}
+
+// inspectVersion downloads and inspects archives for one version.
+func inspectVersion(client *http.Client, pkg, version string, rows []Row, cacheDir string) error {
+	// Group by OS, pick one arch per OS, pick distinct formats.
+	type dlKey struct {
+		os_    string
+		format string
+	}
+	selected := map[dlKey]*Row{}
+
+	for i := range rows {
+		r := &rows[i]
+		if !inspectOSes[r.OS] {
+			continue
+		}
+		if !archiveFormats[r.Format] {
+			continue
+		}
+
+		key := dlKey{r.OS, r.Format}
+		existing := selected[key]
+		if existing == nil {
+			selected[key] = r
+			continue
+		}
+
+		// Prefer the preferred arch.
+		pref := preferredArch(r.OS)
+		if r.Arch == pref && existing.Arch != pref {
+			selected[key] = r
+		}
+		// Skip rocm/jetpack variants.
+		if strings.Contains(r.Filename, "rocm") || strings.Contains(r.Filename, "jetpack") {
+			if !strings.Contains(existing.Filename, "rocm") && !strings.Contains(existing.Filename, "jetpack") {
+				continue // keep existing non-special variant
+			}
+		}
+	}
+
+	if len(selected) == 0 {
+		log.Printf("  %s: no downloadable archives", version)
+		return nil
+	}
+
+	// Sort keys for deterministic output.
+	var keys []dlKey
+	for k := range selected {
+		keys = append(keys, k)
+	}
+	sort.Slice(keys, func(i, j int) bool {
+		if keys[i].os_ != keys[j].os_ {
+			return keys[i].os_ < keys[j].os_
+		}
+		return keys[i].format < keys[j].format
+	})
+
+	for _, key := range keys {
+		r := selected[key]
+		os_ := r.OS
+		if os_ == "" {
+			os_ = "any"
+		}
+		log.Printf("  [%s] %s %s → %s", version, os_, r.Format, r.Filename)
+
+		dlPath, err := download(client, r.Download, r.Filename, filepath.Join(cacheDir, pkg, version))
+		if err != nil {
+			log.Printf("    download error: %v", err)
+			continue
+		}
+
+		contents, err := unpackAndList(dlPath, r.Format)
+		if err != nil {
+			log.Printf("    unpack error: %v", err)
+			continue
+		}
+
+		printContents(contents)
+	}
+
+	return nil
+}
+
+// download fetches a URL to the cache dir. Returns the path to the cached file.
+// Skips download if the file already exists.
+func download(client *http.Client, url, hintFilename, dir string) (string, error) {
+	// Check if already cached by hint filename.
+	cached := filepath.Join(dir, hintFilename)
+	if _, err := os.Stat(cached); err == nil {
+		return cached, nil
+	}
+
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return "", err
+	}
+
+	ctx := context.Background()
+	resp, err := httpclient.Get(ctx, client, url)
+	if err != nil {
+		return "", fmt.Errorf("GET %s: %w", url, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("GET %s: %s", url, resp.Status)
+	}
+
+	// Determine filename from Content-Disposition or hint.
+	filename := hintFilename
+	if cd := resp.Header.Get("Content-Disposition"); cd != "" {
+		_, params, err := mime.ParseMediaType(cd)
+		if err == nil {
+			if fn, ok := params["filename"]; ok && fn != "" {
+				filename = fn
+			}
+		}
+	}
+
+	outPath := filepath.Join(dir, filename)
+
+	// Atomic write: temp file + rename.
+	tmp := outPath + ".tmp"
+	f, err := os.Create(tmp)
+	if err != nil {
+		return "", err
+	}
+
+	n, err := io.Copy(f, resp.Body)
+	if closeErr := f.Close(); closeErr != nil && err == nil {
+		err = closeErr
+	}
+	if err != nil {
+		os.Remove(tmp)
+		return "", fmt.Errorf("download %s: %w", url, err)
+	}
+
+	if err := os.Rename(tmp, outPath); err != nil {
+		os.Remove(tmp)
+		return "", err
+	}
+
+	log.Printf("    downloaded %s (%d bytes)", filename, n)
+	return outPath, nil
+}
+
+// FileEntry describes one file inside an archive.
+type FileEntry struct {
+	Path       string
+	Size       int64
+	Mode       os.FileMode
+	IsDir      bool
+	IsExec     bool
+	IsSymlink  bool
+	LinkTarget string
+}
+
+// unpackAndList extracts an archive to a temp dir and lists contents.
+func unpackAndList(archivePath, format string) ([]FileEntry, error) {
+	tmpDir, err := os.MkdirTemp("", "webi-inspect-*")
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(tmpDir)
+
+	switch format {
+	case ".tar.gz":
+		err = run("tar", "xzf", archivePath, "-C", tmpDir)
+	case ".tar.xz":
+		err = run("tar", "xJf", archivePath, "-C", tmpDir)
+	case ".tar.bz2":
+		err = run("tar", "xjf", archivePath, "-C", tmpDir)
+	case ".tar.zst":
+		err = run("tar", "--zstd", "-xf", archivePath, "-C", tmpDir)
+	case ".zip":
+		err = run("unzip", "-q", "-o", archivePath, "-d", tmpDir)
+	case ".dmg":
+		err = extractDMG(archivePath, tmpDir)
+	case ".gz":
+		// Single file gzip.
+		base := filepath.Base(archivePath)
+		base = strings.TrimSuffix(base, ".gz")
+		outPath := filepath.Join(tmpDir, base)
+		err = run("sh", "-c", fmt.Sprintf("gunzip -c %q > %q", archivePath, outPath))
+	case ".xz":
+		base := filepath.Base(archivePath)
+		base = strings.TrimSuffix(base, ".xz")
+		outPath := filepath.Join(tmpDir, base)
+		err = run("sh", "-c", fmt.Sprintf("xz -dc %q > %q", archivePath, outPath))
+	default:
+		return nil, fmt.Errorf("unsupported format: %s", format)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("extract %s: %w", format, err)
+	}
+
+	return listDir(tmpDir, "")
+}
+
+func extractDMG(dmgPath, outDir string) error {
+	// Try 7z first (doesn't require mounting).
+	if _, err := exec.LookPath("7z"); err == nil {
+		return run("7z", "x", "-o"+outDir, dmgPath)
+	}
+
+	// Fall back to hdiutil mount + copy + unmount.
+	mountPoint, err := os.MkdirTemp("", "webi-dmg-*")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(mountPoint)
+
+	if err := run("hdiutil", "attach", dmgPath, "-mountpoint", mountPoint, "-nobrowse", "-quiet"); err != nil {
+		return fmt.Errorf("mount dmg: %w", err)
+	}
+	defer run("hdiutil", "detach", mountPoint, "-quiet")
+
+	// Copy contents.
+	return run("cp", "-R", mountPoint+"/.", outDir)
+}
+
+func run(name string, args ...string) error {
+	cmd := exec.Command(name, args...)
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func listDir(root, prefix string) ([]FileEntry, error) {
+	entries, err := os.ReadDir(filepath.Join(root, prefix))
+	if err != nil {
+		return nil, err
+	}
+
+	var result []FileEntry
+	for _, e := range entries {
+		relPath := filepath.Join(prefix, e.Name())
+		fullPath := filepath.Join(root, relPath)
+
+		info, err := e.Info()
+		if err != nil {
+			continue
+		}
+
+		entry := FileEntry{
+			Path:  relPath,
+			Size:  info.Size(),
+			Mode:  info.Mode(),
+			IsDir: e.IsDir(),
+		}
+
+		if info.Mode()&os.ModeSymlink != 0 {
+			entry.IsSymlink = true
+			target, _ := os.Readlink(fullPath)
+			entry.LinkTarget = target
+		}
+
+		if !e.IsDir() && info.Mode()&0o111 != 0 {
+			entry.IsExec = true
+		}
+
+		result = append(result, entry)
+
+		if e.IsDir() {
+			sub, err := listDir(root, relPath)
+			if err != nil {
+				continue
+			}
+			result = append(result, sub...)
+		}
+	}
+	return result, nil
+}
+
+func printContents(entries []FileEntry) {
+	for _, e := range entries {
+		marker := "  "
+		if e.IsExec {
+			marker = "* "
+		}
+		if e.IsDir {
+			marker = "d "
+		}
+		if e.IsSymlink {
+			marker = "→ "
+		}
+
+		size := ""
+		if !e.IsDir {
+			size = formatSize(e.Size)
+		}
+
+		line := fmt.Sprintf("    %s%-50s %8s", marker, e.Path, size)
+		if e.IsSymlink {
+			line += " → " + e.LinkTarget
+		}
+		log.Print(line)
+	}
+}
+
+func formatSize(n int64) string {
+	switch {
+	case n >= 1<<30:
+		return fmt.Sprintf("%.1fG", float64(n)/float64(1<<30))
+	case n >= 1<<20:
+		return fmt.Sprintf("%.1fM", float64(n)/float64(1<<20))
+	case n >= 1<<10:
+		return fmt.Sprintf("%.1fK", float64(n)/float64(1<<10))
+	default:
+		return fmt.Sprintf("%dB", n)
+	}
+}

cmd/uaparse/main.go

@@ -0,0 +1,356 @@
+// Command uaparse analyzes User-Agent strings from webi.sh logs.
+//
+// It reads UA strings (one per line) from stdin or a file, parses each
+// through uadetect, and produces summary output showing:
+//   - unique platform tuples (os, arch, libc) with counts
+//   - platform hints extracted from kernel version strings (cloud provider,
+//     container runtime, device info)
+//   - detection failures and malformed UAs
+//
+// Usage:
+//
+//	uaparse < LIVE-UAS.txt
+//	uaparse LIVE-UAS.txt
+//	uaparse -json LIVE-UAS.txt
+//	uaparse -fixtures LIVE-UAS.txt    # output Go test fixtures
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/webinstall/webi-installers/internal/uadetect"
+)
+
+// PlatformKey is the resolution-relevant tuple — everything else is noise
+// for artifact selection.
+type PlatformKey struct {
+	OS   string `json:"os"`
+	Arch string `json:"arch"`
+	Libc string `json:"libc"`
+}
+
+func (k PlatformKey) String() string {
+	return fmt.Sprintf("%-10s %-10s %s", k.OS, k.Arch, k.Libc)
+}
+
+// PlatformEntry holds a unique platform and its metadata.
+type PlatformEntry struct {
+	Key      PlatformKey `json:"key"`
+	Count    int         `json:"count"`
+	Examples []string    `json:"examples"` // up to 3 representative UAs
+	Hints    []string    `json:"hints"`    // unique platform hints seen
+}
+
+// UAIssue records a malformed or undetectable UA.
+type UAIssue struct {
+	Line   int    `json:"line"`
+	UA     string `json:"ua"`
+	Reason string `json:"reason"`
+}
+
+// Hint is a platform detail extracted from the kernel version string.
+type Hint struct {
+	Tag   string // short label: "amzn", "azure", "gcp", "wsl", etc.
+	Count int
+}
+
+func main() {
+	jsonOut := flag.Bool("json", false, "output as JSON")
+	fixtures := flag.Bool("fixtures", false, "output Go test fixture table")
+	flag.Parse()
+
+	var scanner *bufio.Scanner
+	if flag.NArg() > 0 {
+		f, err := os.Open(flag.Arg(0))
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "uaparse: %v\n", err)
+			os.Exit(1)
+		}
+		defer f.Close()
+		scanner = bufio.NewScanner(f)
+	} else {
+		scanner = bufio.NewScanner(os.Stdin)
+	}
+
+	platforms := make(map[PlatformKey]*PlatformEntry)
+	hints := make(map[string]int)
+	var issues []UAIssue
+	lineNum := 0
+
+	for scanner.Scan() {
+		lineNum++
+		ua := strings.TrimSpace(scanner.Text())
+		if ua == "" {
+			continue
+		}
+
+		// Detect corruption: truncated/double-pasted lines.
+		if isMalformed(ua) {
+			issues = append(issues, UAIssue{
+				Line:   lineNum,
+				UA:     ua,
+				Reason: "malformed (truncated or corrupted)",
+			})
+			continue
+		}
+
+		// Parse through uadetect.
+		result := uadetect.Parse(ua)
+
+		// Check for detection failures.
+		if result.OS == "" {
+			issues = append(issues, UAIssue{
+				Line:   lineNum,
+				UA:     ua,
+				Reason: "OS not detected",
+			})
+		}
+		if result.Arch == "" {
+			issues = append(issues, UAIssue{
+				Line:   lineNum,
+				UA:     ua,
+				Reason: "arch not detected",
+			})
+		}
+
+		key := PlatformKey{
+			OS:   string(result.OS),
+			Arch: string(result.Arch),
+			Libc: string(result.Libc),
+		}
+
+		entry, ok := platforms[key]
+		if !ok {
+			entry = &PlatformEntry{Key: key}
+			platforms[key] = entry
+		}
+		entry.Count++
+		if len(entry.Examples) < 3 {
+			entry.Examples = append(entry.Examples, ua)
+		}
+
+		// Extract platform hints from kernel version.
+		for _, h := range extractHints(ua) {
+			if !containsStr(entry.Hints, h) {
+				entry.Hints = append(entry.Hints, h)
+			}
+			hints[h]++
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		fmt.Fprintf(os.Stderr, "uaparse: read error: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Sort platforms by count descending.
+	entries := make([]*PlatformEntry, 0, len(platforms))
+	for _, e := range platforms {
+		entries = append(entries, e)
+	}
+	sort.Slice(entries, func(i, j int) bool {
+		return entries[i].Count > entries[j].Count
+	})
+
+	if *jsonOut {
+		outputJSON(entries, issues, hints)
+	} else if *fixtures {
+		outputFixtures(entries)
+	} else {
+		outputTable(entries, issues, hints, lineNum)
+	}
+}
+
+func outputTable(entries []*PlatformEntry, issues []UAIssue, hints map[string]int, total int) {
+	fmt.Printf("=== UA Analysis: %d lines → %d unique platforms ===\n\n", total, len(entries))
+
+	fmt.Printf("%-10s %-10s %-6s %6s  %s\n", "OS", "ARCH", "LIBC", "COUNT", "HINTS")
+	fmt.Println(strings.Repeat("-", 72))
+	for _, e := range entries {
+		hintStr := ""
+		if len(e.Hints) > 0 {
+			hintStr = strings.Join(e.Hints, ", ")
+		}
+		fmt.Printf("%-10s %-10s %-6s %6d  %s\n",
+			displayOS(e.Key.OS), e.Key.Arch, displayLibc(e.Key.Libc),
+			e.Count, hintStr)
+	}
+
+	if len(hints) > 0 {
+		fmt.Printf("\n=== Platform Hints (environment signals from kernel strings) ===\n\n")
+		sortedHints := make([]Hint, 0, len(hints))
+		for tag, count := range hints {
+			sortedHints = append(sortedHints, Hint{tag, count})
+		}
+		sort.Slice(sortedHints, func(i, j int) bool {
+			return sortedHints[i].Count > sortedHints[j].Count
+		})
+		for _, h := range sortedHints {
+			fmt.Printf("  %-20s %d\n", h.Tag, h.Count)
+		}
+	}
+
+	if len(issues) > 0 {
+		fmt.Printf("\n=== Issues (%d) ===\n\n", len(issues))
+		for _, iss := range issues {
+			fmt.Printf("  line %d: %s\n    %s\n", iss.Line, iss.Reason, iss.UA)
+		}
+	}
+}
+
+func outputJSON(entries []*PlatformEntry, issues []UAIssue, hints map[string]int) {
+	out := struct {
+		Platforms []*PlatformEntry `json:"platforms"`
+		Issues    []UAIssue        `json:"issues"`
+		Hints     map[string]int   `json:"hints"`
+	}{entries, issues, hints}
+
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+	_ = enc.Encode(out)
+}
+
+func outputFixtures(entries []*PlatformEntry) {
+	fmt.Println("// Generated by cmd/uaparse from live UA data.")
+	fmt.Println("// Each entry represents a unique (os, arch, libc) platform seen in production.")
+	fmt.Println("var liveUAPlatforms = []struct {")
+	fmt.Println("\tua   string")
+	fmt.Println("\tos   buildmeta.OS")
+	fmt.Println("\tarch buildmeta.Arch")
+	fmt.Println("\tlibc buildmeta.Libc")
+	fmt.Println("}{")
+
+	for _, e := range entries {
+		if e.Key.OS == "" || e.Key.Arch == "" {
+			continue // skip undetectable
+		}
+		ua := e.Examples[0]
+		fmt.Printf("\t{%q, %s, %s, %s},\n",
+			ua, goConst("OS", e.Key.OS), goConst("Arch", e.Key.Arch), goConst("Libc", e.Key.Libc))
+	}
+
+	fmt.Println("}")
+}
+
+// isMalformed checks for genuinely corrupted UA strings (network truncation).
+func isMalformed(ua string) bool {
+	// Extremely short (less than 10 chars) suggests truncation.
+	if len(ua) < 10 {
+		return true
+	}
+
+	return false
+}
+
+// extractHints finds environment signals in a UA string.
+func extractHints(ua string) []string {
+	lower := strings.ToLower(ua)
+	var out []string
+
+	patterns := []struct {
+		substr string
+		tag    string
+	}{
+		{"amzn", "amzn"},         // Amazon Linux
+		{"-azure", "azure"},     // Azure VM
+		{"-gcp", "gcp"},         // Google Cloud
+		{"-aws", "aws"},         // AWS kernel
+		{"-oracle", "oracle"},   // Oracle Cloud
+		{"el7", "rhel7"},        // RHEL/CentOS 7
+		{"el8", "rhel8"},        // RHEL/CentOS 8
+		{"el9", "rhel9"},        // RHEL/CentOS 9
+		{".fc", "fedora"},       // Fedora
+		{"+deb", "debian"},      // Debian
+		{"-generic", "ubuntu"},  // Ubuntu generic kernel
+		{"-pve", "proxmox"},     // Proxmox VE
+		{"linuxkit", "docker"},  // Docker Desktop / linuxkit
+		{"orbstack", "orbstack"},
+		{"microsoft-standard-wsl", "wsl"},
+		{"android", "android"},
+		{"+rpt-rpi", "rpi"},     // Raspberry Pi
+		{"cygwin", "cygwin"},
+		{"mingw", "mingw"},
+		{"msys", "msys"},
+		{"freebsd", "freebsd"},
+		{"-nvidia", "nvidia"},
+		{"gentoo", "gentoo"},
+		{"coreweave", "coreweave"},
+	}
+
+	for _, p := range patterns {
+		if strings.Contains(lower, p.substr) {
+			out = append(out, p.tag)
+		}
+	}
+
+	return out
+}
+
+// androidDeviceRe extracts device/build info from Android kernel strings.
+var androidDeviceRe = regexp.MustCompile(`ab[A-Z0-9]+`)
+
+func displayOS(os string) string {
+	if os == "" {
+		return "(none)"
+	}
+	return os
+}
+
+func displayLibc(libc string) string {
+	if libc == "" {
+		return "(none)"
+	}
+	return libc
+}
+
+func goConst(prefix, val string) string {
+	m := map[string]map[string]string{
+		"OS": {
+			"darwin":  "buildmeta.OSDarwin",
+			"linux":   "buildmeta.OSLinux",
+			"windows": "buildmeta.OSWindows",
+			"freebsd": "buildmeta.OSFreeBSD",
+			"android": "buildmeta.OSAndroid",
+			"":        `""`,
+		},
+		"Arch": {
+			"aarch64": "buildmeta.ArchARM64",
+			"x86_64":  "buildmeta.ArchAMD64",
+			"armv7":   "buildmeta.ArchARMv7",
+			"armv6":   "buildmeta.ArchARMv6",
+			"x86":     "buildmeta.ArchX86",
+			"ppc64le": "buildmeta.ArchPPC64LE",
+			"ppc64":   "buildmeta.ArchPPC64",
+			"s390x":   "buildmeta.ArchS390X",
+			"riscv64": "buildmeta.ArchRISCV64",
+			"":        `""`,
+		},
+		"Libc": {
+			"gnu":  "buildmeta.LibcGNU",
+			"musl": "buildmeta.LibcMusl",
+			"msvc": "buildmeta.LibcMSVC",
+			"none": "buildmeta.LibcNone",
+			"":     `""`,
+		},
+	}
+	if v, ok := m[prefix][val]; ok {
+		return v
+	}
+	return fmt.Sprintf("%q /* unmapped */", val)
+}
+
+func containsStr(ss []string, s string) bool {
+	for _, v := range ss {
+		if v == s {
+			return true
+		}
+	}
+	return false
+}

cmd/webicached/main.go

@@ -0,0 +1,899 @@
+// Command webicached is the release cache daemon. It fetches releases
+// from upstream sources, classifies build assets, and writes them to
+// the _cache/ directory in the format the Node.js server expects.
+//
+// This is the Go replacement for the Node.js release-fetching pipeline.
+// It reads releases.conf files to discover packages, fetches from the
+// configured source, classifies assets, and writes to fsstore.
+//
+// Default mode: classify all from existing rawcache on startup, then
+// fetch+refresh one package per tick (round-robin, 15m default).
+//
+// Usage:
+//
+//	go run ./cmd/webicached                # default: round-robin, one per tick
+//	go run ./cmd/webicached -eager         # fetch all packages on startup
+//	go run ./cmd/webicached -once -no-fetch # classify from rawcache and exit
+//	go run ./cmd/webicached bat goreleaser # only these packages
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"math/rand/v2"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/joho/godotenv"
+	"github.com/webinstall/webi-installers/internal/classifypkg"
+	"github.com/webinstall/webi-installers/internal/installerconf"
+	"github.com/webinstall/webi-installers/internal/rawcache"
+	"github.com/webinstall/webi-installers/internal/releases/chromedist"
+	"github.com/webinstall/webi-installers/internal/releases/flutterdist"
+	"github.com/webinstall/webi-installers/internal/releases/gitea"
+	"github.com/webinstall/webi-installers/internal/releases/github"
+	"github.com/webinstall/webi-installers/internal/releases/githubish"
+	"github.com/webinstall/webi-installers/internal/releases/gittag"
+	"github.com/webinstall/webi-installers/internal/releases/golang"
+	"github.com/webinstall/webi-installers/internal/releases/gpgdist"
+	"github.com/webinstall/webi-installers/internal/releases/hashicorp"
+	"github.com/webinstall/webi-installers/internal/releases/iterm2dist"
+	"github.com/webinstall/webi-installers/internal/releases/juliadist"
+	"github.com/webinstall/webi-installers/internal/releases/mariadbdist"
+	"github.com/webinstall/webi-installers/internal/releases/nodedist"
+	"github.com/webinstall/webi-installers/internal/releases/servicemandist"
+	"github.com/webinstall/webi-installers/internal/releases/zigdist"
+	"github.com/webinstall/webi-installers/internal/storage"
+	"github.com/webinstall/webi-installers/internal/storage/fsstore"
+	"github.com/webinstall/webi-installers/internal/storage/pgstore"
+)
+
+var (
+	name         = "webicached"
+	version      = "0.0.0-dev"
+	commit       = "0000000"
+	date         = "0001-01-01"
+	licenseYear  = "2024"
+	licenseOwner = "AJ ONeal"
+	licenseType  = "MPL-2.0"
+)
+
+func printVersion(w io.Writer) {
+	b_ver := strings.TrimPrefix(version, "v")
+	_, _ = fmt.Fprintf(w, "%s v%s %s (%s)\n", name, b_ver, commit[:7], date)
+	_, _ = fmt.Fprintf(w, "Copyright (C) %s %s\n", licenseYear, licenseOwner)
+	_, _ = fmt.Fprintf(w, "Licensed under %s\n", licenseType)
+}
+
+type MainConfig struct {
+	envFile   string
+	confDir   string
+	cacheDir  string
+	pgDSN     string
+	rawDir    string
+	token     string
+	once      bool
+	noFetch   bool
+	shallow   bool
+	eager     bool
+	interval  time.Duration
+	pageDelay time.Duration
+}
+
+// WebiCache holds the configuration for the cache daemon.
+type WebiCache struct {
+	ConfDir   string          // root directory with {pkg}/releases.conf files
+	Store     storage.Store   // classified asset storage (fsstore or pgstore)
+	RawDir    string          // raw upstream response cache
+	Client    *http.Client    // HTTP client for upstream calls
+	Auth      *githubish.Auth // GitHub API auth (optional)
+	Shallow   bool            // fetch only the first page of releases
+	NoFetch   bool            // skip fetching, classify from existing raw data only
+	PageDelay time.Duration   // delay between paginated API requests
+}
+
+// delayTransport wraps an http.RoundTripper to add a delay between requests.
+type delayTransport struct {
+	base  http.RoundTripper
+	delay time.Duration
+	last  time.Time
+}
+
+func (t *delayTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	if !t.last.IsZero() && t.delay > 0 {
+		if wait := t.delay - time.Since(t.last); wait > 0 {
+			time.Sleep(wait)
+		}
+	}
+	t.last = time.Now()
+	return t.base.RoundTrip(req)
+}
+
+func main() {
+	if len(os.Args) > 1 {
+		switch os.Args[1] {
+		case "-V", "-version", "--version", "version":
+			printVersion(os.Stdout)
+			os.Exit(0)
+		case "help", "-help", "--help":
+			printVersion(os.Stdout)
+			fmt.Fprintln(os.Stdout, "")
+			fs := flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
+			fs.SetOutput(os.Stdout)
+			registerFlags(fs, &MainConfig{})
+			fs.Usage()
+			os.Exit(0)
+		}
+	}
+
+	cfg := MainConfig{}
+	fs := flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
+	registerFlags(fs, &cfg)
+	if err := fs.Parse(os.Args[1:]); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			os.Exit(0)
+		}
+		os.Exit(1)
+	}
+
+	cfg.cacheDir = expandHome(cfg.cacheDir)
+	cfg.rawDir = expandHome(cfg.rawDir)
+
+	if cfg.envFile != "" {
+		if err := godotenv.Load(cfg.envFile); err != nil {
+			log.Fatalf("envfile: %v", err)
+		}
+	}
+	if cfg.token == "" {
+		cfg.token = os.Getenv("GITHUB_TOKEN")
+	}
+
+	var store storage.Store
+	if cfg.pgDSN != "" {
+		pg, err := pgstore.New(context.Background(), cfg.pgDSN)
+		if err != nil {
+			log.Fatalf("pgstore: %v", err)
+		}
+		store = pg
+	} else {
+		fs, err := fsstore.New(cfg.cacheDir)
+		if err != nil {
+			log.Fatalf("fsstore: %v", err)
+		}
+		store = fs
+	}
+
+	var auth *githubish.Auth
+	if cfg.token != "" {
+		auth = &githubish.Auth{Token: cfg.token}
+	}
+
+	client := &http.Client{Timeout: 30 * time.Second}
+	if cfg.pageDelay > 0 {
+		client.Transport = &delayTransport{
+			base:  http.DefaultTransport,
+			delay: cfg.pageDelay,
+		}
+	}
+
+	wc := &WebiCache{
+		ConfDir:   cfg.confDir,
+		Store:     store,
+		RawDir:    cfg.rawDir,
+		Client:    client,
+		Auth:      auth,
+		Shallow:   cfg.shallow,
+		NoFetch:   cfg.noFetch,
+		PageDelay: cfg.pageDelay,
+	}
+
+	filterPkgs := fs.Args()
+
+	if cfg.eager {
+		wc.Run(filterPkgs)
+		if cfg.once {
+			return
+		}
+	} else if cfg.once {
+		wc.Run(filterPkgs)
+		return
+	} else {
+		saved := wc.NoFetch
+		wc.NoFetch = true
+		wc.Run(filterPkgs)
+		wc.NoFetch = saved
+	}
+
+	packages, err := discover(wc.ConfDir)
+	if err != nil {
+		log.Fatalf("discover: %v", err)
+	}
+	if len(filterPkgs) > 0 {
+		nameSet := make(map[string]bool, len(filterPkgs))
+		for _, a := range filterPkgs {
+			nameSet[a] = true
+		}
+		var filtered []pkgConf
+		for _, p := range packages {
+			if nameSet[p.name] {
+				filtered = append(filtered, p)
+			}
+		}
+		packages = filtered
+	}
+
+	var real []pkgConf
+	for _, pkg := range packages {
+		if pkg.conf.AliasOf == "" {
+			real = append(real, pkg)
+		}
+	}
+
+	log.Printf("refreshing %d packages, interval %s, batch size 20 (ctrl-c to stop)", len(real), cfg.interval)
+	for {
+		stale := wc.stalest(real)
+		if len(stale) == 0 {
+			log.Printf("all packages fresh, sleeping %s", cfg.interval)
+			time.Sleep(cfg.interval)
+			continue
+		}
+
+		batch := stale
+		if len(batch) > 20 {
+			batch = batch[:20]
+		}
+		rand.Shuffle(len(batch), func(i, j int) {
+			batch[i], batch[j] = batch[j], batch[i]
+		})
+
+		log.Printf("batch: %d stale, refreshing %d (most stale first)", len(stale), len(batch))
+		for _, pkg := range batch {
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+			if err := wc.refreshPackage(ctx, pkg); err != nil {
+				log.Printf("  ERROR %s: %v", pkg.name, err)
+			}
+			cancel()
+			time.Sleep(cfg.interval)
+		}
+	}
+}
+
+func registerFlags(fs *flag.FlagSet, cfg *MainConfig) {
+	fs.StringVar(&cfg.envFile, "envfile", "", "path to .env file to load before running")
+	fs.StringVar(&cfg.confDir, "conf", ".", "root directory containing {pkg}/releases.conf files")
+	fs.StringVar(&cfg.cacheDir, "legacy", "~/.cache/webi/legacy", "legacy cache directory (fsstore root)")
+	fs.StringVar(&cfg.pgDSN, "pg", "", "PostgreSQL DSN (enables pgstore; mutually exclusive with -legacy)")
+	fs.StringVar(&cfg.rawDir, "raw", "~/.cache/webi/raw", "raw cache directory for upstream responses")
+	fs.StringVar(&cfg.token, "token", "", "GitHub API token (or set $GITHUB_TOKEN)")
+	fs.BoolVar(&cfg.once, "once", false, "run once then exit (no periodic refresh)")
+	fs.BoolVar(&cfg.noFetch, "no-fetch", false, "skip fetching, classify from existing raw data only")
+	fs.BoolVar(&cfg.shallow, "shallow", false, "fetch only the first page of releases (latest)")
+	fs.BoolVar(&cfg.eager, "eager", false, "fetch all packages on startup (default: one per tick)")
+	fs.DurationVar(&cfg.interval, "interval", 9*time.Second, "delay between individual package fetches")
+	fs.DurationVar(&cfg.pageDelay, "page-delay", 2*time.Second, "delay between paginated API requests")
+}
+
+func expandHome(path string) string {
+	if !strings.HasPrefix(path, "~/") {
+		return path
+	}
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return path
+	}
+	return filepath.Join(home, path[2:])
+}
+
+// stalest returns packages sorted by most stale first (oldest UpdatedAt).
+// Packages with no cache entry or empty assets are considered most stale.
+func (wc *WebiCache) stalest(packages []pkgConf) []pkgConf {
+	type stamped struct {
+		pkg       pkgConf
+		updatedAt time.Time
+	}
+
+	var stale []stamped
+	ctx := context.Background()
+	for _, pkg := range packages {
+		data, err := wc.Store.Load(ctx, pkg.name)
+		var t time.Time
+		hasAssets := false
+		if err == nil && data != nil {
+			t = data.UpdatedAt
+			hasAssets = len(data.Assets) > 0
+		}
+		// Never fetched, or has no assets despite having a timestamp
+		// (e.g. classified from empty rawcache), or older than 10 minutes.
+		if t.IsZero() || !hasAssets || time.Since(t) > 10*time.Minute {
+			stale = append(stale, stamped{pkg: pkg, updatedAt: t})
+		}
+	}
+
+	sort.SliceStable(stale, func(i, j int) bool {
+		ti, tj := stale[i].updatedAt, stale[j].updatedAt
+		if ti.Equal(tj) {
+			return stale[i].pkg.name < stale[j].pkg.name
+		}
+		return ti.Before(tj)
+	})
+
+	result := make([]pkgConf, len(stale))
+	for i, s := range stale {
+		result[i] = s.pkg
+	}
+	return result
+}
+
+// Run discovers packages and refreshes each one.
+func (wc *WebiCache) Run(filterPkgs []string) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+
+	packages, err := discover(wc.ConfDir)
+	if err != nil {
+		log.Printf("discover: %v", err)
+		return
+	}
+
+	if len(filterPkgs) > 0 {
+		nameSet := make(map[string]bool, len(filterPkgs))
+		for _, a := range filterPkgs {
+			nameSet[a] = true
+		}
+		var filtered []pkgConf
+		for _, p := range packages {
+			if nameSet[p.name] {
+				filtered = append(filtered, p)
+			}
+		}
+		packages = filtered
+	}
+
+	var real []pkgConf
+	for _, pkg := range packages {
+		if pkg.conf.AliasOf != "" {
+			continue
+		}
+		real = append(real, pkg)
+	}
+
+	log.Printf("refreshing %d packages", len(real))
+	runStart := time.Now()
+
+	for _, pkg := range real {
+		if err := wc.refreshPackage(ctx, pkg); err != nil {
+			log.Printf("  ERROR %s: %v", pkg.name, err)
+		}
+	}
+
+	log.Printf("refreshed %d packages in %s", len(real), time.Since(runStart))
+}
+
+type pkgConf struct {
+	name string
+	conf *installerconf.Conf
+}
+
+func discover(dir string) ([]pkgConf, error) {
+	pattern := filepath.Join(dir, "*", "releases.conf")
+	matches, err := filepath.Glob(pattern)
+	if err != nil {
+		return nil, err
+	}
+
+	var packages []pkgConf
+	for _, path := range matches {
+		pkgDir := filepath.Dir(path)
+		name := filepath.Base(pkgDir)
+		if strings.HasPrefix(name, "_") {
+			continue
+		}
+
+		// If the package directory is a symlink, treat it as an alias
+		// of the symlink target (e.g. rust.vim → vim-rust).
+		fi, err := os.Lstat(filepath.Join(dir, name))
+		if err != nil {
+			log.Printf("warning: %s: %v", name, err)
+			continue
+		}
+		if fi.Mode()&os.ModeSymlink != 0 {
+			target, err := os.Readlink(filepath.Join(dir, name))
+			if err != nil {
+				log.Printf("warning: readlink %s: %v", name, err)
+				continue
+			}
+			packages = append(packages, pkgConf{
+				name: name,
+				conf: &installerconf.Conf{AliasOf: target},
+			})
+			continue
+		}
+
+		conf, err := installerconf.Read(path)
+		if err != nil {
+			log.Printf("warning: %s: %v", path, err)
+			continue
+		}
+		packages = append(packages, pkgConf{name: name, conf: conf})
+	}
+
+	sort.Slice(packages, func(i, j int) bool {
+		return packages[i].name < packages[j].name
+	})
+	return packages, nil
+}
+
+// refreshPackage does the full pipeline for one package:
+// fetch raw → classify → write to fsstore.
+func (wc *WebiCache) refreshPackage(ctx context.Context, pkg pkgConf) error {
+	pkgStart := time.Now()
+	name := pkg.name
+	conf := pkg.conf
+
+	// Step 1: Fetch raw upstream data to rawcache (unless -no-fetch).
+	if !wc.NoFetch {
+		shallow := wc.Shallow
+		if !shallow {
+			d, err := rawcache.Open(filepath.Join(wc.RawDir, name))
+			if err == nil && d.Populated() {
+				shallow = true
+			}
+		}
+		fetchStart := time.Now()
+		if err := wc.fetchRaw(ctx, pkg, shallow); err != nil {
+			return fmt.Errorf("fetch: %w", err)
+		}
+		log.Printf("  %s: fetch %s", name, time.Since(fetchStart))
+	}
+
+	// Step 2: Classify raw data into assets, tag variants, apply config.
+	classifyStart := time.Now()
+	d, err := rawcache.Open(filepath.Join(wc.RawDir, name))
+	if err != nil {
+		return fmt.Errorf("rawcache open: %w", err)
+	}
+
+	// Open supplementary gittag raw cache if available (for packages with
+	// git_url that use a non-gittag source type like servicemandist).
+	var gitTagDir *rawcache.Dir
+	if conf.GitURL != "" && conf.Source != "gittag" {
+		gd, gdErr := rawcache.Open(filepath.Join(wc.RawDir, "_gittag", name))
+		if gdErr == nil && gd.Populated() {
+			gitTagDir = gd
+		}
+	}
+
+	assets, err := classifypkg.Package(name, conf, d, gitTagDir)
+	if err != nil {
+		return fmt.Errorf("classify: %w", err)
+	}
+	classifyDur := time.Since(classifyStart)
+
+	// Step 3: Write to fsstore.
+	writeStart := time.Now()
+	tx, err := wc.Store.BeginRefresh(ctx, name)
+	if err != nil {
+		return fmt.Errorf("begin refresh: %w", err)
+	}
+	if err := tx.Put(assets); err != nil {
+		tx.Rollback()
+		return fmt.Errorf("put: %w", err)
+	}
+	if err := tx.Commit(ctx); err != nil {
+		return fmt.Errorf("commit: %w", err)
+	}
+	writeDur := time.Since(writeStart)
+
+	log.Printf("  %s: %d assets (classify %s, write %s, total %s)",
+		name, len(assets), classifyDur, writeDur, time.Since(pkgStart))
+	return nil
+}
+
+// --- Fetch raw ---
+
+func (wc *WebiCache) fetchRaw(ctx context.Context, pkg pkgConf, shallow bool) error {
+	switch pkg.conf.Source {
+	case "github", "githubsource":
+		if err := wc.fetchGitHub(ctx, pkg.name, pkg.conf, shallow); err != nil {
+			return err
+		}
+	case "nodedist":
+		return wc.fetchNodeDist(ctx, pkg.name, pkg.conf)
+	case "gittag":
+		return wc.fetchGitTag(ctx, pkg.name, pkg.conf, shallow)
+	case "gitea":
+		return wc.fetchGitea(ctx, pkg.name, pkg.conf, shallow)
+	case "chromedist":
+		return fetchChromeDist(ctx, wc.Client, wc.RawDir, pkg.name)
+	case "flutterdist":
+		return fetchFlutterDist(ctx, wc.Client, wc.RawDir, pkg.name)
+	case "golang":
+		return fetchGolang(ctx, wc.Client, wc.RawDir, pkg.name)
+	case "gpgdist":
+		return fetchGPGDist(ctx, wc.Client, wc.RawDir, pkg.name)
+	case "hashicorp":
+		return fetchHashiCorp(ctx, wc.Client, wc.RawDir, pkg.name, pkg.conf)
+	case "iterm2dist":
+		return fetchITerm2Dist(ctx, wc.Client, wc.RawDir, pkg.name)
+	case "juliadist":
+		return fetchJuliaDist(ctx, wc.Client, wc.RawDir, pkg.name)
+	case "mariadbdist":
+		return fetchMariaDBDist(ctx, wc.Client, wc.RawDir, pkg.name)
+	case "servicemandist":
+		if err := servicemandist.Fetch(ctx, wc.Client, wc.RawDir, pkg.name, wc.Auth, shallow); err != nil {
+			return err
+		}
+	case "zigdist":
+		return fetchZigDist(ctx, wc.Client, wc.RawDir, pkg.name)
+	default:
+		log.Printf("  %s: source %q not yet supported, skipping", pkg.name, pkg.conf.Source)
+		return nil
+	}
+
+	// For non-gittag sources with a git_url, also clone the repo to get
+	// commit hashes. Git entries are classified from this data in
+	// refreshPackage, not from the main raw cache.
+	if pkg.conf.GitURL != "" && pkg.conf.Source != "gittag" {
+		gitShallow := shallow
+		if !wc.Shallow {
+			gd, gdErr := rawcache.Open(filepath.Join(wc.RawDir, "_gittag", pkg.name))
+			if gdErr == nil && !gd.Populated() {
+				gitShallow = false
+			}
+		}
+		if err := wc.fetchGitTagSupplementary(ctx, pkg.name, pkg.conf.GitURL, gitShallow); err != nil {
+			log.Printf("  %s: supplementary gittag fetch: %v", pkg.name, err)
+		}
+	}
+	return nil
+}
+
+// fetchGitTagSupplementary clones a git repo to get commit hashes for
+// packages that use a non-gittag source type (servicemandist, githubsource)
+// but also have a git_url for source installs.
+func (wc *WebiCache) fetchGitTagSupplementary(ctx context.Context, pkgName, gitURL string, shallow bool) error {
+	d, err := rawcache.Open(filepath.Join(wc.RawDir, "_gittag", pkgName))
+	if err != nil {
+		return err
+	}
+
+	repoDir := filepath.Join(wc.RawDir, "_repos")
+	os.MkdirAll(repoDir, 0o755)
+
+	for batch, err := range gittag.Fetch(ctx, gitURL, repoDir) {
+		if err != nil {
+			return err
+		}
+		for _, entry := range batch {
+			tag := entry.Version
+			if tag == "" {
+				tag = "HEAD-" + entry.CommitHash
+			}
+			data, _ := json.Marshal(entry)
+			d.Merge(tag, data)
+		}
+		if shallow {
+			break
+		}
+	}
+	return nil
+}
+
+func (wc *WebiCache) fetchGitHub(ctx context.Context, pkgName string, conf *installerconf.Conf, shallow bool) error {
+	owner, repo := conf.Owner, conf.Repo
+	if owner == "" || repo == "" {
+		return fmt.Errorf("missing owner or repo")
+	}
+
+	d, err := rawcache.Open(filepath.Join(wc.RawDir, pkgName))
+	if err != nil {
+		return err
+	}
+
+	tagPrefix := conf.TagPrefix
+	for batch, err := range github.Fetch(ctx, wc.Client, owner, repo, wc.Auth) {
+		if err != nil {
+			return fmt.Errorf("github %s/%s: %w", owner, repo, err)
+		}
+		for _, rel := range batch {
+			if rel.Draft {
+				continue
+			}
+			tag := rel.TagName
+			if tagPrefix != "" && !strings.HasPrefix(tag, tagPrefix) {
+				continue
+			}
+			data, _ := json.Marshal(rel)
+			d.Merge(tag, data)
+		}
+		if shallow {
+			break
+		}
+	}
+	return nil
+}
+
+func (wc *WebiCache) fetchNodeDist(ctx context.Context, pkgName string, conf *installerconf.Conf) error {
+	baseURL := conf.BaseURL
+	if baseURL == "" {
+		return fmt.Errorf("missing url")
+	}
+
+	d, err := rawcache.Open(filepath.Join(wc.RawDir, pkgName))
+	if err != nil {
+		return err
+	}
+
+	// Fetch from primary URL. Tag with "official/" prefix so unofficial
+	// entries for the same version don't overwrite.
+	for batch, err := range nodedist.Fetch(ctx, wc.Client, baseURL) {
+		if err != nil {
+			return err
+		}
+		for _, entry := range batch {
+			data, _ := json.Marshal(entry)
+			d.Merge("official/"+entry.Version, data)
+		}
+	}
+
+	// Fetch from unofficial URL if configured (e.g. Node.js unofficial builds
+	// which add musl, riscv64, loong64 targets).
+	if unofficialURL := conf.Extra["unofficial_url"]; unofficialURL != "" {
+		for batch, err := range nodedist.Fetch(ctx, wc.Client, unofficialURL) {
+			if err != nil {
+				log.Printf("warning: %s unofficial fetch: %v", pkgName, err)
+				break
+			}
+			for _, entry := range batch {
+				data, _ := json.Marshal(entry)
+				d.Merge("unofficial/"+entry.Version, data)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (wc *WebiCache) fetchGitTag(ctx context.Context, pkgName string, conf *installerconf.Conf, shallow bool) error {
+	gitURL := conf.BaseURL
+	if gitURL == "" {
+		return fmt.Errorf("missing url")
+	}
+
+	d, err := rawcache.Open(filepath.Join(wc.RawDir, pkgName))
+	if err != nil {
+		return err
+	}
+
+	repoDir := filepath.Join(wc.RawDir, "_repos")
+	os.MkdirAll(repoDir, 0o755)
+
+	for batch, err := range gittag.Fetch(ctx, gitURL, repoDir) {
+		if err != nil {
+			return err
+		}
+		for _, entry := range batch {
+			tag := entry.Version
+			if tag == "" {
+				tag = "HEAD-" + entry.CommitHash
+			}
+			data, _ := json.Marshal(entry)
+			d.Merge(tag, data)
+		}
+		if shallow {
+			break
+		}
+	}
+	return nil
+}
+
+func (wc *WebiCache) fetchGitea(ctx context.Context, pkgName string, conf *installerconf.Conf, shallow bool) error {
+	baseURL, owner, repo := conf.BaseURL, conf.Owner, conf.Repo
+	if baseURL == "" || owner == "" || repo == "" {
+		return fmt.Errorf("missing base_url, owner, or repo")
+	}
+
+	d, err := rawcache.Open(filepath.Join(wc.RawDir, pkgName))
+	if err != nil {
+		return err
+	}
+
+	for batch, err := range gitea.Fetch(ctx, wc.Client, baseURL, owner, repo, nil) {
+		if err != nil {
+			return err
+		}
+		for _, rel := range batch {
+			if rel.Draft {
+				continue
+			}
+			data, _ := json.Marshal(rel)
+			d.Merge(rel.TagName, data)
+		}
+		if shallow {
+			break
+		}
+	}
+	return nil
+}
+
+func fetchChromeDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
+	d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
+	if err != nil {
+		return err
+	}
+
+	for batch, err := range chromedist.Fetch(ctx, client) {
+		if err != nil {
+			return fmt.Errorf("chromedist: %w", err)
+		}
+		for _, ver := range batch {
+			data, _ := json.Marshal(ver)
+			d.Merge(ver.Version, data)
+		}
+	}
+	return nil
+}
+
+func fetchFlutterDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
+	d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
+	if err != nil {
+		return err
+	}
+
+	for batch, err := range flutterdist.Fetch(ctx, client) {
+		if err != nil {
+			return fmt.Errorf("flutterdist: %w", err)
+		}
+		for _, rel := range batch {
+			// Key by version+channel+os for uniqueness.
+			key := rel.Version + "-" + rel.Channel + "-" + rel.OS
+			data, _ := json.Marshal(rel)
+			d.Merge(key, data)
+		}
+	}
+	return nil
+}
+
+func fetchGolang(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
+	d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
+	if err != nil {
+		return err
+	}
+
+	for batch, err := range golang.Fetch(ctx, client) {
+		if err != nil {
+			return fmt.Errorf("golang: %w", err)
+		}
+		for _, rel := range batch {
+			data, _ := json.Marshal(rel)
+			d.Merge(rel.Version, data)
+		}
+	}
+	return nil
+}
+
+func fetchGPGDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
+	d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
+	if err != nil {
+		return err
+	}
+
+	for batch, err := range gpgdist.Fetch(ctx, client) {
+		if err != nil {
+			return fmt.Errorf("gpgdist: %w", err)
+		}
+		for _, entry := range batch {
+			data, _ := json.Marshal(entry)
+			d.Merge(entry.Version, data)
+		}
+	}
+	return nil
+}
+
+func fetchHashiCorp(ctx context.Context, client *http.Client, rawDir, pkgName string, conf *installerconf.Conf) error {
+	product := conf.Repo
+	if product == "" {
+		product = pkgName
+	}
+
+	d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
+	if err != nil {
+		return err
+	}
+
+	for idx, err := range hashicorp.Fetch(ctx, client, product) {
+		if err != nil {
+			return fmt.Errorf("hashicorp %s: %w", product, err)
+		}
+		for ver, vdata := range idx.Versions {
+			data, _ := json.Marshal(vdata)
+			d.Merge(ver, data)
+		}
+	}
+	return nil
+}
+
+func fetchITerm2Dist(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
+	d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
+	if err != nil {
+		return err
+	}
+
+	for batch, err := range iterm2dist.Fetch(ctx, client) {
+		if err != nil {
+			return fmt.Errorf("iterm2dist: %w", err)
+		}
+		for _, entry := range batch {
+			key := entry.Version
+			if entry.Channel == "beta" {
+				key += "-beta"
+			}
+			data, _ := json.Marshal(entry)
+			d.Merge(key, data)
+		}
+	}
+	return nil
+}
+
+func fetchJuliaDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
+	d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
+	if err != nil {
+		return err
+	}
+
+	for batch, err := range juliadist.Fetch(ctx, client) {
+		if err != nil {
+			return fmt.Errorf("juliadist: %w", err)
+		}
+		for _, rel := range batch {
+			data, _ := json.Marshal(rel)
+			d.Merge(rel.Version, data)
+		}
+	}
+	return nil
+}
+
+func fetchMariaDBDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
+	d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
+	if err != nil {
+		return err
+	}
+
+	for batch, err := range mariadbdist.Fetch(ctx, client) {
+		if err != nil {
+			return fmt.Errorf("mariadbdist: %w", err)
+		}
+		for _, rel := range batch {
+			data, _ := json.Marshal(rel)
+			d.Merge(rel.ReleaseID, data)
+		}
+	}
+	return nil
+}
+
+func fetchZigDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
+	d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
+	if err != nil {
+		return err
+	}
+
+	for batch, err := range zigdist.Fetch(ctx, client) {
+		if err != nil {
+			return fmt.Errorf("zigdist: %w", err)
+		}
+		for _, rel := range batch {
+			data, _ := json.Marshal(rel)
+			d.Merge(rel.Version, data)
+		}
+	}
+	return nil
+}

comrak/releases.conf

@@ -0,0 +1 @@
+github_releases = kivikakk/comrak

comrak/releases.js

@@ -1,40 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'kivikakk';
-var repo = 'comrak';
-
-var ODDITIES = ['-musleabihf.1-'];
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    let builds = [];
-
-    loopBuilds: for (let build of all.releases) {
-      let isOddity;
-      for (let oddity of ODDITIES) {
-        isOddity = build.name.includes(oddity);
-        if (isOddity) {
-          break;
-        }
-      }
-      if (isOddity) {
-        continue;
-      }
-
-      builds.push(build);
-    }
-
-    all.releases = builds;
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    all.releases = all.releases.slice(0, 10);
-    //console.info(JSON.stringify(all));
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

crabz/releases.conf

@@ -0,0 +1 @@
+github_releases = sstadick/crabz

crabz/releases.js

@@ -1,31 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'sstadick';
-var repo = 'crabz';
-
-module.exports = async function () {
-  let all = await github(null, owner, repo);
-
-  let releases = [];
-  for (let rel of all.releases) {
-    let isSrc = rel.download.includes('-src.');
-    if (isSrc) {
-      continue;
-    }
-
-    releases.push(rel);
-  }
-  all.releases = releases;
-
-  return all;
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    // just select the first 5 for demonstration
-    all.releases = all.releases.slice(0, 5);
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

curlie/releases.conf

@@ -0,0 +1 @@
+github_releases = rs/curlie

curlie/releases.js

@@ -1,21 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'rs';
-var repo = 'curlie';
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    all._names = ['curlie', 'curl-httpie'];
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    all.releases = all.releases.slice(0, 10);
-    //console.info(JSON.stringify(all));
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

dashcore/releases.conf

@@ -0,0 +1 @@
+github_releases = dashpay/dash

dashcore/releases.js

@@ -1,31 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'dashpay';
-var repo = 'dash';
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    all.releases.forEach(function (rel) {
-      if (rel.name.includes('osx64')) {
-        rel.os = 'macos';
-      }
-
-      if (rel.version.startsWith('v')) {
-        rel._version = rel.version.slice(1);
-      }
-    });
-
-    all._names = ['dashd', 'dashcore'];
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    // just select the first 5 for demonstration
-    all.releases = all.releases.slice(0, 5);
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

dashd/releases.conf

@@ -0,0 +1 @@
+alias_of = dashcore

dashd/releases.js

@@ -1,3 +0,0 @@
-'use strict';
-
-module.exports = require('../dashcore/releases.js');

dashmsg/releases.conf

@@ -0,0 +1 @@
+github_releases = dashhive/dashmsg

dashmsg/releases.js

@@ -1,18 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'dashhive';
-var repo = 'dashmsg';
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

delta/releases.conf

@@ -0,0 +1 @@
+github_releases = dandavison/delta

delta/releases.js

@@ -1,20 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'dandavison';
-var repo = 'delta';
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    // just select the first 15 for demonstration
-    all.releases = all.releases.slice(0, 15);
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

deno/releases.conf

@@ -0,0 +1 @@
+github_releases = denoland/deno

deno/releases.js

@@ -1,42 +0,0 @@
-'use strict';
-
-var path = require('path');
-
-var github = require('../_common/github.js');
-var owner = 'denoland';
-var repo = 'deno';
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    // remove checksums and .deb
-    all.releases = all.releases
-      .filter(function (rel) {
-        let isMeta = rel.name.endsWith('.d.ts');
-        if (isMeta) {
-          return false;
-        }
-
-        return true;
-      })
-      .map(function (rel) {
-        var ext;
-        if (!rel.name.match(rel.version)) {
-          ext = path.extname(rel.name);
-          rel.filename =
-            rel.name.slice(0, rel.name.length - ext.length) +
-            '-' +
-            rel.version +
-            ext;
-        }
-        return rel;
-      });
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

docs/installer-patterns.md

@@ -0,0 +1,121 @@
+# Installer Archive Patterns
+
+Every package falls into one of these archive structure patterns. When writing
+or modifying an `install.sh`, identify the pattern first — it determines the
+extraction and installation strategy.
+
+## Pattern A: Bare Binary in Archive
+
+Archive contains the binary (and maybe LICENSE/README) at the top level.
+
+Examples: awless, caddy, cilium, curlie, dashmsg, deno, dotenv, dotenv-linter,
+ffuf, fzf, gitdeploy, gprox, grype, hugo, hugo-extended, k9s, keypairs, koji,
+lf, monorel, ots, runzip, sclient, sqlc, sqlpkg, sttr, terraform, uuidv7, xcaddy
+
+Install: extract, move binary to `~/.local/opt/{pkg}-{ver}/bin/{binary}`, symlink.
+
+## Pattern B: Subdirectory with Binary Only
+
+Archive contains a version-named directory wrapping the binary and docs.
+
+Examples: delta, hexyl, kubectx, kubens, shellcheck, trip, xsv
+
+Typical directory naming: `{tool}-{ver}-{triplet}/`
+
+Install: extract, find binary in subdirectory, move to opt, symlink.
+
+Special cases:
+- `pathman`: bare binary named with full release tag (needs rename)
+- `yq`: binary named with platform suffix `yq_linux_amd64` (needs rename)
+
+## Pattern C: Binary + Completions + Man Pages
+
+Archive includes shell completions and/or man pages alongside the binary.
+
+| Package | Completions Dir | Man Page |
+|---------|----------------|----------|
+| bat | `autocomplete/` | `bat.1` |
+| fd | `autocomplete/{fd.bash,.fish,_fd}` | `fd.1` |
+| goreleaser | `completions/{.bash,.fish,.zsh}` | `manpages/*.1.gz` |
+| lsd | `autocomplete/{lsd.bash-completion,.fish,_lsd}` | `lsd.1` |
+| rg | `complete/{rg.bash,.fish,_rg}` | `doc/rg.1` |
+| sd | `completions/{sd.bash,.fish,_sd}` | `sd.1` |
+| watchexec | `completions/{bash,fish,zsh}` | `watchexec.1` |
+| zoxide | `completions/{zoxide.bash,.fish,_zoxide}` | `man/man1/zoxide*.1` |
+
+Install: extract, install binary, install completions to standard dirs, install
+man pages. Completion naming varies: `autocomplete/`, `completions/`, `complete/`.
+
+## Pattern D: Binary + Libraries
+
+Complex packages that bundle shared libraries.
+
+| Package | Layout |
+|---------|--------|
+| ollama (Linux) | `bin/ollama` + `lib/ollama/{cuda_v12,cuda_v13,vulkan}/` |
+| pg/postgres/psql | `bin/psql` + `lib/{libpq,libz,...}.so` + `include/` |
+| sass | `dart-sass/sass` (wrapper) + `dart-sass/src/{dart,sass.snapshot}` |
+| syncthing | `syncthing-{triplet}-{ver}/syncthing` + `etc/{systemd,...}/` |
+| xz | `xz-{ver}-{triplet}/xz` + `xz-{ver}-{triplet}/unxz` |
+
+Install: extract entire directory tree into opt, symlink binary.
+
+## Pattern E: FHS-like Layout (bin/ + share/)
+
+Archive already follows standard layout.
+
+| Package | Layout |
+|---------|--------|
+| gh | `gh_{ver}_{os}_{arch}/bin/gh` + `share/man/man1/*.1` |
+| pandoc | `pandoc-{ver}/bin/{pandoc,...}` + `share/man/man1/*.1.gz` |
+
+Install: extract directly into opt (already correct layout).
+
+## Pattern G: Full SDK/Toolchain
+
+Self-contained toolchain with compiler, runtime, standard library.
+
+| Package | Layout |
+|---------|--------|
+| cmake | `cmake-{ver}-{os}-{arch}/bin/{cmake,ctest,...}` + `share/` + `man/` |
+| tinygo | `tinygo/bin/tinygo` + `tinygo/src/` + `tinygo/targets/` |
+| go | `go/bin/{go,gofmt}` + `go/src/` + `go/pkg/` |
+| zig | `zig-{os}-{arch}-{ver}/zig` + `lib/` |
+| flutter | `flutter/bin/flutter` + full SDK |
+| julia | `julia-{ver}/bin/julia` + full SDK |
+| node | `node-{ver}-{os}-{arch}/bin/{node,npm,npx}` + `lib/` |
+
+Install: extract entire tree into `~/.local/opt/{pkg}-{ver}/`, symlink `bin/*`.
+
+## Pattern H: .NET Runtime Bundle
+
+Flat archive with hundreds of DLLs.
+
+Example: pwsh — `pwsh` binary + `*.dll` + locale dirs
+
+Install: extract entire directory into opt, symlink primary binary.
+
+## Pattern I: Multi-Binary Distribution
+
+Archive contains multiple related binaries + libs.
+
+| Package | Layout |
+|---------|--------|
+| dashcore | `dashcore-{ver}/bin/{dashd,dash-cli,...}` + `lib/` + `share/man/` |
+| mutagen | `mutagen` + `mutagen-agents.tar.gz` (embedded agent archive) |
+
+Install: extract into opt, symlink primary binary.
+
+## Format Changes Over Time
+
+Most packages have stable formats. Notable structural changes:
+
+| Package | When | Change |
+|---------|------|--------|
+| sd | 2023 | zip → tar.gz, added completions + man page |
+| ollama | 2025-2026 | bare binary → no GitHub release → tar.zst with lib/ |
+| deno | 2020-2021 | .gz (gzipped binary) → .zip |
+| hugo | 2017-2018 | zip → tar.gz; 2024: macOS → .pkg only |
+| gh | 2024 | darwin: tar.gz → .pkg |
+| sclient | 2023 | tar.gz → tar.xz |
+| watchexec | 2019-2020 | tar.gz → tar.xz |

docs/version-oddities.md

@@ -0,0 +1,74 @@
+# Version & Release Oddities
+
+Non-standard version formats and tag prefixes that affect parsing, sorting,
+and classification. The Go classifier and `internal/lexver` must handle all
+of these.
+
+## Non-Numeric Tag Prefixes
+
+| Package | Raw Tag | Cleaned | Transform |
+|---------|---------|---------|-----------|
+| lf | `r21` | `0.21.0` | `r` prefix → prepend `0.` |
+| bun | `bun-v1.0.0` | `1.0.0` | Strip `bun-` prefix |
+| jq | `jq-1.7` | `1.7` | Strip `jq-` prefix |
+| watchexec | `cli-v1.2.3` | `1.2.3` | Strip `cli-` prefix |
+| ffmpeg | `b6.0` | `6.0` | Strip `b` prefix |
+
+## Underscore-Delimited Tags
+
+| Package | Raw Tag | Cleaned | Transform |
+|---------|---------|---------|-----------|
+| postgres | `REL_17_0` | `17.0` | Strip `REL_`, replace `_` with `.` |
+| psql | `REL_17_0` | `17.0` | Same as postgres |
+
+## Platform Suffix in Version
+
+| Package | Raw Tag | Cleaned | Transform |
+|---------|---------|---------|-----------|
+| git (Windows) | `2.41.0.windows.1` | `2.41.0` | Strip `.windows.N` suffix |
+
+## 4-Part Versions
+
+| Package | Example | Notes |
+|---------|---------|-------|
+| chromedriver | `121.0.6120.0` | Google Chrome's versioning |
+| gpg | `2.2.19.0` | 4th segment is build metadata |
+
+## Date-Based Versions
+
+| Package | Notes |
+|---------|-------|
+| atomicparsley | Date-based version strings |
+
+## Complex Pre-Release Formats
+
+| Package | Example | Notes |
+|---------|---------|-------|
+| flutter | `2.3.0-16.0.pre` | Extra dots and numeric segments |
+| iterm2 | `iTerm2_3_5_0beta17` | Underscores, beta attached → `3.5.0-beta17` |
+
+## Channel Detection
+
+- Node.js: odd major = "current" not LTS (v15, v17, v19, v21, v23)
+- Go: `go` prefix stripped (`go1.23.6` → `1.23.6`)
+- Terraform: `-alpha`, `-beta`, `-rc` suffixes → beta channel
+
+## Directory Symlinks (Aliases)
+
+These are directory-level symlinks. They share all files (including
+releases.conf) with their target automatically.
+
+```
+msvc-runtime    → vcruntime
+msvcruntime     → vcruntime
+rust.vim        → vim-rust
+vc-redist       → vcruntime
+vc-runtime      → vcruntime
+vc_redist       → vcruntime
+vcredist        → vcruntime
+vcruntime140    → vcruntime
+vim-essential   → vim-essentials
+vim-mouse       → vim-gui
+vps-myip        → myip
+xcode-cli       → commandlinetools
+```

dotenv-linter/releases.conf

@@ -0,0 +1 @@
+github_releases = dotenv-linter/dotenv-linter

dotenv-linter/releases.js

@@ -1,20 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'dotenv-linter';
-var repo = 'dotenv-linter';
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    // just select the first 5 for demonstration
-    all.releases = all.releases.slice(0, 5);
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

dotenv/releases.conf

@@ -0,0 +1 @@
+github_releases = therootcompany/dotenv

dotenv/releases.js

@@ -1,18 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'therootcompany';
-var repo = 'dotenv';
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    console.info(JSON.stringify(all));
-  });
-}

duckdns.sh/releases.conf

@@ -0,0 +1,2 @@
+github_sources = BeyondCodeBootcamp/DuckDNS.sh
+git_url = https://github.com/BeyondCodeBootcamp/DuckDNS.sh.git

duckdns.sh/releases.js

@@ -1,22 +0,0 @@
-'use strict';
-
-let Releases = module.exports;
-
-let GitHubSource = require('../_common/github-source.js');
-let owner = 'BeyondCodeBootcamp';
-let repo = 'DuckDNS.sh';
-
-Releases.latest = async function () {
-  let all = await GitHubSource.getDistributables({ owner, repo });
-  for (let pkg of all.releases) {
-    pkg.os = 'posix_2017';
-  }
-  return all;
-};
-
-if (module === require.main) {
-  Releases.latest().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

fd/releases.conf

@@ -0,0 +1 @@
+github_releases = sharkdp/fd

fd/releases.js

@@ -1,31 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'sharkdp';
-var repo = 'fd';
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    let builds = [];
-
-    for (let build of all.releases) {
-      if (build.name === 'fd') {
-        continue;
-      }
-
-      builds.push(build);
-    }
-
-    all.releases = builds;
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    all.releases = all.releases.slice(0, 10);
-    //console.info(JSON.stringify(all));
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

ffmpeg/releases.conf

@@ -0,0 +1,4 @@
+source = ffmpegdist
+github_releases = eugeneware/ffmpeg-static
+asset_filter = ffmpeg
+version_prefix = b

ffmpeg/releases.js

@@ -1,45 +0,0 @@
-'use strict';
-
-var path = require('path');
-
-var github = require('../_common/github.js');
-var owner = 'eugeneware';
-var repo = 'ffmpeg-static';
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    all.releases = all.releases
-      .filter(function (rel) {
-        let isFfmpeg = rel.name.includes('ffmpeg');
-        if (!isFfmpeg) {
-          return;
-        }
-
-        // remove README and LICENSE
-        return !['.README', '.LICENSE'].includes(path.extname(rel.name));
-      })
-      .map(function (rel) {
-        rel.version = rel.version.replace(/^b/, '');
-
-        if (/win32/.test(rel.name)) {
-          rel.os = 'windows';
-          rel.ext = 'exe';
-        }
-        if (/ia32/.test(rel.name)) {
-          rel.arch = '386';
-        } else if (/x64/.test(rel.name)) {
-          rel.arch = 'amd64';
-        }
-
-        return rel;
-      });
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    console.info(JSON.stringify(all));
-  });
-}

ffuf/releases.conf

@@ -0,0 +1 @@
+github_releases = ffuf/ffuf

ffuf/releases.js

@@ -1,20 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'ffuf';
-var repo = 'ffuf';
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    // just select the first 5 for demonstration
-    all.releases = all.releases.slice(0, 5);
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

fish/releases.conf

@@ -0,0 +1,2 @@
+github_releases = fish-shell/fish-shell
+exclude = bundledpcre fish-static OpenBeta

fish/releases.js

@@ -1,39 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'fish-shell';
-var repo = 'fish-shell';
-
-var ODDITIES = ['bundledpcre'];
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    all.releases = all.releases
-      .map(function (rel) {
-        for (let oddity of ODDITIES) {
-          let isOddity = rel.name.includes(oddity);
-          if (isOddity) {
-            return;
-          }
-        }
-
-        // We can extract the macos bins from the .app
-        if (/\.app\.zip$/.test(rel.name)) {
-          rel.os = 'macos';
-          rel.arch = 'amd64';
-          return rel;
-        }
-      })
-      .filter(Boolean);
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    // just select the first 5 for demonstration
-    all.releases = all.releases.slice(0, 5);
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

flutter/releases.conf

@@ -0,0 +1 @@
+source = flutterdist

flutter/releases.js

@@ -1,132 +0,0 @@
-'use strict';
-
-let Fetcher = require('../_common/fetcher.js');
-
-let FLUTTER_OSES = ['macos', 'linux', 'windows'];
-
-/**
- * stable, beta, dev
- * @type {Object.<String, Boolean>}
- */
-let channelMap = {};
-
-// This can be spot-checked against
-// https://docs.flutter.dev/release/archive?tab=windows
-
-// The release URLs are
-// - https://storage.googleapis.com/flutter_infra_release/releases/releases_macos.json
-// - https://storage.googleapis.com/flutter_infra_release/releases/releases_linux.json
-// - https://storage.googleapis.com/flutter_infra_release/releases/releases_windows.json
-// The old release URLs are
-// - https://storage.googleapis.com/flutter_infra/releases/releases_macos.json
-// - https://storage.googleapis.com/flutter_infra/releases/releases_linux.json
-// - https://storage.googleapis.com/flutter_infra/releases/releases_windows.json
-
-// The data looks like
-// {
-//   "base_url": "https://storage.googleapis.com/flutter_infra/releases",
-//   "current_release": {
-//     "beta": "b22742018b3edf16c6cadd7b76d9db5e7f9064b5",
-//     "dev": "fa5883b78e566877613ad1ccb48dd92075cb5c23",
-//     "stable": "02c026b03cd31dd3f867e5faeb7e104cce174c5f"
-//   },
-//   "releases": [
-//     {
-//       "hash": "fa5883b78e566877613ad1ccb48dd92075cb5c23",
-//       "channel": "dev",
-//       "version": "2.3.0-16.0.pre",
-//       "release_date": "2021-05-27T23:58:47.683121Z",
-//       "archive": "dev/macos/flutter_macos_2.3.0-16.0.pre-dev.zip",
-//       "sha256": "f572b42d36714e6c58a3ed170b93bb414e2ced3ca4bde5094fbe18061cbcba6c"
-//     },
-//     {
-//       "hash": "02c026b03cd31dd3f867e5faeb7e104cce174c5f",
-//       "channel": "stable",
-//       "version": "2.2.1",
-//       "release_date": "2021-05-27T23:06:07.243882Z",
-//       "archive": "stable/macos/flutter_macos_2.2.1-stable.zip",
-//       "sha256": "6373d39ec563c337600baf42a42b258420208e4523d85479373e113d61d748df"
-//     },
-//     {
-//       "hash": "b22742018b3edf16c6cadd7b76d9db5e7f9064b5",
-//       "channel": "beta",
-//       "version": "2.2.0",
-//       "release_date": "2021-05-19T21:14:59.281482Z",
-//       "archive": "beta/macos/flutter_macos_2.2.0-beta.zip",
-//       "sha256": "31ab530e708f8d1274712211253a27a4ce7d676f139d30f2ec021df22382f052"
-//     }
-//   ]
-// }
-
-/**
- * @typedef BuildInfo
- * @prop {String} version
- * @prop {String} [_version]
- * @prop {Boolean} lts
- * @prop {String} channel
- * @prop {String} date
- * @prop {String} download
- * @prop {String} [_filename]
- */
-
-module.exports = async function () {
-  let all = {
-    download: '',
-    /** @type {Array<BuildInfo>} */
-    releases: [],
-    /** @type {Array<String>} */
-    channels: [],
-  };
-
-  for (let osname of FLUTTER_OSES) {
-    let resp;
-    try {
-      let url = `https://storage.googleapis.com/flutter_infra_release/releases/releases_${osname}.json`;
-      resp = await Fetcher.fetch(url, {
-        headers: { Accept: 'application/json' },
-      });
-    } catch (e) {
-      /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error
-      let err = e;
-      if (err.code === 'E_FETCH_RELEASES') {
-        err.message = `failed to fetch 'flutter' release data for ${osname}: ${err.response.status} ${err.response.body}`;
-      }
-      throw e;
-    }
-    let data = JSON.parse(resp.body);
-
-    let osBaseUrl = data.base_url;
-    let osReleases = data.releases;
-
-    for (let asset of osReleases) {
-      if (!channelMap[asset.channel]) {
-        channelMap[asset.channel] = true;
-      }
-
-      all.releases.push({
-        version: asset.version,
-        _version: `${asset.version}-${asset.channel}`,
-        lts: false,
-        channel: asset.channel,
-        date: asset.release_date.replace(/T.*/, ''),
-        download: `${osBaseUrl}/${asset.archive}`,
-        _filename: asset.archive,
-      });
-    }
-  }
-
-  all.channels = Object.keys(channelMap);
-
-  // note: versions have a waterfall relationship with channels:
-  // - a release that is in beta today may become stable tomorrow
-  // - semver prereleases are either beta or dev
-
-  return all;
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all.releases = all.releases.slice(25);
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

fzf/releases.conf

@@ -0,0 +1 @@
+github_releases = junegunn/fzf

fzf/releases.js

@@ -1,20 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'junegunn';
-var repo = 'fzf';
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    all.releases = all.releases.slice(0, 10);
-    //console.info(JSON.stringify(all));
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

gh/releases.conf

@@ -0,0 +1 @@
+github_releases = cli/cli

gh/releases.js

@@ -1,20 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'cli';
-var repo = 'cli';
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    // just select the first 5 for demonstration
-    all.releases = all.releases.slice(0, 5);
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

git/releases.conf

@@ -0,0 +1,4 @@
+github_releases = git-for-windows/git
+asset_filter = MinGit
+exclude = busybox
+variants = installer

git/releases.js

@@ -1,33 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'git-for-windows';
-var repo = 'git';
-
-module.exports = function () {
-  // TODO support mac and linux tarballs
-  return github(null, owner, repo).then(function (all) {
-    // See https://github.com/git-for-windows/git/wiki/MinGit
-    // also consider https://github.com/git-for-windows/git/wiki/Silent-or-Unattended-Installation
-    all.releases = all.releases
-      .filter(function (rel) {
-        rel.os = 'windows';
-        rel._version = rel.version.replace(/\.windows.1.*/, '');
-        rel._version = rel._version.replace(/\.windows(\.\d)/, '$1');
-        return (
-          /MinGit/i.test(rel.name || rel.download) &&
-          !/busybox/i.test(rel.name || rel.download)
-        );
-      })
-      .slice(0, 20);
-    all._names = ['MinGit', 'git'];
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    console.info(JSON.stringify(all, null, 2));
-  });
-}

gitdeploy/releases.conf

@@ -0,0 +1 @@
+github_releases = therootcompany/gitdeploy

gitdeploy/releases.js

@@ -1,18 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'therootcompany';
-var repo = 'gitdeploy';
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    console.info(JSON.stringify(all));
-  });
-}

gitea/releases.conf

@@ -0,0 +1,2 @@
+github_releases = go-gitea/gitea
+exclude = -src- -docs-

gitea/releases.js

@@ -1,34 +0,0 @@
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'go-gitea';
-var repo = 'gitea';
-
-var ODDITIES = ['-gogit-', '-docs-'];
-
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
-    // remove checksums and .deb
-    all.releases = all.releases.filter(function (rel) {
-      for (let oddity of ODDITIES) {
-        let isOddity = rel.name.includes(oddity);
-        if (isOddity) {
-          return false;
-        }
-      }
-
-      return true;
-    });
-
-    // "windows-4.0" as a nod to Windows NT  ¯\_(ツ)_/¯
-    all._names = ['gitea', '-4.0-'];
-    return all;
-  });
-};
-
-if (module === require.main) {
-  module.exports().then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    console.info(JSON.stringify(all));
-  });
-}

gnupg/releases.conf

@@ -0,0 +1 @@
+alias_of = gpg

go.mod

@@ -0,0 +1,15 @@
+module github.com/webinstall/webi-installers
+
+go 1.26.1
+
+require (
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
+	github.com/jackc/pgx/v5 v5.8.0 // indirect
+	github.com/jackc/puddle/v2 v2.2.2 // indirect
+	github.com/joho/godotenv v1.5.1 // indirect
+	github.com/jszwec/csvutil v1.10.0 // indirect
+	github.com/therootcompany/golib/http/middleware/v2 v2.0.1 // indirect
+	golang.org/x/sync v0.17.0 // indirect
+	golang.org/x/text v0.29.0 // indirect
+)

go.sum

@@ -0,0 +1,25 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo=
+github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw=
+github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
+github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/jszwec/csvutil v1.10.0 h1:upMDUxhQKqZ5ZDCs/wy+8Kib8rZR8I8lOR34yJkdqhI=
+github.com/jszwec/csvutil v1.10.0/go.mod h1:/E4ONrmGkwmWsk9ae9jpXnv9QT8pLHEPcCirMFhxG9I=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/therootcompany/golib/http/middleware/v2 v2.0.1 h1:VNKpHcwyEW7cMct7/eO4fyrxwIQk2ycb6juVXSPs2Sk=
+github.com/therootcompany/golib/http/middleware/v2 v2.0.1/go.mod h1:g5gb9qBidw74nW6/mwIauTKMpOKchiN2l0gt5qzJ2aQ=
+golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
+golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
+golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

go/releases.conf

@@ -0,0 +1 @@
+source = golang