WIP: feat(webi): show supported channels

.github/ISSUE_TEMPLATE/new_installer.md

@@ -35,18 +35,15 @@ info, and doing a find and replace on a few file system path names.
    ```
 2. Copy the example template and update with info from Official Releases:
    <https://github.com/___CHANGE/ME___/releases>
-
    ```bash
    rsync -av _example/ CHANGE-ME/
    ```
-
    - [ ] update `CHANGE-ME/release.js` to use the official repo
    - [ ] Learn how `CHANGE-ME` unpacks (i.e. as a single file? as a .tar.gz? as
          a .tar.gz with a folder named CHANGE-ME?)
    - [ ] find and replace to change the name
      - [ ] update `CHANGE-ME/install.sh` (see `bat` and `jq` as examples)
      - [ ] update `CHANGE-ME/install.ps1` (see `bat` and `jq` as examples)
-
 3. Needs an updated tagline and cheat sheet
    - [ ] update `CHANGE-ME/README.md`
      - [ ] official URL

.github/workflows/node.js.yml

@@ -23,15 +23,10 @@ jobs:
           sh ./_scripts/install-ci-deps
           echo "${HOME}/.local/bin" >> $GITHUB_PATH
           echo "${HOME}/.local/opt/pwsh" >> $GITHUB_PATH
-      - run: |
-          git submodule init
-          git submodule update
       - run: shfmt --version
       - run: shellcheck -V
       - run: node --version
       - run: npm run fmt
       - run: npm ci
       - run: npm run lint
-      - env:
-          GITHUB_TOKEN: ${{ github.token }}
-        run: npm run test
+      - run: npm run test

.gitignore

@@ -1,40 +1,7 @@
-# generated artifacts
+.env
+node_modules
 install-*.sh
 install-*.bat
 install-*.ps1
-
-# Go build outputs (from go run/build in repo root)
-/classify
-/e2etest
-/fetchraw
-/inspect
-/uaparse
-/webicached
-/zigtest
-/distributables.csv
-
-# local config
-.env.*
-*.env
-.env
-!example.env
-
-# caches
-_cache/
-node_modules/
-
-# temporary & backup files
 .*.sw*
 *.bak
-*.bak.*
-
-# agent session files
-agents/
-
-# other
-.DS_Store
-desktop.ini
-.directory
-LIVE_cache
-/webid
-bin/

.jshintrc

@@ -1,7 +1,4 @@
 {
-  "globals": {
-    "AbortController": false
-  },
   "browser": true,
   "node": true,
   "esversion": 11,

AGENTS.md

@@ -1,492 +0,0 @@
-# Webi Installers — Agent Guide
-
-Webi installs dev tools to `~/.local/` without sudo. Each installer is a small
-package of 3-4 files. This guide tells you how to create and modify them.
-
-## Why Webi Exists
-
-Webi makes tool installation trivially repeatable for people who aren't
-sysadmins — freelance clients, junior devs, anyone who shouldn't have to care
-about PATH, permissions, or platform differences. Three things matter:
-
-1. **Install without friction.** No sudo, no manual PATH edits, no "necessary
-   but unimportant" steps leaking into the experience.
-2. **Know where things are.** The Files section tells you exactly what got
-   created or modified. Nothing should be mysterious.
-3. **Copy-paste recipes.** The cheat sheet is what you'd send someone less
-   experienced than yourself instead of a project's full README — scannable,
-   concrete, easy to reference by name.
-
-## Quick Start: Adding a New Installer
-
-1. Identify the **package type** (see [Categories](#categories) below)
-2. Find an existing installer of the same type to use as a template
-3. Create `<name>/releases.js`, `install.sh`, `install.ps1`, `README.md`
-4. Test with the command in [Testing releases.js](#testing-releasesjs)
-5. Run formatters before committing (see [Code Style](#code-style))
-
-## Directory Layout
-
-```
-<package-name>/
-  README.md          # YAML frontmatter + docs
-  releases.js        # Fetches release metadata (Node.js)
-  install.sh         # POSIX shell installer (macOS/Linux)
-  install.ps1        # PowerShell installer (Windows) — optional
-```
-
-Key infrastructure directories (do not modify without good reason):
-
-- `_webi/` — bootstrap templates, `normalize.js` (auto-detects OS/arch/ext from
-  filenames)
-- `_common/` — shared JS: `github.js`, `githubish.js`, `gitea.js`, `fetcher.js`
-- `_example/` — canonical template for new packages
-- `_examples/` — specialized templates (goreleaser, xz-compressed)
-
-## Categories
-
-Ref: <https://github.com/webinstall/webi-installers/issues/412>
-
-| Type  | Description                            | Template to copy |
-| ----- | -------------------------------------- | ---------------- |
-| `bin` | Single binary in tar/zip               | `koji`, `delta`  |
-| `bin` | Single bare binary (no archive)        | `arc`, `shfmt`   |
-| `bin` | Goreleaser-style archives              | `keypairs`       |
-| 📦    | Self-contained package (bin/man/share) | `node`, `go`     |
-| 📂    | Multiple binaries/scripts              | `pg`             |
-| 🔗    | Alias/redirect to another package      | `ripgrep` → `rg` |
-| 📝    | Bespoke / custom install               | `rustlang`       |
-
-## releases.js
-
-Fetches release metadata and returns a normalized object. Most packages use
-GitHub releases:
-
-```js
-'use strict';
-
-var github = require('../_common/github.js');
-var owner = 'OWNER';
-var repo = 'REPO';
-
-let Releases = module.exports;
-
-Releases.latest = async function () {
-  let all = await github(null, owner, repo);
-  return all;
-};
-
-Releases.sample = async function () {
-  let normalize = require('../_webi/normalize.js');
-  let all = await Releases.latest();
-  all = normalize(all);
-  all.releases = all.releases.slice(0, 5);
-  return all;
-};
-
-if (module === require.main) {
-  (async function () {
-    let samples = await Releases.sample();
-    console.info(JSON.stringify(samples, null, 2));
-  })();
-}
-```
-
-### Common release transformations
-
-**Strip version prefix** (monorepo or tool-prefixed tags):
-
-```js
-// e.g. "tools/monorel/v0.6.5" → "v0.6.5"
-rel.version = rel.version.replace(/^tools\/monorel\//, '');
-
-// e.g. "cli-v1.2.3" → "v1.2.3"
-rel.version = rel.version.replace(/^cli-/, '');
-```
-
-**Filter releases** (monorepo with multiple tools, or unwanted assets):
-
-```js
-all.releases = all.releases.filter(function (rel) {
-  // Keep only releases for this tool
-  return rel.version.startsWith('tools/monorel/');
-});
-```
-
-Apply transformations inside `Releases.latest`, before returning `all`.
-
-**Available sources** beyond `github.js`:
-
-- `_common/gitea.js` — Gitea servers
-- `_common/git-tag.js` — Git tag listing
-- Custom fetch from any JSON API (see `go/releases.js`, `terraform/releases.js`)
-
-### Testing releases.js
-
-```sh
-node -e "
-  let Releases = require('./<name>/releases.js');
-  Releases.sample().then(function (all) {
-    console.log(JSON.stringify(all, null, 2));
-  });
-"
-```
-
-Verify: versions are clean semver (`0.6.5` not `tools/monorel/v0.6.5`), OS/arch
-detected correctly, download URLs resolve.
-
-## install.sh
-
-POSIX shell (`sh`, not bash). Always wrapped in a function:
-
-```sh
-#!/bin/sh
-# shellcheck disable=SC2034
-
-set -e
-set -u
-
-__init_pkgname() {
-    # These 6 variables are required
-    pkg_cmd_name="cmd"
-
-    pkg_dst_cmd="$HOME/.local/bin/cmd"
-    pkg_dst="$pkg_dst_cmd"
-
-    pkg_src_cmd="$HOME/.local/opt/cmd-v$WEBI_VERSION/bin/cmd"
-    pkg_src_dir="$HOME/.local/opt/cmd-v$WEBI_VERSION"
-    pkg_src="$pkg_src_cmd"
-
-    pkg_install() {
-        mkdir -p "$(dirname "$pkg_src_cmd")"
-        mv ./cmd "$pkg_src_cmd"
-    }
-
-    pkg_get_current_version() {
-        cmd --version 2> /dev/null | head -n 1 | cut -d' ' -f2
-    }
-}
-
-__init_pkgname
-```
-
-### Framework variables available in install.sh
-
-Set by the webi bootstrap (`_webi/package-install.tpl.sh`):
-
-| Variable        | Example             | Description           |
-| --------------- | ------------------- | --------------------- |
-| `WEBI_VERSION`  | `1.2.3`             | Selected version      |
-| `WEBI_PKG_URL`  | `https://...`       | Download URL          |
-| `WEBI_PKG_FILE` | `foo-v1.2.3.tar.gz` | Download filename     |
-| `WEBI_OS`       | `linux`             | Detected OS           |
-| `WEBI_ARCH`     | `amd64`             | Detected architecture |
-| `WEBI_EXT`      | `tar.gz`            | Archive extension     |
-| `WEBI_CHANNEL`  | `stable`            | Release channel       |
-| `PKG_NAME`      | `foo`               | Package name          |
-
-### Override functions
-
-| Function                    | Purpose                                       |
-| --------------------------- | --------------------------------------------- |
-| `pkg_install()`             | **Required.** Move files to `$pkg_src`        |
-| `pkg_get_current_version()` | Parse installed version from command output   |
-| `pkg_post_install()`        | Post-install setup (git config, shell config) |
-| `pkg_done_message()`        | Custom completion message                     |
-| `pkg_link()`                | Override default symlink behavior             |
-| `pkg_pre_install()`         | Custom pre-install logic                      |
-
-### Framework helper functions
-
-| Function                 | Purpose                            |
-| ------------------------ | ---------------------------------- |
-| `webi_download()`        | Download package if not cached     |
-| `webi_extract()`         | Extract archive by extension       |
-| `webi_path_add <dir>`    | Add to PATH via envman             |
-| `webi_link()`            | Create versioned symlinks          |
-| `webi_check_installed()` | Check if version already installed |
-
-### pkg_install patterns
-
-**Bare binary in archive root:**
-
-```sh
-mv ./cmd "$pkg_src_cmd"
-```
-
-**Binary in a subdirectory (goreleaser-style `cmd-OS-arch/cmd`):**
-
-```sh
-mv ./cmd-*/cmd "$pkg_src_cmd"
-```
-
-**Flexible detection (handles multiple archive layouts):**
-
-```sh
-if test -f ./cmd; then
-    mv ./cmd "$pkg_src_cmd"
-elif test -e ./cmd-*/cmd; then
-    mv ./cmd-*/cmd "$pkg_src_cmd"
-elif test -e ./cmd-*; then
-    mv ./cmd-* "$pkg_src_cmd"
-fi
-```
-
-## install.ps1
-
-PowerShell for Windows. Uses `$Env:` variables. See `_example/install.ps1` for
-the full template. Key differences from install.sh:
-
-- Paths use backslashes, commands end in `.exe`
-- `$Env:USERPROFILE` instead of `$HOME`
-- `Test-Path`, `Move-Item`, `Copy-Item` instead of shell equivalents
-- Downloads go to `$Env:USERPROFILE\Downloads\webi\`
-- Temp work in `.local\tmp`, use `Push-Location`/`Pop-Location`
-- Symlinks done via `Copy-Item` (not actual symlinks)
-
-## README.md
-
-````markdown
----
-title: toolname
-homepage: https://github.com/owner/repo
-tagline: |
-  toolname: A short one-line description.
----
-
-To update or switch versions, run `webi toolname@stable` (or `@v2`, `@beta`,
-etc).
-
-### Files
-
-These are the files that are created and/or modified with this installer:
-
-```text
-~/.config/envman/PATH.env
-~/.local/bin/toolname
-~/.local/opt/toolname-VERSION/bin/toolname
-```
-
-## Cheat Sheet
-
-> `toolname` does X. Brief description.
-
-### How to use toolname
-
-```sh
-toolname --example
-```
-````
-
-Note: **Files goes above Cheat Sheet**, not inside it.
-
-### Cheat Sheet tone and style
-
-Webi cheat sheets are **opinionated quick-reference guides**, not comprehensive
-documentation. Think "colleague's sticky note" — not the project's official
-README.
-
-The tool is the topic, but **the problem is the reason**. Cheat sheets are
-organized around tasks the reader already wants to do — the tool is how they get
-there. Headings reference the tool (the reader came to this page on purpose),
-but the content solves the underlying problem completely:
-
-- "How to reverse proxy to Node" (caddy knowledge, not just node)
-- "How to run a Node app as a System Service" (serviceman knowledge)
-- "How to Enable Secure Remote Postgres Access" (openssl, pg_hba.conf, systemd)
-- "How to manually configure git to use delta" (gitconfig, not delta flags)
-- "How to make fish the default shell in iTerm2" (iTerm2 knowledge, not fish)
-
-The reader's question is "how do I do X?" and the cheat sheet answers it
-completely — including configs, adjacent tools, and platform-specific
-variations. A goreleaser cheat sheet teaches you goreleaser YAML. A postgres
-cheat sheet teaches you pg_hba.conf, openssl certs, and systemd units.
-
-Cheat sheets cross tool boundaries freely. Node's references caddy, serviceman,
-setcap-netbind, GitHub Actions. Postgres references serviceman, openrc, launchd.
-They link to each other's webi pages. The scope is "everything you need to
-accomplish this task," not "everything this one binary does."
-
-They show the actual files and configs that matter — not documentation _about_
-configs, but the configs themselves, copy-pasteable, with inline comments
-explaining the non-obvious parts.
-
-**Guidelines:**
-
-- **Show the 3-5 things someone will actually do**, with copy-pasteable
-  commands. Skip exhaustive flag lists and API docs.
-- **Lead with practical integration.** Show the exact `git config` lines, the
-  exact hook script, the exact shell alias — don't just explain the feature and
-  leave wiring up to the reader.
-- **Skip what they already know.** No need to re-explain what the tool is at
-  length — the tagline and one-liner blockquote handle that. Get to the
-  commands.
-- **Prefer concrete over abstract.** Instead of "you can configure X via a
-  config file", show the config file contents.
-
-## Shell Naming Conventions
-
-**Variables:**
-
-- `ALL_CAPS` — environment variables only (`PATH`, `HOME`, `WEBI_VERSION`)
-- `b_varname` — block-scoped (inside a function, loop, or conditional)
-- `g_varname` — global to the script (and sourced scripts)
-- `a_varname` — function arguments
-
-**Functions and commands:**
-
-- `fn_name` — helper functions (anything other than the script's main/entry
-  function)
-- `cmd_name` — command aliases, e.g. `cmd_curl='curl --fail-with-body -sSL'`
-
-## Code Style
-
-Requires `node`, `shfmt`, `pwsh`, and `pwsh-essentials` (install all via webi).
-Run before committing:
-
-```sh
-npm run fmt         # prettier (JS/MD) + shfmt (sh) + pwsh-fmt (ps1)
-npm run lint        # jshint + shellcheck + pwsh-lint
-```
-
-Commit messages: `feat(<pkg>): add installer`, `fix(<pkg>): update install.sh`,
-`docs(<pkg>): add cheat sheet`.
-
-## Naming Conventions
-
-- The canonical package name is the **command name** you type: `go`, `node`,
-  `rg`
-- The alternate/alias name is the project name: `golang`, `nodejs`, `ripgrep`
-- Package directories are lowercase with hyphens
-
-## Common Pitfalls
-
-- **Monorepo releases**: The GitHub API returns ALL releases for the repo. You
-  must filter in `releases.js` and strip the tag prefix from the version.
-- **No `--version` flag**: Some tools lack version introspection. Comment out
-  `pkg_get_current_version` — webi still works, it just can't skip reinstalls.
-- **normalize.js auto-detection**: OS/arch/ext are guessed from download
-  filenames. If the tool uses non-standard naming, you may need to set `os`,
-  `arch`, or `ext` explicitly in `releases.js`.
-- **Goreleaser archives**: Typically contain a bare binary at the archive root
-  (not nested in a directory). Use `mv ./cmd "$pkg_src_cmd"`.
-
----
-
-## Go Cache Daemon
-
-The Go pipeline (`cmd/webicached`) replaces the Node.js release-fetching code.
-It reads `releases.conf` files, fetches upstream release metadata, classifies
-build assets, and writes to `~/.cache/webi/legacy/` in the format the Node.js server expects.
-
-### Canonical Vocabulary
-
-The classifier MUST use exactly these strings. They match the production API.
-
-**OS**: `macos` (NOT `darwin`), `linux`, `windows`, `freebsd`, `openbsd`,
-`netbsd`, `dragonfly`, `aix`, `illumos`, `plan9`, `solaris`, `posix_2017`
-
-**Arch** — exact equivalences:
-- `amd64` (NOT `x86_64`), `x86` (NOT `i386`/`i686`/`386`)
-- `arm64` (NOT `aarch64`)
-- `armv7l` (NOT `armv7`), `armv6l` (NOT `armv6`)
-- `mipsle` (NOT `mipsel`), `mips64le` (NOT `mips64el`)
-
-**Arch** — compatibility downcasts:
-- `armhf` → `armv7l`, `armv7a` → `armv7l`, `armel` → `arm`
-
-**Arch** — other: `arm`, `ppc64le`, `ppc64`, `loong64`, `riscv64`, `s390x`,
-`mips`, `mips64`
-
-**Libc**: `none` (never empty), `gnu`, `musl`, `msvc`
-
-**Ext**: `tar.gz`, `tar.xz`, `zip`, `exe`, `7z`, `pkg`, `msi`
-(no leading dot; `exe` for bare binaries)
-
-### releases.conf
-
-Each package directory contains a `releases.conf` that tells the daemon where
-to fetch releases. Format is `key = value`, one per line. `#` comments and
-blank lines are ignored.
-
-#### Source types (mutually exclusive — pick one)
-
-```ini
-# GitHub binary releases (most common)
-github_releases = sharkdp/bat
-
-# GitHub source tarballs (with optional git fallback)
-github_sources = bnnanet/serviceman
-git_url = https://github.com/bnnanet/serviceman.git
-
-# Git tag enumeration (vim plugins, shell scripts — git_url alone)
-git_url = https://github.com/tpope/vim-commentary.git
-
-# Gitea (full URL required, or short form + base_url)
-gitea_releases = https://git.rootprojects.org/root/pathman
-
-# GitLab (defaults to gitlab.com)
-gitlab_releases = owner/repo
-
-# HashiCorp releases API
-hashicorp_product = terraform
-
-# Custom source (servicemandist, nodedist, zigdist, etc.)
-source = nodedist
-url = https://nodejs.org/download/release
-```
-
-#### Filtering, versioning, and platform
-
-```ini
-tag_prefix = bun-                    # monorepo: strip prefix from version
-version_prefixes = jq-               # strip from version string (space-separated)
-asset_filter = MinGit                # filename must contain this substring
-exclude = busybox -src- -docs-       # skip assets containing these (space-separated)
-os = posix_2017                      # restrict ALL versions to this OS (blanket)
-alias_of = rg                        # mirrors another package's releases
-```
-
-#### Design rules
-
-- `os` is a blanket tag on ALL versions. Only use for packages that are always
-  POSIX-only. For version-dependent OS tagging, use a custom `TagVariants` in
-  `internal/releases/{pkg}/variants.go`.
-- `git_url` can be primary (gittag source when it's the only key) or secondary
-  fallback alongside `github_sources`/`gitea_sources`.
-- Full URL forms accepted for github/gitea/gitlab (e.g.
-  `github_releases = https://github.com/sharkdp/bat`).
-
-### Testing
-
-Test tools: `cmd/e2etest` (pipeline comparison), `cmd/comparecache` (cache diff),
-`cmd/inspect` (single-package debug). Run each with `--help` for usage.
-
-### Classifier vs Per-Package Tagger
-
-The general classifier (`internal/classify/`) handles patterns common across
-many projects. It MUST NOT contain one-off logic for a single package.
-
-Per-package taggers (`internal/releases/{pkg}/variants.go`) handle
-project-specific knowledge. Read the existing taggers for conventions.
-
-MUST: Derive arch/OS from concrete evidence — not blanket defaults.
-MUST: New general classifier patterns must apply to 2-3+ packages.
-
-### Deploying
-
-```sh
-./scripts/deploy-webicached.sh beta.webi.sh
-./scripts/deploy-webicached.sh next.webi.sh
-```
-
-First-time setup on a new host uses `serviceman`:
-
-```sh
-serviceman add --name webicached \
-  --workdir ~/srv/webid/installers/ -- \
-  ~/bin/webicached \
-    --envfile ~/srv/webid/.env.secret \
-    --conf ~/srv/webid/installers/ \
-    --raw ~/.cache/webi/raw
-```

CONTRIBUTING.md

@@ -10,6 +10,7 @@
 
 - You'll be asked to make changes if you don't run the code formatters and
   linters:
+
   - Node / JavaScript:
     - [prettier](https://webinstall.dev/prettier)
       ```sh
@@ -20,6 +21,7 @@
       npm run lint
       ```
   - Bash
+
     - [shfmt](https://webinstall.dev/shfmt)
       ```sh
       npm run shfmt

README.md

@@ -98,10 +98,9 @@ You just fill in the blanks.
 Just create an empty directory and run the tests until you get a good result.
 
 ```sh
-git clone git@github.com:webinstall/webi-installers.git
-pushd ./webi-installers/
-git submodule update --init
-npm clean-install
+git clone git@github.com:webinstall/packages.git
+pushd packages
+npm install
 ```
 
 ```sh
@@ -145,8 +144,8 @@ It looks like this:
 `releases.js`:
 
 ```js
-module.exports = function () {
-  return github(null, owner, repo).then(function (all) {
+module.exports = function (request) {
+  return github(request, owner, repo).then(function (all) {
     // if you need to do something special, you can do it here
     // ...
     return all;

_common/brew.js

@@ -0,0 +1,64 @@
+'use strict';
+
+/**
+ * Gets a releases from 'brew'.
+ *
+ * @param request
+ * @param {string} formula
+ * @returns {PromiseLike<any> | Promise<any>}
+ */
+function getAllReleases(request, formula) {
+  if (!formula) {
+    return Promise.reject('missing formula for brew');
+  }
+  return request({
+    url: 'https://formulae.brew.sh/api/formula/' + formula + '.json',
+    fail: true, // https://git.coolaj86.com/coolaj86/request.js/issues/2
+    json: true,
+  })
+    .then(failOnBadStatus)
+    .then(function (resp) {
+      var ver = resp.body.versions.stable;
+      var dl = (
+        resp.body.bottle.stable.files.high_sierra ||
+        resp.body.bottle.stable.files.catalina
+      ).url.replace(new RegExp(ver.replace(/\./g, '\\.'), 'g'), '{{ v }}');
+      return [
+        {
+          version: ver,
+          download: dl.replace(/{{ v }}/g, ver),
+        },
+      ].concat(
+        resp.body.versioned_formulae.map(function (f) {
+          var ver = f.replace(/.*@/, '');
+          return {
+            version: ver,
+            download: dl,
+          };
+        }),
+      );
+    })
+    .catch(function (err) {
+      console.error('Error fetching MariaDB versions (brew)');
+      console.error(err);
+      return [];
+    });
+}
+
+function failOnBadStatus(resp) {
+  if (resp.statusCode >= 400) {
+    var err = new Error('Non-successful status code: ' + resp.statusCode);
+    err.code = 'ESTATUS';
+    err.response = resp;
+    throw err;
+  }
+  return resp;
+}
+
+module.exports = getAllReleases;
+
+if (module === require.main) {
+  getAllReleases(require('@root/request'), 'mariadb').then(function (all) {
+    console.info(JSON.stringify(all, null, 2));
+  });
+}

_common/git-tag.js

@@ -0,0 +1,218 @@
+'use strict';
+
+require('dotenv').config({ path: '.env' });
+
+var Crypto = require('crypto');
+var util = require('util');
+var exec = util.promisify(require('child_process').exec);
+var Fs = require('node:fs/promises');
+var FsSync = require('node:fs');
+var Path = require('node:path');
+
+var repoBaseDir = process.env.REPO_BASE_DIR || '';
+if (!repoBaseDir) {
+  repoBaseDir = Path.resolve('./_repos');
+  // for stderr
+  console.error(`[Warn] REPO_BASE_DIR= not set, ${repoBaseDir}`);
+}
+
+var Repos = {};
+
+Repos.clone = async function (repoPath, gitUrl) {
+  let uuid = Crypto.randomUUID();
+  let tmpPath = `${repoPath}.${uuid}.tmp`;
+  let bakPath = `${repoPath}.${uuid}.dup`;
+  await exec(`git clone --bare --filter=tree:0 ${gitUrl} ${tmpPath}`);
+
+  try {
+    FsSync.accessSync(repoPath);
+    return;
+  } catch (e) {
+    if (e.code !== 'ENOENT') {
+      throw e;
+    }
+  }
+
+  // sync to avoid race conditions
+  try {
+    FsSync.renameSync(repoPath, bakPath);
+  } catch (e) {
+    if (e.code !== 'ENOENT') {
+      throw e;
+    }
+  }
+  FsSync.renameSync(tmpPath, repoPath);
+
+  await Fs.rm(bakPath, { force: true, recursive: true });
+};
+
+Repos.checkExists = async function (repoPath) {
+  let err = await Fs.access(repoPath).catch(Object);
+  if (!err) {
+    return true;
+  }
+
+  if (err.code !== 'ENOENT') {
+    throw err;
+  }
+  return false;
+};
+
+Repos.fetch = async function (repoPath) {
+  await exec(`git --git-dir=${repoPath} fetch`);
+};
+
+Repos.getTags = async function (repoPath) {
+  var { stdout } = await exec(`git --git-dir=${repoPath} tag`);
+  var rawTags = stdout.trim().split('\n');
+
+  let tags = [];
+  for (let tag of rawTags) {
+    // ex: v1, v2, v1.1, 1.1.0-rc
+    let maybeVersionRe = /^(v\d+|v?\d+\.\d+)/;
+    let maybeVersion = maybeVersionRe.test(tag);
+    if (maybeVersion) {
+      tags.push(tag);
+    }
+  }
+
+  tags = tags.reverse();
+  return tags;
+};
+
+Repos.getTipInfo = async function (repoPath) {
+  var { stdout } = await exec(
+    `git --git-dir=${repoPath} rev-parse --abbrev-ref HEAD`,
+  );
+  var branch = stdout.trim();
+
+  var info = await Repos.getCommitInfo(repoPath, 'HEAD');
+  info.commitish = branch;
+
+  return info;
+};
+
+Repos.getCommitInfo = async function (repoPath, commitish) {
+  var { stdout } = await exec(
+    `git --git-dir=${repoPath} log -1 --format="%h %H %ad %cd" --date=iso-strict ${commitish}`,
+  );
+  stdout = stdout.trim();
+  var commitParts = stdout.split(/\s+/g);
+  return {
+    commitish: commitish,
+    commit_id: `${commitParts[0]}`,
+    commit: `${commitParts[1]}`,
+    date: commitParts[2],
+    date_authored: commitParts[3],
+  };
+};
+
+/**
+ * Lists GitHub Releases (w/ uploaded assets)
+ *
+ * @param request
+ * @param {string} owner
+ * @param {string} gitUrl
+ * @returns {PromiseLike<any> | Promise<any>}
+ */
+async function getAllReleases(gitUrl) {
+  let all = {
+    releases: [],
+    download: '',
+  };
+
+  let repoName = gitUrl.split('/').pop();
+  repoName = repoName.replace(/\.git$/, '');
+
+  let repoPath = `${repoBaseDir}/${repoName}.git`;
+
+  let isCloned = await Repos.checkExists(repoPath);
+  if (!isCloned) {
+    await Repos.clone(repoPath, gitUrl);
+  } else {
+    await Repos.fetch(repoPath);
+  }
+
+  let commitInfos = [];
+  let tags = await Repos.getTags(repoPath);
+  for (let tag of tags) {
+    let commitInfo = await Repos.getCommitInfo(repoPath, tag);
+    Object.assign(commitInfo, { version: tag, channel: '' });
+    commitInfos.push(commitInfo);
+  }
+
+  {
+    let tipInfo = await Repos.getTipInfo(repoPath);
+    // "2024-01-01T00:00:00-05:00" => "2024-01-01T05:00:00"
+    let date = new Date(tipInfo.date);
+    // "2024-01-01T05:00:00" => "v2024.01.01-05.00.00"
+    let version = date.toISOString();
+    // strip '.000Z'
+    version = version.replace(/\.\d+Z/, '');
+    version = version.replace(/[:\-]/g, '.');
+    version = version.replace(/T/, '-');
+    Object.assign(tipInfo, { version: `v${version}`, channel: '' });
+    if (commitInfos.length > 1) {
+      tipInfo.channel = 'beta';
+    }
+    commitInfos.push(tipInfo);
+  }
+
+  let releases = [];
+  for (let commitInfo of commitInfos) {
+    let version = commitInfo.version.replace(/^v/, '');
+
+    let date = new Date(commitInfo.date);
+    let isoDate = date.toISOString();
+    isoDate = isoDate.replace(/\.\d+Z/, '');
+
+    // tags and HEAD qualify for '--branch <branchish>'
+    let branch = commitInfo.commitish;
+
+    let rel = {
+      name: `${repoName}-v${version}`,
+      version: version,
+      git_tag: branch,
+      git_commit_hash: commitInfo.commit_id,
+      lts: false,
+      channel: commitInfo.channel,
+      date: isoDate,
+      os: '*',
+      arch: '*',
+      ext: 'git',
+      download: gitUrl,
+    };
+
+    releases.push(rel);
+  }
+  all.releases = releases;
+
+  return all;
+}
+
+module.exports = getAllReleases;
+
+if (module === require.main) {
+  (async function main() {
+    let testRepos = [
+      // just a few tags, and a different HEAD
+      'https://github.com/tpope/vim-commentary.git',
+      // no tags, just HEAD
+      'https://github.com/ziglang/zig.vim.git',
+      // many, many tags
+      //'https://github.com/dense-analysis/ale.git',
+    ];
+    for (let url of testRepos) {
+      let all = await getAllReleases(url);
+
+      all = require('../_webi/normalize.js')(all);
+      console.info(JSON.stringify(all, null, 2));
+    }
+  })()
+    .then(function () {
+      process.exit(0);
+    })
+    .catch(function (err) {
+      console.error(err);
+    });
+}

_common/gitea.js

@@ -0,0 +1,39 @@
+'use strict';
+
+var ghRelease = require('./github.js');
+
+/**
+ * Gets the releases for 'ripgrep'. This function could be trimmed down and made
+ * for use with any github release.
+ *
+ * @param request
+ * @param {string} owner
+ * @param {string} repo
+ * @returns {PromiseLike<any> | Promise<any>}
+ */
+function getAllReleases(request, owner, repo, baseurl) {
+  if (!baseurl) {
+    return Promise.reject('missing baseurl');
+  }
+  return ghRelease(request, owner, repo, baseurl + '/api/v1').then(
+    function (all) {
+      return all;
+    },
+  );
+}
+
+module.exports = getAllReleases;
+
+if (module === require.main) {
+  getAllReleases(
+    require('@root/request'),
+    'coolaj86',
+    'go-pathman',
+    'https://git.coolaj86.com',
+  ).then(
+    //getAllReleases(require('@root/request'), 'root', 'serviceman', 'https://git.rootprojects.org').then(
+    function (all) {
+      console.info(JSON.stringify(all, null, 2));
+    },
+  );
+}

_common/github-source.js

@@ -0,0 +1,133 @@
+'use strict';
+
+require('dotenv').config();
+
+/**
+ * Gets the releases for 'ripgrep'. This function could be trimmed down and made
+ * for use with any github release.
+ *
+ * @param request
+ * @param {string} owner
+ * @param {string} repo
+ * @returns {PromiseLike<any> | Promise<any>}
+ */
+async function getAllReleases(
+  request,
+  owner,
+  repo,
+  oses,
+  arches,
+  baseurl = 'https://api.github.com',
+) {
+  if (!owner) {
+    return Promise.reject('missing owner for repo');
+  }
+  if (!repo) {
+    return Promise.reject('missing repo name');
+  }
+
+  let req = {
+    url: `${baseurl}/repos/${owner}/${repo}/releases`,
+    json: true,
+  };
+
+  // TODO I really don't like global config, find a way to do better
+  if (process.env.GITHUB_USERNAME) {
+    req.auth = {
+      user: process.env.GITHUB_USERNAME,
+      pass: process.env.GITHUB_TOKEN,
+    };
+  }
+
+  let resp = await request(req);
+  let gHubResp = resp.body;
+  let all = {
+    releases: [],
+    // TODO make this ':baseurl' + ':releasename'
+    download: '',
+  };
+
+  for (let release of gHubResp) {
+    // TODO tags aren't always semver / sensical
+    let tag = release['tag_name'];
+    let lts = /(\b|_)(lts)(\b|_)/.test(release['tag_name']);
+    let channel = 'stable';
+    if (release['prerelease']) {
+      channel = 'beta';
+    }
+    let date = release['published_at'] || '';
+    date = date.replace(/T.*/, '');
+
+    let urls = [release.tarball_url, release.zipball_url];
+    for (let url of urls) {
+      let resp = await request({
+        method: 'HEAD',
+        followRedirect: true,
+        followAllRedirects: true,
+        followOriginalHttpMethod: true,
+        url: url,
+        stream: true,
+      });
+      // Workaround for bug where method changes to GET
+      resp.destroy();
+
+      // content-disposition: attachment; filename=BeyondCodeBootcamp-DuckDNS.sh-v1.0.1-0-ga2f4bde.zip
+      let name = resp.headers['content-disposition'].replace(
+        /.*filename=([^;]+)(;|$)/,
+        '$1',
+      );
+      all.releases.push({
+        name: name,
+        version: tag,
+        lts: lts,
+        channel: channel,
+        date: date,
+        os: '', // will be guessed by download filename
+        arch: '', // will be guessed by download filename
+        ext: '', // will be normalized
+        download: resp.request.uri.href,
+      });
+    }
+  }
+
+  if (oses) {
+    return combinate(all, oses, arches);
+  }
+
+  return all;
+}
+
+function combinate(all, oses, arches) {
+  let releases = all.releases;
+  // ex: arches = ['amd64', 'arm64', 'armv7l', 'armv6l', 'x86'];
+  // ex: oses = ['macos', 'linux', 'bsd', 'posix'];
+
+  let combos = [];
+  for (let release of releases) {
+    for (let arch of arches) {
+      for (let os of oses) {
+        let combo = {
+          arch: arch,
+          os: os,
+        };
+        let rel = Object.assign({}, release, combo);
+        combos.push(rel);
+      }
+    }
+  }
+  all.releases = combos;
+
+  return all;
+}
+
+module.exports = getAllReleases;
+
+if (module === require.main) {
+  getAllReleases(
+    require('@root/request'),
+    'BeyondCodeBootcamp',
+    'DuckDNS.sh',
+  ).then(function (all) {
+    console.info(JSON.stringify(all, null, 2));
+  });
+}

_common/github.js

@@ -0,0 +1,102 @@
+'use strict';
+
+require('dotenv').config();
+
+/**
+ * Lists GitHub Releases (w/ uploaded assets)
+ *
+ * @param request
+ * @param {string} owner
+ * @param {string} repo
+ * @returns {PromiseLike<any> | Promise<any>}
+ */
+async function getAllReleases(
+  request,
+  owner,
+  repo,
+  baseurl = 'https://api.github.com',
+) {
+  if (!owner) {
+    throw new Error('missing owner for repo');
+  }
+  if (!repo) {
+    throw new Error('missing repo name');
+  }
+
+  var req = {
+    url: `${baseurl}/repos/${owner}/${repo}/releases`,
+    json: true,
+  };
+
+  // TODO I really don't like global config, find a way to do better
+  if (process.env.GITHUB_USERNAME) {
+    req.auth = {
+      user: process.env.GITHUB_USERNAME,
+      pass: process.env.GITHUB_TOKEN,
+    };
+  }
+
+  let resp = await request(req);
+  if (!resp.ok) {
+    console.error('Bad Resp Headers:', resp.headers);
+    console.error('Bad Resp Body:', resp.body);
+    throw new Error('the elusive releases BOOGEYMAN strikes again');
+  }
+
+  let gHubResp = resp.body;
+  let all = {
+    releases: [],
+    // todo make this ':baseurl' + ':releasename'
+    download: '',
+  };
+
+  try {
+    gHubResp.forEach(transformReleases);
+  } catch (e) {
+    console.error(e.message);
+    console.error('Error Headers:', resp.headers);
+    console.error('Error Body:', resp.body);
+    throw e;
+  }
+
+  function transformReleases(release) {
+    for (let asset of release['assets']) {
+      let name = asset['name'];
+      let date = release['published_at']?.replace(/T.*/, '');
+      let download = asset['browser_download_url'];
+
+      // TODO tags aren't always semver / sensical
+      let version = release['tag_name'];
+      let channel;
+      if (release['prerelease']) {
+        // -rcX, -preview, -beta, etc will be checked in _webi/normalize.js
+        channel = 'beta';
+      }
+      let lts = /(\b|_)(lts)(\b|_)/.test(release['tag_name']);
+
+      all.releases.push({
+        name: name,
+        version: version,
+        lts: lts,
+        channel: channel,
+        date: date,
+        os: '', // will be guessed by download filename
+        arch: '', // will be guessed by download filename
+        ext: '', // will be normalized
+        download: download,
+      });
+    }
+  }
+
+  return all;
+}
+
+module.exports = getAllReleases;
+
+if (module === require.main) {
+  getAllReleases(require('@root/request'), 'BurntSushi', 'ripgrep').then(
+    function (all) {
+      console.info(JSON.stringify(all, null, 2));
+    },
+  );
+}

_example/README.md

@@ -23,7 +23,7 @@ install:
 ```text
 ~/.config/envman/PATH.env
 ~/.local/bin/foo
-~/.local/opt/foo/
+~/.local/opt/foo
 ```
 
 ## Cheat Sheet

_example/install.ps1

@@ -19,13 +19,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading foobar from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-if (!(Test-Path -Path "$pkg_src_cmd")) {
+IF (!(Test-Path -Path "$pkg_src_cmd")) {
     Write-Output "Installing foobar"
 
     # TODO: create package-specific temp directory

_example/releases.conf

@@ -1,3 +0,0 @@
-# Example releases.conf — uses ripgrep as a sample project.
-# Copy this file into your package directory and adjust.
-github_releases = BurntSushi/ripgrep

_example/releases.js

@@ -0,0 +1,28 @@
+'use strict';
+
+var github = require('../_common/github.js');
+var owner = 'BurntSushi';
+var repo = 'ripgrep';
+
+/******************************************************************************/
+/** Note: Delete this Comment!                                               **/
+/**                                                                          **/
+/** Need a an example that filters out miscellaneous release files?          **/
+/**   See `deno`, `gitea`, or `caddy`                                        **/
+/**                                                                          **/
+/******************************************************************************/
+
+module.exports = function (request) {
+  return github(request, owner, repo).then(function (all) {
+    return all;
+  });
+};
+
+if (module === require.main) {
+  module.exports(require('@root/request')).then(function (all) {
+    all = require('../_webi/normalize.js')(all);
+    // just select the first 5 for demonstration
+    all.releases = all.releases.slice(0, 5);
+    console.info(JSON.stringify(all, null, 2));
+  });
+}

_npm/scripts/install-webi.js

@@ -4,6 +4,7 @@
 
 //var pkg = require('../package.json');
 var os = require('os');
+//var request = require('@root/request');
 //var promisify = require('util').promisify;
 //var exec = promisify(require('child_process').exec);
 var exec = require('child_process').exec;

_scripts/pwsh-fmt.ps1

@@ -15,8 +15,8 @@ foreach ($my_dir in $my_dirs) {
         $my_ps1 = [System.IO.Path]::GetRelativePath($my_cwd, $my_file.FullName)
         $my_dir = [System.IO.Path]::GetDirectoryName($my_file.FullName)
 
-        if (-not (Test-Path -PathType Leaf -Path $my_ps1) -or
-            -not (Test-Path -PathType Container -Path $my_dir)) {
+        if (-Not (Test-Path -PathType Leaf -Path $my_ps1) -or
+            -Not (Test-Path -PathType Container -Path $my_dir)) {
             Write-Host ("    SKIP {0} (non-regular file or parent directory)" -f $my_ps1)
             continue
         }
@@ -31,7 +31,7 @@ foreach ($my_dir in $my_dirs) {
         $my_new_file | Set-Content -Path $my_ps1
 
         $my_new_file = $my_new_file + "`n"
-        if ($text -ne $my_new_file) {
+        IF ($text -ne $my_new_file) {
             $my_status = 1
         }
     }

_scripts/pwsh-lint.ps1

@@ -15,8 +15,8 @@ foreach ($my_dir in $my_dirs) {
         $my_ps1 = [System.IO.Path]::GetRelativePath($my_cwd, $my_file.FullName)
         $my_dir = [System.IO.Path]::GetDirectoryName($my_file.FullName)
 
-        if (-not (Test-Path -PathType Leaf -Path $my_ps1) -or
-            -not (Test-Path -PathType Container -Path $my_dir)) {
+        if (-Not (Test-Path -PathType Leaf -Path $my_ps1) -or
+            -Not (Test-Path -PathType Container -Path $my_dir)) {
             Write-Host ("    SKIP {0} (non-regular file or parent directory)" -f $my_ps1)
             continue
         }
@@ -39,7 +39,7 @@ foreach ($my_dir in $my_dirs) {
         $my_new_file | Set-Content -Path $my_ps1
 
         $my_new_file = $my_new_file + "`n"
-        if ($my_old_file -ne $my_new_file) {
+        IF ($my_old_file -ne $my_new_file) {
             $my_status = 1
         }
     }

_vim-example/install.ps1

@@ -1,6 +1,6 @@
 #!/usr/bin/env pwsh
 
-if (!(Test-Path -Path "$Env:USERPROFILE\.vim\pack\plugins\start")) {
+IF (!(Test-Path -Path "$Env:USERPROFILE\.vim\pack\plugins\start")) {
     New-Item -Path "$Env:USERPROFILE\.vim\pack\plugins\start" -ItemType Directory -Force | Out-Null
 }
 Remove-Item -Path "$Env:USERPROFILE\.vim\pack\plugins\start\example" -Recurse -ErrorAction Ignore

_webi/builds-cacher-test.js

@@ -1,72 +0,0 @@
-'use strict';
-
-let Path = require('node:path');
-
-let BuildsCacher = require('./builds-cacher.js');
-// let Parallel = require('./parallel.js');
-
-var INSTALLERS_DIR = Path.join(__dirname, '..');
-var CACHE_DIR = Path.join(__dirname, '../_cache');
-
-async function main() {
-  let bc = BuildsCacher.create({
-    caches: CACHE_DIR,
-    installers: INSTALLERS_DIR,
-  });
-
-  // let dirs = await bc.getProjectsByType();
-  // let projNames = Object.keys(dirs.valid);
-
-  let lastUpdate;
-
-  let projName = 'k9s';
-  {
-    let packages = await bc.getPackages({
-      //Releases: Releases,
-      name: projName,
-      date: new Date(),
-    });
-    lastUpdate = packages.updated;
-    console.info(
-      `Last update for '${projName}': ${packages.updated} (${packages.releases.length} assets)`,
-    );
-  }
-
-  console.info('Waiting 5s');
-  {
-    setTimeout(async function () {
-      let packages = await bc.getPackages({
-        //Releases: Releases,
-        name: projName,
-        date: new Date(),
-      });
-      console.info(
-        `Last update for '${projName}': ${packages.updated} (${packages.releases.length} assets)`,
-      );
-      if (lastUpdate < packages.updated) {
-        console.info(`PASS`);
-      } else {
-        console.info(`MAYBE fail`);
-      }
-    }, 5 * 1000);
-  }
-
-  //let parallel = 25;
-  //await Parallel.run(parallel, projNames, getAll);
-  //async function getAll(name) {
-  //  void (await bc.getPackages({
-  //    //Releases: Releases,
-  //    name: name,
-  //    date: new Date(),
-  //  }));
-  //}
-}
-
-main()
-  .then(function () {
-    console.log('Done');
-  })
-  .catch(function (e) {
-    console.error(e.stack || e);
-    process.exit(1);
-  });

_webi/builds-cacher.js

@@ -3,16 +3,15 @@
 var BuildsCacher = module.exports;
 
 let Fs = require('node:fs/promises');
-let Os = require('node:os');
 let Path = require('node:path');
 
-let LEGACY_CACHE_DIR = Path.join(Os.homedir(), '.cache/webi/legacy');
-
 let HostTargets = require('./build-classifier/host-targets.js');
 let Lexver = require('./build-classifier/lexver.js');
 let Triplet = require('./build-classifier/triplet.js');
 
-var ALIAS_RE = /^alias: ([\w.-]+)$/m;
+let request = require('@root/request');
+
+var ALIAS_RE = /^alias: (\w+)$/m;
 
 var LEGACY_ARCH_MAP = {
   '*': 'ANYARCH',
@@ -55,54 +54,6 @@ var TERMS_META = [
   'stable',
 ];
 
-/** @typedef {String} TripletString - {arch}-{vendor}-{os}-{libc} */
-/** @typedef {String} VersionString */
-/** @typedef {Object.<VersionString, Array<BuildAsset>>} PackagesByRelease */
-
-/**
- * @typedef ProjectInfo
- * @prop {Array<BuildAsset>} releases
- * @prop {Array<BuildAsset>} packages
- * @prop {Object.<TripletString, PackagesByRelease>} releasesByTriplet
- * @prop {Array<import('./build-classifier/types.js').ArchString>} arches
- * @prop {Array<import('./build-classifier/types.js').OsString>} oses
- * @prop {Array<import('./build-classifier/types.js').LibcString>} libcs
- * @prop {Array<String>} channels
- * @prop {Array<String>} formats
- * @prop {Array<String>} triplets
- * @prop {Array<String>} versions
- * @prop {Array<String>} lexvers
- * @prop {Object.<String, String>} lexversMap
- */
-
-/**
- * @typedef BuildAsset
- * @prop {String} name
- * @prop {String} version
- * @prop {Boolean} lts
- * @prop {String} date
- * @prop {String} arch
- * @prop {String} os
- * @prop {String} libc
- * @prop {String} ext
- * @prop {String} download
- */
-
-/**
- * @typedef VersionTarget
- * @prop {String} version
- * @prop {Boolean} lts
- * @prop {String} channel
- */
-
-/** @typedef {TargetTriplet & HostTargetPartial} HostTarget */
-/** @typedef {import('./build-classifier/types.js').TargetTriplet} TargetTriplet */
-/**
- * @typedef HostTargetPartial
- * @prop {String} target.triplet - os-vendor-arch-libc
- * @prop {Error} target.error
- */
-
 async function getPartialHeader(path) {
   let readme = `${path}/README.md`;
   let head = await readFirstBytes(readme).catch(function (err) {
@@ -121,7 +72,7 @@ async function readFirstBytes(path) {
   let start = 0;
   let n = 1024;
   let fh = await Fs.open(path, 'r');
-  let buf = Buffer.alloc(n);
+  let buf = new Buffer.alloc(n);
   let result = await fh.read(buf, start, n);
   let str = result.buffer.toString('utf8');
   await fh.close();
@@ -129,27 +80,75 @@ async function readFirstBytes(path) {
   return str;
 }
 
-BuildsCacher.create = function ({ ALL_TERMS, installers }) {
+let promises = {};
+async function getLatestBuilds(Releases, installersDir, cacheDir, name, date) {
+  if (!Releases) {
+    Releases = require(`${installersDir}/${name}/releases.js`);
+  }
+  // TODO update all releases files with module.exports.xxxx = 'foo';
+  if (!Releases.latest) {
+    Releases.latest = Releases;
+  }
+
+  let id = `${cacheDir}/${name}`;
+  if (!promises[id]) {
+    promises[id] = Promise.resolve();
+  }
+
+  promises[id] = promises[id].then(async function () {
+    return await getLatestBuildsInner(Releases, cacheDir, name, date);
+  });
+
+  return await promises[id];
+}
+
+async function getLatestBuildsInner(Releases, cacheDir, name, date) {
+  let data = await Releases.latest(request);
+
+  if (!date) {
+    date = new Date();
+  }
+  let isoDate = date.toISOString();
+  let yearMonth = isoDate.slice(0, 7);
+
+  // TODO hash file
+  let dataFile = `${cacheDir}/${yearMonth}/${name}.json`;
+  // TODO fsstat releases.js vs require-ing time as well
+  let tsFile = `${cacheDir}/${yearMonth}/${name}.updated.txt`;
+
+  let dirPath = Path.dirname(dataFile);
+  await Fs.mkdir(dirPath, { recursive: true });
+
+  let json = JSON.stringify(data, null, 2);
+  await Fs.writeFile(dataFile, json, 'utf8');
+
+  let seconds = date.valueOf();
+  let ms = seconds / 1000;
+  let msStr = ms.toFixed(3);
+  await Fs.writeFile(tsFile, msStr, 'utf8');
+
+  return data;
+}
+
+BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
   let installersDir = installers;
+  let cacheDir = caches;
 
   if (!ALL_TERMS) {
     ALL_TERMS = Triplet.TERMS_PRIMARY_MAP;
   }
 
   let bc = {};
-  bc.ALL_TERMS = ALL_TERMS;
-  bc.orphanTerms = Object.assign({}, bc.ALL_TERMS);
-  bc.unknownTerms = {};
   bc.usedTerms = {};
+  bc.orphanTerms = Object.assign({}, ALL_TERMS);
+  bc.unknownTerms = {};
   bc.formats = [];
   bc._triplets = {};
   bc._targetsByBuildIdCache = {};
   bc._caches = {};
+  bc._staleAge = 15 * 60 * 1000;
   bc._allFormats = {};
   bc._allTriplets = {};
-  // Per-name lock: serializes cold-cache getPackages so concurrent
-  // callers can't corrupt bc._caches[name] via a transformAndUpdate race.
-  bc._inflight = {};
 
   for (let term of TERMS_META) {
     delete bc.orphanTerms[term];
@@ -168,6 +167,19 @@ BuildsCacher.create = function ({ ALL_TERMS, installers }) {
     let entries = await Fs.readdir(installersDir, { withFileTypes: true });
     for (let entry of entries) {
       let meta = await bc.getProjectTypeByEntry(entry);
+      if (meta.type === 'not_found') {
+        let err = meta.detail;
+        console.error('');
+        console.error('PROBLEM');
+        console.error(`    ${err.message}`);
+        console.error('');
+        console.error('SOLUTION');
+        console.error('    npm clean-install');
+        console.error('');
+        throw new Error(
+          '[SANITY FAIL] should never have missing modules in prod',
+        );
+      }
       dirs[meta.type][entry.name] = meta.detail;
     }
 
@@ -236,16 +248,19 @@ BuildsCacher.create = function ({ ALL_TERMS, installers }) {
       return { type: 'alias', detail: link };
     }
 
-    let cacheFile = `${LEGACY_CACHE_DIR}/${entry.name}.json`;
-    let hasCacheFile = await Fs.access(cacheFile)
-      .then(function () {
-        return true;
-      })
-      .catch(function () {
-        return false;
-      });
-    if (!hasCacheFile) {
-      return { type: 'selfhosted', detail: true };
+    let releasesPath = Path.join(path, 'releases.js');
+    try {
+      void require(releasesPath);
+    } catch (err) {
+      if (err.code !== 'MODULE_NOT_FOUND') {
+        return { type: 'errors', detail: err };
+      }
+
+      if (err.message.includes(`Cannot find module '${releasesPath}'`)) {
+        return { type: 'selfhosted', detail: true };
+      }
+
+      return { type: 'not_found', detail: err };
     }
 
     return { type: 'valid', detail: true };
@@ -253,40 +268,24 @@ BuildsCacher.create = function ({ ALL_TERMS, installers }) {
 
   // Typically a package is organized by release (ex: go has 1.20, 1.21, etc),
   // but we will organize by the build (ex: go1.20-darwin-arm64.tar.gz, etc).
-  bc.getPackages = async function (args) {
-    let name = args.name;
-    let warm = bc._caches[name];
-    if (warm) {
-      return _doGetPackages(args);
+  bc.getPackages = async function ({ Releases, name, date }) {
+    if (!date) {
+      date = new Date();
     }
-    let inflight = bc._inflight[name];
-    if (inflight) {
-      return inflight;
-    }
-    let p = _doGetPackages(args).finally(function () {
-      delete bc._inflight[name];
-    });
-    bc._inflight[name] = p;
-    return p;
-  };
+    let isoDate = date.toISOString();
+    let yearMonth = isoDate.slice(0, 7);
+    let dataFile = `${cacheDir}/${yearMonth}/${name}.json`;
 
-  async function _doGetPackages({ name }) {
-    let dataFile = `${LEGACY_CACHE_DIR}/${name}.json`;
-    let tsFile = `${LEGACY_CACHE_DIR}/${name}.updated.txt`;
+    // let secondsStr = await Fs.readFile(tsFile, 'ascii').catch(function (err) {
+    //   if (err.code !== 'ENOENT') {
+    //     throw err;
+    //   }
+    //   return '0';
+    // });
+    // secondsStr = secondsStr.trim();
+    // let seconds = parseFloat(secondsStr) || 0;
 
-    let tsDate;
-    {
-      let secondsStr = await Fs.readFile(tsFile, 'ascii').catch(function (err) {
-        if (err.code !== 'ENOENT') {
-          throw err;
-        }
-        return '0';
-      });
-      secondsStr = secondsStr.trim();
-      let seconds = parseFloat(secondsStr) || 0;
-      let ms = seconds * 1000;
-      tsDate = new Date(ms);
-    }
+    // let age = now - seconds;
 
     let projInfo = bc._caches[name];
 
@@ -326,20 +325,272 @@ BuildsCacher.create = function ({ ALL_TERMS, installers }) {
       }
     }
     if (!projInfo) {
-      return meta;
+      projInfo = await getLatestBuilds(Releases, installersDir, cacheDir, name);
     }
-    let latestProjInfo = await BuildsCacher.transformAndUpdate(
-      name,
-      projInfo,
-      meta,
-      tsDate,
-      bc,
-    );
-    bc._caches[name] = latestProjInfo;
 
-    return latestProjInfo;
+    process.nextTick(async function () {
+      let now = date.valueOf();
+      let age = now - projInfo.updated;
+      if (age < bc._staleAge) {
+        return;
+      }
+
+      projInfo = await getLatestBuilds(Releases, installersDir, cacheDir, name);
+      transformAndUpdate(name, projInfo, meta, date);
+    });
+
+    transformAndUpdate(name, projInfo, meta, date);
+    return projInfo;
+  };
+
+  function transformAndUpdate(name, projInfo, meta, date) {
+    meta.packages = [];
+
+    let updated = date.valueOf();
+    Object.assign(projInfo, { name, updated }, meta);
+    for (let build of projInfo.releases) {
+      let buildTarget = bc.classify(projInfo, build);
+      if (!buildTarget) {
+        // ignore known, non-package extensions
+        continue;
+      }
+
+      if (buildTarget.error) {
+        let err = buildTarget.error;
+        let code = err.code || '';
+        console.error(`[ERROR]: ${code} ${projInfo.name}: ${build.name}`);
+        console.error(`>>> ${err.message} <<<`);
+        console.error(projInfo);
+        console.error(build);
+        console.error(`^^^ ${err.message} ^^^`);
+        console.error(err.stack);
+        continue;
+      }
+
+      build.target = buildTarget;
+      meta.packages.push(build);
+    }
+
+    updateReleasesByTriplet(meta);
+    updateAndSortVersions(projInfo, meta);
+
+    Object.assign(projInfo, { name, updated }, meta);
+    bc._caches[name] = projInfo;
   }
 
+  function updateReleasesByTriplet(meta) {
+    for (let build of meta.packages) {
+      let target = build.target;
+
+      let triplet = `${target.os}-${target.arch}-${target.libc}`;
+      if (!meta.releasesByTriplet[triplet]) {
+        meta.releasesByTriplet[triplet] = {};
+      }
+
+      let buildsByRelease = meta.releasesByTriplet[triplet];
+      if (!buildsByRelease[build.version]) {
+        buildsByRelease[build.version] = [];
+      }
+
+      let packages = buildsByRelease[build.version];
+      packages.push(build);
+    }
+  }
+
+  // TODO
+  //   - tag channels
+  function updateAndSortVersions(projInfo, meta) {
+    for (let build of projInfo.packages) {
+      let hasVersion = meta.versions.includes(build.version);
+      if (!hasVersion) {
+        build.lexver = Lexver.parseVersion(build.version);
+        meta.lexversMap[build.lexver] = build.version;
+      }
+    }
+
+    meta.lexvers = Object.keys(meta.lexversMap);
+    meta.lexvers.sort();
+    meta.lexvers.reverse();
+
+    meta.versions = [];
+    for (let lexver of meta.lexvers) {
+      let version = meta.lexversMap[lexver];
+      meta.versions.push(version);
+    }
+
+    projInfo.packages.sort(function (a, b) {
+      if (a.lexver > b.lexver) {
+        return -1;
+      }
+      if (a.lexver < b.lexver) {
+        return 1;
+      }
+      return 0;
+    });
+  }
+
+  // Makes sure that packages are updated once an hour, on average
+  bc._staleNames = [];
+  bc._freshenTimeout = null;
+  bc.freshenRandomPackage = async function (minDelay) {
+    if (!minDelay) {
+      minDelay = 15 * 1000;
+    }
+
+    if (bc._staleNames.length === 0) {
+      let dirs = await bc.getProjectsByType();
+      bc._staleNames = Object.keys(dirs.valid);
+      bc._staleNames.sort(function () {
+        return 0.5 - Math.random();
+      });
+    }
+
+    let name = bc._staleNames.pop();
+    void (await bc.getPackages({
+      //Releases: Releases,
+      name: name,
+      date: new Date(),
+    }));
+
+    let hour = 60 * 60 * 1000;
+    let delay = minDelay;
+    let spread = hour / bc._staleNames.length;
+    let seed = Math.random();
+    delay += seed * spread;
+
+    clearTimeout(bc._freshenTimeout);
+    bc._freshenTimeout = setTimeout(bc.freshenRandomPackage, delay);
+    bc._freshenTimeout.unref();
+  };
+
+  bc.classify = function (projInfo, build) {
+    /* jshint maxcomplexity: 25 */
+    let maybeInstallable = Triplet.maybeInstallable(projInfo, build);
+    if (!maybeInstallable) {
+      return null;
+    }
+
+    if (LEGACY_OS_MAP[build.os]) {
+      build.os = LEGACY_OS_MAP[build.os];
+    }
+    if (LEGACY_ARCH_MAP[build.arch]) {
+      build.arch = LEGACY_ARCH_MAP[build.arch];
+    }
+
+    // because some packages are shimmed to match a single download against
+    let preTarget = Object.assign({ os: '', arch: '', libc: '' }, build);
+
+    let targetId = `${preTarget.os}:${preTarget.arch}:${preTarget.libc}`;
+    let buildId = `${projInfo.name}:${targetId}@${build.download}`;
+    let target = bc._targetsByBuildIdCache[buildId];
+    if (target) {
+      Object.assign(build, { target: target, triplet: target.triplet });
+      return target;
+    }
+
+    let pattern = Triplet.toPattern(projInfo, build);
+    if (!pattern) {
+      let err = new Error(`no pattern generated for ${name}`);
+      err.code = 'E_BUILD_NO_PATTERN';
+      target = { error: err };
+      bc._targetsByBuildIdCache[buildId] = target;
+      return target;
+    }
+
+    let rawTerms = pattern.split(/[_\{\}\/\.\-]+/g);
+    for (let term of rawTerms) {
+      delete bc.orphanTerms[term];
+      bc.usedTerms[term] = true;
+    }
+
+    // {NAME}/{NAME}-{VER}-Windows-x86_64_v2-musl.exe =>
+    //     {NAME}.windows.x86_64v2.musl.exe
+    let terms = Triplet.patternToTerms(pattern);
+    if (!terms.length) {
+      let err = new Error(`'${terms}' was trimmed to ''`);
+      target = { error: err };
+      bc._targetsByBuildIdCache[buildId] = target;
+      return target;
+    }
+
+    for (let term of terms) {
+      if (!term) {
+        continue;
+      }
+
+      if (ALL_TERMS[term]) {
+        delete bc.orphanTerms[term];
+        bc.usedTerms[term] = true;
+        continue;
+      }
+
+      bc.unknownTerms[term] = true;
+    }
+
+    // {NAME}.windows.x86_64v2.musl.exe
+    //     windows-x86_64_v2-musl
+    target = { triplet: '' };
+    void Triplet.termsToTarget(target, projInfo, build, terms);
+
+    target.triplet = `${target.arch}-${target.vendor}-${target.os}-${target.libc}`;
+
+    {
+      // TODO I don't love this hidden behavior
+      // perhaps classify should just happen when the package is loaded
+      // (and the sanity error should be removed, or thrown after the loop is complete)
+      let hasTriplet = projInfo.triplets.includes(target.triplet);
+      if (!hasTriplet) {
+        projInfo.triplets.push(target.triplet);
+      }
+      let hasOs = projInfo.oses.includes(target.os);
+      if (!hasOs) {
+        projInfo.oses.push(target.os);
+      }
+      let hasArch = projInfo.arches.includes(target.arch);
+      if (!hasArch) {
+        projInfo.arches.push(target.arch);
+      }
+      let hasLibc = projInfo.libcs.includes(target.libc);
+      if (!hasLibc) {
+        projInfo.libcs.push(target.libc);
+      }
+
+      if (!build.ext) {
+        build.ext = Triplet.buildToPackageType(build);
+      }
+      if (build.ext) {
+        if (!build.ext.startsWith('.')) {
+          build.ext = `.${build.ext}`;
+        }
+      }
+      let hasExt = projInfo.formats.includes(build.ext);
+      if (!hasExt) {
+        projInfo.formats.push(build.ext);
+      }
+      let hasGlobalExt = bc.formats.includes(build.ext);
+      if (!hasGlobalExt) {
+        bc.formats.push(build.ext);
+      }
+    }
+
+    bc._triplets[target.triplet] = true;
+    bc._targetsByBuildIdCache[buildId] = target;
+
+    let triple = [target.arch, target.vendor, target.os, target.libc];
+    for (let term of triple) {
+      if (!ALL_TERMS[term]) {
+        throw new Error(
+          `[SANITY FAIL] '${projInfo.name}' '${target.triplet}' generated unknown term '${term}'`,
+        );
+      }
+
+      delete bc.orphanTerms[term];
+      bc.usedTerms[term] = true;
+    }
+
+    return target;
+  };
+
   /**
    * Given a list of acceptable formats, get the sorted list of of formats.
    * Actually used (as per node _webi/lint-builds.js):
@@ -430,13 +681,9 @@ BuildsCacher.create = function ({ ALL_TERMS, installers }) {
       exts.push('.gz');
       exts.push('.sh');
     }
-    let hasGit = formats.includes('git') || formats.includes('.git');
-    if (hasGit) {
-      exts.push('.git');
-    }
+    exts.push('.git');
 
     // Fallbacks
-    // (we include everything to bubble an extract error over not found)
     exts.push('.app.zip');
     exts.push('.dmg');
     exts.push('.pkg');
@@ -487,34 +734,21 @@ BuildsCacher.create = function ({ ALL_TERMS, installers }) {
   };
 
   /**
-   * @param {ProjectInfo} projInfo
+   * @param {Object} target
+   * @param {String} target.os - linux, darwin, windows, *bsd
+   * @param {String} target.arch - arm64, x86_64, ppc64le
+   * @param {String} target.libc - none, libc, gnu, musl, bionic, msvc
+   * @param {String} target.triplet - os-vendor-arch-libc
+   * @param {Error} target.error
    */
-  bc.enumerateLatestVersions = function (projInfo) {
-    let lexPrefix = '';
-    let matchInfo = Lexver.matchSorted(projInfo.lexvers, lexPrefix);
-    let verInfo = {
-      default: projInfo.lexversMap[matchInfo.default],
-      previous: projInfo.lexversMap[matchInfo.previous],
-      stable: projInfo.lexversMap[matchInfo.stable],
-      latest: projInfo.lexversMap[matchInfo.latest],
-    };
-
-    return verInfo;
-  };
-
-  /**
-   * @param {ProjectInfo} projInfo
-   * @param {HostTarget} hostTarget
-   * @param {VersionTarget} verTarget
-   */
-  bc.findMatchingPackages = function (projInfo, hostTarget, verTarget) {
-    let matchInfo = bc._enumerateVersions(projInfo, verTarget.version);
+  bc.findMatchingPackages = function (pkgInfo, hostTarget, verTarget) {
+    let matchInfo = bc._enumerateVersions(pkgInfo, verTarget.version);
     let triplets = bc._enumerateTriplets(hostTarget);
     //console.log('dbg: matchInfo', matchInfo);
 
     if (matchInfo) {
       for (let _triplet of triplets) {
-        let targetReleases = projInfo.releasesByTriplet[_triplet];
+        let targetReleases = pkgInfo.releasesByTriplet[_triplet];
         if (!targetReleases) {
           continue;
         }
@@ -522,7 +756,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers }) {
         // Make sure that these releases are the expected version
         // (ex: jq1.7 => darwin-arm64-libc, jq1.6 => darwin-x86_64-libc)
         for (let matchver of matchInfo.matches) {
-          let ver = projInfo.lexversMap[matchver] || matchver;
+          let ver = pkgInfo.lexversMap[matchver] || matchver;
           let packages = targetReleases[ver];
           if (!packages) {
             continue;
@@ -531,7 +765,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers }) {
           let match = {
             triplet: _triplet,
             packages: packages,
-            latest: projInfo.versions[0],
+            latest: pkgInfo.versions[0],
             version: ver,
             versions: matchInfo,
           };
@@ -542,19 +776,29 @@ BuildsCacher.create = function ({ ALL_TERMS, installers }) {
       return null;
     }
 
-    // Version-first iteration, not triplet-first: take the newest
-    // version even when its only build lives in a fallback triplet
-    // (e.g. serviceman v1.0.1 only exists at posix_2017-ANYARCH-none).
-    for (let lexver of projInfo.lexvers) {
-      let ver = projInfo.lexversMap[lexver] || lexver;
+    for (let _triplet of triplets) {
+      let targetReleases = pkgInfo.releasesByTriplet[_triplet];
+      if (!targetReleases) {
+        continue;
+      }
 
-      for (let _triplet of triplets) {
-        let targetReleases = projInfo.releasesByTriplet[_triplet];
-        if (!targetReleases) {
-          continue;
-        }
+      let versions = Object.keys(targetReleases);
+      //console.log('dbg: targetRelease versions', versions);
+      let lexvers = [];
+      for (let version of versions) {
+        let lexPrefix = Lexver.parseVersion(version);
+        lexvers.push(lexPrefix);
+      }
+      lexvers.sort();
+      lexvers.reverse();
+      // TODO get the other matchInfo props
 
+      // Make sure that these releases are the expected version
+      // (ex: jq1.7 => darwin-arm64-libc, jq1.6 => darwin-x86_64-libc)
+      for (let matchver of lexvers) {
+        let ver = pkgInfo.lexversMap[matchver] || matchver;
         let packages = targetReleases[ver];
+        //console.log('dbg: packages', packages);
         if (!packages) {
           continue;
         }
@@ -568,7 +812,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers }) {
           let match = {
             triplet: _triplet,
             packages: packages,
-            latest: projInfo.versions[0],
+            latest: pkgInfo.versions[0],
             version: ver,
             versions: matchInfo,
           };
@@ -587,7 +831,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers }) {
         let match = {
           triplet: _triplet,
           packages: packages,
-          latest: projInfo.versions[0],
+          latest: pkgInfo.versions[0],
           version: ver,
           versions: matchInfo,
         };
@@ -605,32 +849,22 @@ BuildsCacher.create = function ({ ALL_TERMS, installers }) {
       return triplets;
     }
 
-    // Prefer platform-specific matches over ANYOS/ANYARCH fallbacks.
-    // This ensures e.g. darwin-aarch64-none matches before
-    // ANYOS-ANYARCH-none (.git source URLs from old releases).
     let oses = [];
     if (hostTarget.os === 'windows') {
-      oses = ['windows', 'ANYOS'];
+      oses = ['ANYOS', 'windows'];
     } else if (hostTarget.os === 'android') {
-      oses = ['android', 'linux', 'posix_2017', 'posix_2024', 'ANYOS'];
+      oses = ['ANYOS', 'posix_2017', 'android', 'linux'];
     } else {
-      oses = [hostTarget.os, 'posix_2017', 'posix_2024', 'ANYOS'];
+      oses = ['ANYOS', 'posix_2017', hostTarget.os];
     }
 
     let waterfall = HostTargets.WATERFALL[hostTarget.os] || {};
     let arches = waterfall[hostTarget.arch] ||
       HostTargets.WATERFALL.ANYOS[hostTarget.arch] || [hostTarget.arch];
-    arches = arches.concat(['ANYARCH']);
+    arches = ['ANYARCH'].concat(arches);
     let libcs = waterfall[hostTarget.libc] ||
       HostTargets.WATERFALL.ANYOS[hostTarget.libc] || [hostTarget.libc];
 
-    // Extend the glibc-host waterfall: the table only lists [none, libc]
-    // but Rust projects (bat, rg) and node ship libc='gnu' builds, and
-    // static musl builds also run on glibc hosts.
-    if (hostTarget.libc === 'libc' && !libcs.includes('gnu')) {
-      libcs = ['none', 'gnu', 'musl', 'libc'];
-    }
-
     for (let os of oses) {
       for (let arch of arches) {
         for (let libc of libcs) {
@@ -644,262 +878,15 @@ BuildsCacher.create = function ({ ALL_TERMS, installers }) {
     return triplets;
   };
 
-  bc._enumerateVersions = function (projInfo, ver) {
+  bc._enumerateVersions = function (pkgInfo, ver) {
     if (!ver) {
       return null;
     }
     let lexPrefix = Lexver.parsePrefix(ver);
-    let matchInfo = Lexver.matchSorted(projInfo.lexvers, lexPrefix);
+    let matchInfo = Lexver.matchSorted(pkgInfo.lexvers, lexPrefix);
 
     return matchInfo;
   };
 
   return bc;
 };
-
-BuildsCacher._classify = function (bc, projInfo, build) {
-  /* jshint maxcomplexity: 30 */
-  // Cache entries arrive pre-classified (os/arch/libc/ext set). Skip
-  // maybeInstallable for those — it false-rejects names ending in a
-  // version tag (`serviceman-v1.0.1`, `v1.0.1.zip`).
-  let cacheClassified =
-    build.os && build.arch && build.libc && build.ext;
-  if (!cacheClassified) {
-    let maybeInstallable = Triplet.maybeInstallable(projInfo, build);
-    if (!maybeInstallable) {
-      return null;
-    }
-  }
-
-  if (LEGACY_OS_MAP[build.os]) {
-    build.os = LEGACY_OS_MAP[build.os];
-  }
-  if (LEGACY_ARCH_MAP[build.arch]) {
-    build.arch = LEGACY_ARCH_MAP[build.arch];
-  }
-
-  // because some packages are shimmed to match a single download against
-  let preTarget = Object.assign({ os: '', arch: '', libc: '' }, build);
-
-  let targetId = `${preTarget.os}:${preTarget.arch}:${preTarget.libc}`;
-  let buildId = `${projInfo.name}:${targetId}@${build.download}`;
-  //console.log(`dbg: buildId`, buildId);
-  let target = bc._targetsByBuildIdCache[buildId];
-  if (target) {
-    Object.assign(build, { target: target, triplet: target.triplet });
-    return target;
-  }
-
-  let pattern = Triplet.toPattern(projInfo, build);
-  //console.log(`dbg: pattern`, pattern);
-  if (!pattern) {
-    let err = new Error(`no pattern generated for ${projInfo.name}`);
-    err.code = 'E_BUILD_NO_PATTERN';
-    target = { error: err };
-    bc._targetsByBuildIdCache[buildId] = target;
-    return target;
-  }
-
-  let rawTerms = pattern.split(/[_\{\}\/\.\-]+/g);
-  //console.log(`dbg: rawTerms`, rawTerms);
-  for (let term of rawTerms) {
-    delete bc.orphanTerms[term];
-    bc.usedTerms[term] = true;
-  }
-
-  // {NAME}/{NAME}-{VER}-Windows-x86_64_v2-musl.exe =>
-  //     {NAME}.windows.x86_64v2.musl.exe
-  let terms = Triplet.patternToTerms(pattern);
-  //console.log(`dbg: terms`, terms);
-  if (!terms.length) {
-    let err = new Error(`'${terms}' was trimmed to ''`);
-    target = { error: err };
-    bc._targetsByBuildIdCache[buildId] = target;
-    return target;
-  }
-
-  for (let term of terms) {
-    if (!term) {
-      continue;
-    }
-
-    if (bc.ALL_TERMS[term]) {
-      delete bc.orphanTerms[term];
-      bc.usedTerms[term] = true;
-      continue;
-    }
-
-    bc.unknownTerms[term] = true;
-  }
-
-  // Skip termsToTarget for cache-classified entries: it false-flags
-  // e.g. .git URLs as os=ANYOS while the cache says os=posix_2017,
-  // and the mismatch check throws.
-  target = { triplet: '' };
-  if (cacheClassified) {
-    target.os = build.os;
-    target.arch = build.arch;
-    target.libc = build.libc;
-    target.vendor = build.vendor || 'unknown';
-    target.android = false;
-    target.unknownTerms = [];
-  } else {
-    try {
-      void Triplet.termsToTarget(target, projInfo, build, terms);
-    } catch (e) {
-      console.error(`PACKAGE FORMAT CHANGE for '${projInfo.name}':`);
-      console.error(e.message);
-      console.error(build);
-      return null;
-    }
-  }
-
-  target.triplet = `${target.arch}-${target.vendor}-${target.os}-${target.libc}`;
-
-  {
-    // TODO I don't love this hidden behavior
-    // perhaps classify should just happen when the package is loaded
-    // (and the sanity error should be removed, or thrown after the loop is complete)
-    let hasTriplet = projInfo.triplets.includes(target.triplet);
-    if (!hasTriplet) {
-      projInfo.triplets.push(target.triplet);
-    }
-    let hasOs = projInfo.oses.includes(target.os);
-    if (!hasOs) {
-      projInfo.oses.push(target.os);
-    }
-    let hasArch = projInfo.arches.includes(target.arch);
-    if (!hasArch) {
-      projInfo.arches.push(target.arch);
-    }
-    let hasLibc = projInfo.libcs.includes(target.libc);
-    if (!hasLibc) {
-      projInfo.libcs.push(target.libc);
-    }
-
-    if (!build.ext) {
-      build.ext = Triplet.buildToPackageType(build);
-    }
-    if (build.ext) {
-      if (!build.ext.startsWith('.')) {
-        build.ext = `.${build.ext}`;
-      }
-    }
-    let hasExt = projInfo.formats.includes(build.ext);
-    if (!hasExt) {
-      projInfo.formats.push(build.ext);
-    }
-    let hasGlobalExt = bc.formats.includes(build.ext);
-    if (!hasGlobalExt) {
-      bc.formats.push(build.ext);
-    }
-  }
-
-  bc._triplets[target.triplet] = true;
-  bc._targetsByBuildIdCache[buildId] = target;
-
-  let triple = [target.arch, target.vendor, target.os, target.libc];
-  for (let term of triple) {
-    if (!bc.ALL_TERMS[term]) {
-      throw new Error(
-        `[SANITY FAIL] '${projInfo.name}' '${target.triplet}' generated unknown term '${term}'`,
-      );
-    }
-
-    delete bc.orphanTerms[term];
-    bc.usedTerms[term] = true;
-  }
-
-  return target;
-};
-
-BuildsCacher.transformAndUpdate = function (name, projInfo, meta, date, bc) {
-  meta.packages = [];
-
-  let updated = date.valueOf();
-
-  Object.assign(projInfo, { name, updated }, meta);
-  for (let build of projInfo.releases) {
-    let buildTarget = BuildsCacher._classify(bc, projInfo, build);
-    if (!buildTarget) {
-      // ignore known, non-package extensions
-      continue;
-    }
-
-    if (buildTarget.error) {
-      let err = buildTarget.error;
-      let code = err.code || '';
-      console.error(`[ERROR]: ${code} ${projInfo.name}: ${build.name}`);
-      console.error(`>>> ${err.message} <<<`);
-      console.error(projInfo);
-      console.error(build);
-      console.error(`^^^ ${err.message} ^^^`);
-      console.error(err.stack);
-      continue;
-    }
-
-    if (!build.name) {
-      build.name = build.download.replace(/.*\//, '');
-    }
-
-    build.target = buildTarget;
-    meta.packages.push(build);
-  }
-
-  BuildsCacher.updateReleasesByTriplet(meta);
-  BuildsCacher.updateAndSortVersions(projInfo, meta);
-
-  Object.assign(projInfo, { name, updated }, meta);
-  return projInfo;
-};
-
-// TODO
-//   - tag channels
-BuildsCacher.updateAndSortVersions = function (projInfo, meta) {
-  for (let build of projInfo.packages) {
-    let hasVersion = meta.versions.includes(build.version);
-    if (!hasVersion) {
-      build.lexver = Lexver.parseVersion(build.version);
-      meta.lexversMap[build.lexver] = build.version;
-    }
-  }
-
-  meta.lexvers = Object.keys(meta.lexversMap);
-  meta.lexvers.sort();
-  meta.lexvers.reverse();
-
-  meta.versions = [];
-  for (let lexver of meta.lexvers) {
-    let version = meta.lexversMap[lexver];
-    meta.versions.push(version);
-  }
-
-  projInfo.packages.sort(function (a, b) {
-    if (a.lexver > b.lexver) {
-      return -1;
-    }
-    if (a.lexver < b.lexver) {
-      return 1;
-    }
-    return 0;
-  });
-};
-
-BuildsCacher.updateReleasesByTriplet = function (meta) {
-  for (let build of meta.packages) {
-    let target = build.target;
-
-    let triplet = `${target.os}-${target.arch}-${target.libc}`;
-    if (!meta.releasesByTriplet[triplet]) {
-      meta.releasesByTriplet[triplet] = {};
-    }
-
-    let buildsByRelease = meta.releasesByTriplet[triplet];
-    if (!buildsByRelease[build.version]) {
-      buildsByRelease[build.version] = [];
-    }
-
-    let packages = buildsByRelease[build.version];
-    packages.push(build);
-  }
-};

_webi/builds.js

@@ -15,23 +15,26 @@ let bc = BuildsCacher.create({
   caches: CACHE_DIR,
   installers: INSTALLERS_DIR,
 });
+bc.freshenRandomPackage(600 * 1000);
 
 Builds.init = async function () {
-  let dirs = await bc.getProjectsByType();
+  bc.freshenRandomPackage(600 * 1000);
+
+  let dirs = await bc.getProjects();
   let projNames = Object.keys(dirs.valid);
 
   let parallel = 25;
   await Parallel.run(parallel, projNames, getAll);
   async function getAll(name) {
     void (await bc.getPackages({
+      //Releases: Releases,
       name: name,
       date: new Date(),
     }));
   }
 };
 
-Builds.enumerateLatestVersions = bc.enumerateLatestVersions;
-Builds.findMatchingPackages = bc.findMatchingPackages;
-Builds.getPackage = bc.getPackages;
 Builds.getProjectType = bc.getProjectType;
+Builds.getPackage = bc.getPackages;
+Builds.findMatchingPackages = bc.findMatchingPackages;
 Builds.selectPackage = bc.selectPackage;

_webi/classify-one.js

@@ -1,90 +0,0 @@
-'use strict';
-
-let Fs = require('node:fs/promises');
-let Os = require('node:os');
-let Path = require('node:path');
-
-let BuildsCacher = require('./builds-cacher.js');
-let Triplet = require('./build-classifier/triplet.js');
-
-let LEGACY_CACHE_DIR = Path.join(Os.homedir(), '.cache/webi/legacy');
-
-async function main() {
-  let projName = process.argv[2];
-  if (!projName) {
-    console.error(``);
-    console.error(`USAGE`);
-    console.error(``);
-    console.error(`    classify-one <project-name>`);
-    console.error(``);
-    console.error(`EXAMPLE`);
-    console.error(``);
-    console.error(`    classify-one caddy`);
-    console.error(``);
-    return;
-  }
-
-  let tsDate = new Date(0);
-  let meta = {
-    // version info
-    versions: [],
-    lexvers: [],
-    lexversMap: {},
-    // culled release assets
-    packages: [],
-    releasesByTriplet: {},
-    // target info
-    triplets: [],
-    oses: [],
-    arches: [],
-    libcs: [],
-    formats: [],
-  };
-
-  let dataFile = Path.join(LEGACY_CACHE_DIR, `${projName}.json`);
-  let json = await Fs.readFile(dataFile, 'utf8');
-  let projInfo = JSON.parse(json);
-
-  // let packages = await Builds.getPackage({ name: projName });
-  // console.log(packages);
-
-  let bc = {};
-  bc.ALL_TERMS = Triplet.TERMS_PRIMARY_MAP;
-  bc.orphanTerms = Object.assign({}, bc.ALL_TERMS);
-  bc.unknownTerms = {};
-  bc.usedTerms = {};
-  bc.formats = [];
-  bc._targetsByBuildIdCache = {};
-  bc._triplets = {};
-
-  let transformed = BuildsCacher.transformAndUpdate(
-    projName,
-    projInfo,
-    meta,
-    tsDate,
-    bc,
-  );
-
-  console.log(`[DEBUG] transformed`);
-  let sample = transformed.packages.slice(0, 20);
-  console.log('packages:', sample, ':packages');
-  let firstTriplet = Object.keys(transformed.releasesByTriplet)[0];
-  let firstVersion = transformed.versions[0];
-  console.log(
-    `releasesByTriplet[${firstTriplet}][${firstVersion}]:`,
-    transformed.releasesByTriplet[firstTriplet]?.[firstVersion],
-    ':releasesByTriplet',
-  );
-  console.log('versions:', transformed.versions, ':versions');
-  console.log('triplets:', transformed.triplets, ':triplets');
-  console.log('oses:', transformed.oses, ':oses');
-  console.log('arches:', transformed.arches, ':arches');
-  console.log('libcs:', transformed.libcs, ':libcs');
-  console.log('formats:', transformed.formats, ':formats');
-  console.log(Object.keys(transformed));
-}
-
-main().catch(function (err) {
-  console.error('Error:');
-  console.error(err);
-});

_webi/curl-pipe-bootstrap.tpl.ps1

@@ -6,7 +6,7 @@
 ############################################################
 New-Item -Path "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 New-Item -Path "$Env:USERPROFILE\.local\bin" -ItemType Directory -Force | Out-Null
-if ($null -eq $Env:WEBI_HOST -or $Env:WEBI_HOST -eq "") { $Env:WEBI_HOST = "https://webinstall.dev" }
+IF ($null -eq $Env:WEBI_HOST -or $Env:WEBI_HOST -eq "") { $Env:WEBI_HOST = "https://webinstall.dev" }
 curl.exe -s -A "windows" "$Env:WEBI_HOST/packages/webi/webi-pwsh.ps1" -o "$Env:USERPROFILE\.local\bin\webi-pwsh.ps1"
 Set-ExecutionPolicy -Scope Process Bypass
 & "$Env:USERPROFILE\.local\bin\webi-pwsh.ps1" "{{ exename }}"

_webi/curl-pipe-bootstrap.tpl.sh

@@ -267,23 +267,12 @@ webi_bootstrap() { (
 fn_checksum() {
     a_filepath="${1}"
 
-    if command -v sha1sum > /dev/null; then
-        sha1sum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
-        return 0
-    fi
-
+    cmd_shasum='sha1sum'
     if command -v shasum > /dev/null; then
-        shasum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
-        return 0
+        cmd_shasum='shasum'
     fi
 
-    if command -v sha1 > /dev/null; then
-        sha1 "${a_filepath}" | cut -d'=' -f2 | cut -c 2-9
-        return 0
-    fi
-
-    echo >&2 "    warn: no sha1 sum program"
-    date '+%F %H:%M'
+    $cmd_shasum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
 }
 
 ##############################################

_webi/installers.js

@@ -16,7 +16,7 @@ var BAD_SH_RE = /[<>'"`$\\]/;
 Installers.renderBash = async function (
   pkgdir,
   rel,
-  { baseurl, pkg, tag, ver, os = '', arch = '', libc = '', formats },
+  { baseurl, pkg, tag, ver, os = '', arch = '', libc = '', formats, latest },
 ) {
   if (!Array.isArray(formats)) {
     formats = [];
@@ -93,18 +93,18 @@ Installers.renderBash = async function (
     ['WEBI_GIT_TAG', rel.git_tag], // TODO replace with branch
     ['WEBI_LTS', rel.lts],
     ['WEBI_CHANNEL', rel.channel],
-    ['WEBI_EXT', rel.ext],
+    ['WEBI_EXT', rel.ext.replace(/tar.*/, 'tar')],
     ['WEBI_FORMATS', formats.join(',')],
     ['WEBI_PKG_URL', rel.download],
     ['WEBI_PKG_PATHNAME', pkgFile],
     ['WEBI_PKG_FILE', pkgFile], // TODO replace with pathname
     ['PKG_NAME', pkg],
-    ['PKG_STABLE', rel.stable],
-    ['PKG_LATEST', rel.latest],
     ['PKG_OSES', (rel.oses || []).join(' ')],
     ['PKG_ARCHES', (rel.arches || []).join(' ')],
     ['PKG_LIBCS', (rel.libcs || []).join(' ')],
     ['PKG_FORMATS', (rel.formats || []).join(' ')],
+    ['PKG_CHANNELS', (rel.channels || []).join(' ')],
+    ['PKG_LATEST', latest],
   ];
 
   for (let env of envReplacements) {

_webi/lint-builds.js

@@ -125,7 +125,6 @@ let bc = BuildsCacher.create({
 });
 
 async function main() {
-  /* jshint maxcomplexity: 25 */
   // TODO
   //     node ./_webi/lint-builds.js caddy@beta 'x86_64/unknown Darwin libc'
   //
@@ -139,10 +138,19 @@ async function main() {
     console.info('');
   }
 
+  bc.freshenRandomPackage(600 * 1000);
+
   let rows = [];
   let triples = [];
   let valids = Object.keys(dirs.valid);
 
+  let index = valids.indexOf('webi');
+  if (index >= 0) {
+    // TODO fix the webi faux package
+    // (not sure why I even created it)
+    void valids.splice(index, 1);
+  }
+
   if (projName) {
     if (!valids.includes(projName)) {
       throw new Error(`'${projName}' is not a valid installable project`);
@@ -183,7 +191,6 @@ async function main() {
         // non-build file
         continue;
       }
-
       if (target.error) {
         let e = target.error;
         if (e.code === 'E_BUILD_NO_PATTERN') {
@@ -195,12 +202,6 @@ async function main() {
         throw e;
       }
 
-      if (target.unknownTerms?.length) {
-        let msg = `${projInfo.name}: unrecognized term(s) '${target.unknownTerms}' in '${build.download}'`;
-        let err = new Error(msg);
-        throw err;
-      }
-
       triples.push(target.triplet);
       // if (!build.version) {
       //   throw new Error(`no version for ${pkg.name} ${build.name}`);

_webi/normalize.js

@@ -0,0 +1,258 @@
+'use strict';
+
+// this may need customizations between packages
+var osMap = {
+  macos: /(\b|_)(apple|os(\s_-)?x\b|mac|darwin|iPhone|iOS|iPad)/i,
+  linux: /(\b|_)(linux)/i,
+  freebsd: /(\b|_)(freebsd)/i,
+  windows: /(\b|_)(win|microsoft|msft)/i,
+  sunos: /(\b|_)(sun)/i,
+  aix: /(\b|_)(aix)/i,
+};
+
+var maps = {
+  oses: {},
+  arches: {},
+  libcs: {},
+  formats: {},
+};
+
+Object.keys(osMap).forEach(function (name) {
+  maps.oses[name] = true;
+});
+
+var formats = ['zip', 'xz', 'tar', 'pkg', 'msi', 'git', 'exe', 'dmg', 'git'];
+formats.forEach(function (name) {
+  maps.formats[name] = true;
+});
+
+// evaluation order matters
+// (i.e. otherwise x86 and x64 can cross match)
+var arches = [
+  // arm64/aarch64 has very high specificity, so it comes first
+  'arm64',
+  // arm 7 is also generic aarch/arm/arm32
+  'armv7l',
+  // arm6 can run on armv7
+  'armv6l',
+  // amd64 is more likely and less often specified than arm64
+  'amd64',
+  'x86',
+  'ppc64le',
+  'ppc64',
+  's390x',
+];
+// Used for detecting system arch from package download url, for example:
+//
+// https://git.com/org/foo/releases/v0.7.9/foo-aarch64-linux-musl.tar.gz
+// https://git.com/org/foo/releases/v0.7.9/foo-arm-linux-musleabihf.tar.gz
+// https://git.com/org/foo/releases/v0.7.9/foo-armv7-linux-musleabihf.tar.gz
+// https://git.com/org/foo/releases/v0.7.9/foo-x86_64-linux-musl.tar.gz
+//
+var archMap = {
+  arm64: /(\b|_)(aarch64|arm64)/i,
+  armv7l: /(\b|_)(arm32|arm[_\-]?v?7l?)/i,
+  armv6l: /(\b|_)(arm|aarch32|arm[_\-]?v?6l?)(\b|_)/i,
+  //amd64: /(amd.?64|x64|[_\-]64)/i,
+  amd64:
+    /(\b|_|amd|(dar)?win(dows)?|mac(os)?|linux|osx|x)64([_\-]?bit)?(\b|_)/i,
+  //x86: /(86)(\b|_)/i,
+  x86: /(\b|_|amd|(dar)?win(dows)?|mac(os)?|linux|osx|x)(86|32)([_\-]?bit)(\b|_)/i,
+  ppc64le: /(\b|_)(ppc64le)/i,
+  ppc64: /(\b|_)(ppc64)(\b|_)/i,
+  s390x: /(\b|_)(s390x)/i,
+};
+arches.forEach(function (name) {
+  maps.arches[name] = true;
+});
+
+var libcs = ['none', 'musl', 'gnu', 'msvc', 'libc'];
+libcs.forEach(function (name) {
+  maps.libcs[name] = true;
+});
+
+function normalize(all) {
+  /* jshint maxcomplexity:50 */
+  /* jshint maxdepth:10 */
+  var supported = {
+    oses: {},
+    arches: {},
+    libcs: {},
+    formats: {},
+  };
+
+  for (let rel of all.releases) {
+    rel.version = rel.version.replace(/^v/i, '');
+
+    if (!rel.name) {
+      rel.name = rel.download.replace(/.*\//, '');
+    }
+
+    if (!rel.os) {
+      rel.os = 'unknown';
+
+      let osNames = Object.keys(osMap);
+      for (let osName of osNames) {
+        let relName = rel.name || rel.download;
+        let osRegExp = osMap[osName];
+        let matches = osRegExp.test(relName);
+        if (matches) {
+          rel.os = osName;
+          break;
+        }
+      }
+    }
+    supported.oses[rel.os] = true;
+
+    if (!rel.arch) {
+      for (let arch of arches) {
+        let name = rel.name || rel.download;
+        let isArch = name.match(archMap[arch]);
+        if (isArch) {
+          rel.arch = arch;
+          break;
+        }
+      }
+    }
+    if (!rel.arch) {
+      if ('macos' === rel.os) {
+        rel.arch = 'amd64';
+      }
+    }
+    supported.arches[rel.arch] = true;
+
+    // note: depends on rel.os
+    if (!rel.libc) {
+      let isMusl;
+      let isMsvc;
+      let isStatic;
+      let isGnu;
+
+      // extra blocks to prevent copy pasta errors
+
+      {
+        let muslRe = /(\b|_)(musl)(\b|_)/i;
+        isMusl = muslRe.test(rel.download) || muslRe.test(rel.name);
+      }
+
+      {
+        let msvcRe = /(\b|_)(msvc)(\b|_)/i;
+        isMsvc = msvcRe.test(rel.download) || msvcRe.test(rel.name);
+      }
+
+      {
+        let staticRe = /(\b|_)(static)(\b|_)/i;
+        isStatic = staticRe.test(rel.download) || staticRe.test(rel.name);
+      }
+
+      {
+        let gnuRe = /(\b|_)(gnu|glibc|libc)(\b|_)/i;
+        isGnu = gnuRe.test(rel.download) || gnuRe.test(rel.name);
+      }
+
+      if (isMusl) {
+        // we specifically tag things that need musl++ in their own releases
+        rel.libc = 'none';
+      } else if (isStatic) {
+        rel.libc = 'none';
+      } else if (isGnu) {
+        rel.libc = 'gnu';
+        if (rel.os === 'windows') {
+          // windows gnu is static
+          rel.libc = 'none';
+        } else if (rel.os === 'darwin') {
+          // if glibc is required on macos, it'll be static
+          rel.libc = 'none';
+        }
+      } else if (isMsvc) {
+        rel.libc = 'msvc';
+      } else {
+        // The default is no requirement for any particular libc
+        // (Go, Zig, POSIX Shell, JS, etc)
+        // and hopefully we never have to worry about mingw and friends
+        rel.libc = 'none';
+      }
+    }
+    supported.libcs[rel.libc] = true;
+
+    var tarExt;
+    if (!rel.ext) {
+      // pkg-v1.0.tar.gz => ['gz', 'tar', '0', 'pkg-v1']
+      // pkg-v1.0.tar => ['tar', '0' ,'pkg-v1']
+      // pkg-v1.0.zip => ['zip', '0', 'pkg-v1']
+      var exts = (rel.name || rel.download).split('.');
+      if (1 === exts.length) {
+        // for bare releases in the format of foo-linux-amd64
+        rel.ext = 'exe';
+      }
+      exts = exts.reverse().slice(0, 2);
+      if ('tar' === exts[1]) {
+        rel.ext = exts.reverse().join('.');
+        tarExt = 'tar';
+      } else if ('tgz' === exts[0]) {
+        rel.ext = 'tar.gz';
+        tarExt = 'tar';
+      } else {
+        rel.ext = exts[0];
+      }
+      if (/\-|linux|mac|os[_\-]?x|arm|amd|86|64|mip/i.test(rel.ext)) {
+        // for bare releases in the format of foo.linux-amd64
+        rel.ext = 'exe';
+      }
+    }
+    supported.formats[tarExt || rel.ext] = true;
+
+    if (!rel.channel) {
+      // basically like this: (+.-_)(beta|rc)(0-9)(+.-_)
+      // matches:
+      //   - v1.0-beta
+      //   - v1.0-beta1.1
+      //   - v1.0-beta-11
+      // won't match:
+      //   - v1.0beta
+      //   - v1.0-beta1b
+      let isBetaRe = /(\b|_)(preview|rc|beta|alpha)(\d+)(\b|_)/;
+      let isBeta = isBetaRe.test(rel.name);
+      if (isBeta) {
+        rel.channel = 'beta';
+      } else {
+        rel.channel = 'stable';
+      }
+    }
+
+    if (all.download) {
+      rel.download = all.download.replace(/{{ download }}/, rel.download);
+    }
+  }
+
+  all.oses = Object.keys(supported.oses).filter(function (name) {
+    return maps.oses[name];
+  });
+  all.arches = Object.keys(supported.arches).filter(function (name) {
+    return maps.arches[name];
+  });
+  all.libcs = Object.keys(supported.libcs).filter(function (name) {
+    return maps.libcs[name];
+  });
+  all.formats = Object.keys(supported.formats).filter(function (name) {
+    return maps.formats[name];
+  });
+
+  return all;
+}
+
+module.exports = normalize;
+module.exports._debug = function (all) {
+  all = normalize(all);
+  all.releases = all.releases
+    .filter(function (r) {
+      return ['windows', 'macos', 'linux'].includes(r.os) && 'amd64' === r.arch;
+    })
+    .slice(0, 10);
+  return all;
+};
+// NOT in order of priority (which would be tar, xz, zip, ...)
+module.exports.formats = formats;
+module.exports.arches = arches;
+module.exports.libcs = libcs;
+module.exports.formatsMap = maps.formats;

_webi/package-install.tpl.ps1

@@ -45,15 +45,15 @@ $TDim = "${Esc}[2m"
 $TReset = "${Esc}[0m"
 
 function Invoke-DownloadUrl {
-    param (
+    Param (
         [string]$URL,
         [string]$Params,
         [string]$Path,
         [switch]$Force
     )
 
-    if (Test-Path -Path "$Path") {
-        if (-not $Force.IsPresent) {
+    IF (Test-Path -Path "$Path") {
+        IF (-Not $Force.IsPresent) {
             Write-Host "    ${TDim}Found${TReset} $Path"
             return
         }
@@ -65,7 +65,7 @@ function Invoke-DownloadUrl {
 
     Write-Host "    Downloading ${TDim}from${TReset}"
     Write-Host "      ${TDim}${URL}${TReset}"
-    if ($Params.Length -ne 0) {
+    IF ($Params.Length -ne 0) {
         Write-Host "        ?$Params"
         $URL = "${URL}?${Params}"
     }
@@ -80,18 +80,18 @@ function Get-UserAgent {
     # This is the canonical CPU arch when the process is emulated
     $my_arch = "$Env:PROCESSOR_ARCHITEW6432"
 
-    if ($my_arch -eq $null -or $my_arch -eq "") {
+    IF ($my_arch -eq $null -or $my_arch -eq "") {
         # This is the canonical CPU arch when the process is native
         $my_arch = "$Env:PROCESSOR_ARCHITECTURE"
     }
 
-    if ($my_arch -eq "AMD64") {
+    IF ($my_arch -eq "AMD64") {
         # Because PowerShell is sometimes AMD64 on Windows 10 ARM
         # See https://oofhours.com/2020/02/04/powershell-on-windows-10-arm64/
-        $my_os_arch = (Get-CimInstance -ClassName Win32_OperatingSystem).OSArchitecture
+        $my_os_arch = wmic os get osarchitecture
 
         # Using -clike because of the trailing newline
-        if ($my_os_arch -clike "ARM 64*") {
+        IF ($my_os_arch -clike "ARM 64*") {
             $my_arch = "ARM64"
         }
     }
@@ -124,7 +124,7 @@ function webi_path_add($pathname) {
             $exists_in_path = $true
         }
     }
-    if (-not $exists_in_path) {
+    if (-Not $exists_in_path) {
         $all_user_paths = "${pathname};${all_user_paths}".Trim(';')
         [Environment]::SetEnvironmentVariable("Path", $all_user_paths, "User")
         $null = Sync-EnvPath

_webi/package-install.tpl.sh

@@ -27,7 +27,6 @@ __bootstrap_webi() {
     #PKG_LIBCS=
     #PKG_FORMATS=
     #PKG_LATEST=
-    #PKG_STABLE=
     WEBI_PKG_DOWNLOAD=""
     WEBI_DOWNLOAD_DIR="${HOME}/Downloads"
     if command -v xdg-user-dir > /dev/null; then
@@ -37,7 +36,7 @@ __bootstrap_webi() {
         fi
     fi
 
-    WEBI_PKG_PATH="${WEBI_DOWNLOAD_DIR}/webi/${PKG_NAME:-error}/${WEBI_VERSION:-stable}"
+    WEBI_PKG_PATH="${WEBI_DOWNLOAD_DIR}/webi/${PKG_NAME:-error}/${WEBI_VERSION:-latest}"
 
     # get the special formatted version
     # (i.e. "go is go1.14" while node is "node v12.10.8")
@@ -128,10 +127,7 @@ __bootstrap_webi() {
             echo ""
             echo "    $(t_err "Error: no '${PKG_NAME:-"Unknown Package"}@${WEBI_TAG:-"Unknown Tag"}' release for '${WEBI_OS:-"Unknown OS"}' (${WEBI_LIBC:-"Unknown Libc"}) on '${WEBI_ARCH:-"Unknown CPU"}' as one of '${WEBI_FORMATS:-"Unknown File Type"}'")"
             echo ""
-            echo "        Latest Stable: ${PKG_STABLE}"
-            if test "${PKG_LATEST}" != "${PKG_STABLE}"; then
-                echo "        Next Version: ${PKG_LATEST}"
-            fi
+            echo "        Latest Version: ${PKG_LATEST}"
             echo "        CPUs: $PKG_ARCHES"
             echo "        OSes: $PKG_OSES"
             echo "        libcs: $PKG_LIBCS"
@@ -201,32 +197,20 @@ __bootstrap_webi() {
         my_dl_rel="$(
             fn_sub_home "${WEBI_PKG_PATH}/${WEBI_PKG_FILE}"
         )"
-        if test "$WEBI_EXT" = "tar.zst"; then
-            echo "    Extracting $(t_path "${my_dl_rel}")"
-            unzstd -c --keep "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" | tar xf -
-        elif test "$WEBI_EXT" = "tar.xz"; then
-            echo "    Extracting $(t_path "${my_dl_rel}")"
-            unxz -c -k "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" | tar xf -
-        elif test "$WEBI_EXT" = "tar.gz"; then
-            echo "    Extracting $(t_path "${my_dl_rel}")"
-            tar xzf "${WEBI_PKG_PATH}/$WEBI_PKG_FILE"
-        elif test "$WEBI_EXT" = "tar.bz2"; then
-            echo "    Extracting $(t_path "${my_dl_rel}")"
-            tar xjf "${WEBI_PKG_PATH}/$WEBI_PKG_FILE"
-        elif test "$WEBI_EXT" = "tar"; then
+        if [ "tar" = "$WEBI_EXT" ]; then
             echo "    Extracting $(t_path "${my_dl_rel}")"
             tar xf "${WEBI_PKG_PATH}/$WEBI_PKG_FILE"
-        elif test "$WEBI_EXT" = "zip" || test "$WEBI_EXT" = "app.zip"; then
+        elif [ "zip" = "$WEBI_EXT" ]; then
             echo "    Extracting $(t_path "${my_dl_rel}")"
             unzip "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" > __unzip__.log
-        elif test "$WEBI_EXT" = "exe"; then
+        elif [ "exe" = "$WEBI_EXT" ]; then
             echo "    Moving $(t_path "${my_dl_rel}")"
             echo "      to $(t_path "$(fn_sub_home "$(pwd)")")"
             mv "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" .
-        elif test "$WEBI_EXT" = "git"; then
+        elif [ "git" = "$WEBI_EXT" ]; then
             echo "    Moving $(t_path "${my_dl_rel}")"
             mv "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" .
-        elif test "$WEBI_EXT" = "xz"; then
+        elif [ "xz" = "$WEBI_EXT" ]; then
             echo "    Inflating $(t_path "${my_dl_rel}")"
             unxz -c "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" > "$(basename "$WEBI_PKG_FILE")"
         else
@@ -239,6 +223,8 @@ __bootstrap_webi() {
     webi_path_add() {
         my_path="${1}"
 
+        fn_envman_init
+
         # \v was chosen as it is extremely unlikely for a filename
         # \1 could be an even better choice, but needs more testing.
         # (currently tested working on: linux & mac)
@@ -401,6 +387,7 @@ __bootstrap_webi() {
     export _webi_tmp="${_webi_tmp:-"$HOME/.local/opt/webi-tmp.d"}"
 
     mkdir -p "${WEBI_PKG_PATH}"
+    mkdir -p "$HOME/.local/bin"
     mkdir -p "$HOME/.local/opt"
 
     if test -e ~/.local/bin; then
@@ -409,7 +396,6 @@ __bootstrap_webi() {
         echo "    Creating$(t_path ' ~/.local/bin')"
         mkdir -p "$HOME/.local/bin"
     fi
-    fn_envman_init
 
     ##
     ##
@@ -790,23 +776,12 @@ webi_upgrade() { (
 fn_checksum() {
     a_filepath="${1}"
 
-    if command -v sha1sum > /dev/null; then
-        sha1sum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
-        return 0
-    fi
-
+    cmd_shasum='sha1sum'
     if command -v shasum > /dev/null; then
-        shasum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
-        return 0
+        cmd_shasum='shasum'
     fi
 
-    if command -v sha1 > /dev/null; then
-        sha1 "${a_filepath}" | cut -d'=' -f2 | cut -c 2-9
-        return 0
-    fi
-
-    echo >&2 "    warn: no sha1 sum program"
-    date '+%F %H:%M'
+    $cmd_shasum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
 }
 
 ##############################################

_webi/serve-installer.js

@@ -70,20 +70,31 @@ InstallerServer.helper = async function ({
 
   console.log(`dbg: Get Project Installer Type for '${projectName}':`);
   let proj = await Builds.getProjectType(projectName);
-  if (proj.type === 'alias') {
-    console.log(`dbg: alias`, proj);
-    projectName = proj.detail;
-    proj = await Builds.getProjectType(projectName); // an alias should never resolve to an alias
-  }
-  console.log(`dbg: proj`, proj);
+  console.log(proj);
 
-  let validTypes = ['selfhosted', 'valid'];
+  let validTypes = ['alias', 'selfhosted', 'valid'];
   if (!validTypes.includes(proj.type)) {
-    let msg = `'${projectName}' doesn't have an installer: '${proj.type}': '${proj.detail}'`;
-    let err = new Error(msg);
-    err.code = 'ENOENT';
-    throw err;
+    throw new Error(
+      `'${projectName}' doesn't have an installer: '${proj.type}': '${proj.detail}'`,
+    );
   }
+  if (proj.type === 'alias') {
+    projectName = proj.detail;
+  }
+
+  let projInfo = await Builds.getPackage({
+    name: projectName,
+    date: new Date(),
+  });
+  //console.log('projInfo', projInfo);
+
+  let buildTargetInfo = {
+    triplets: projInfo.triplets,
+    oses: projInfo.oses,
+    arches: projInfo.arches,
+    libcs: projInfo.libcs,
+    formats: projInfo.formats,
+  };
 
   let tmplParams = {
     pkg: projectName,
@@ -94,6 +105,8 @@ InstallerServer.helper = async function ({
     formats: hostFormats,
     limit: 1,
   };
+  let latest = projInfo.versions[0];
+  Object.assign(tmplParams, { latest });
   Object.assign(tmplParams, releaseTarget);
   console.log('tmplParams', tmplParams);
 
@@ -114,44 +127,7 @@ InstallerServer.helper = async function ({
       "Check query parameters. Should be something like '/api/releases/{package}@{version}.tab?os={macos|linux|windows|-}&arch={amd64|x86|aarch64|arm64|armv7l|-}&libc={musl|gnu|msvc|libc|static}&limit=10'",
   };
 
-  if (proj.type === 'selfhosted') {
-    return [errPackage, tmplParams];
-  }
-
-  let projInfo = await Builds.getPackage({
-    name: projectName,
-    date: new Date(),
-  });
-  let latestVersions = Builds.enumerateLatestVersions(projInfo);
-  //console.log('projInfo', projInfo);
-
-  let buildTargetInfo = {
-    triplets: projInfo.triplets,
-    oses: projInfo.oses,
-    arches: projInfo.arches,
-    libcs: projInfo.libcs,
-    formats: projInfo.formats,
-    latest: latestVersions.latest,
-    stable: latestVersions.stable,
-  };
-
-  // TODO .findMatchingPackages() should probably account for this
   let hasOs = projInfo.oses.includes(hostTarget.os);
-  let maybePosix = !hasOs && hostTarget.os !== 'windows';
-  if (maybePosix) {
-    let posixes = ['posix_2017', 'posix_2024'];
-    for (let posixYear of posixes) {
-      let hasPosix = projInfo.oses.includes(posixYear);
-      if (hasPosix) {
-        hasOs = true;
-        break;
-      }
-    }
-  }
-  if (!hasOs) {
-    hasOs = projInfo.oses.includes('ANYOS');
-  }
-
   if (!hasOs) {
     let pkg1 = Object.assign(buildTargetInfo, errPackage);
     return [pkg1, tmplParams];
@@ -171,6 +147,11 @@ InstallerServer.helper = async function ({
     return [pkg1, tmplParams];
   }
 
+  if (!targetRelease.packages) {
+    let pkg1 = Object.assign(buildTargetInfo, errPackage);
+    return [pkg1, tmplParams];
+  }
+
   let buildPkg = Builds.selectPackage(targetRelease.packages, hostFormats);
   let ext = buildPkg.ext || '.exe';
   if (ext.startsWith('.')) {

_webi/test.js

@@ -32,7 +32,7 @@ if (/\b-?-h(elp)?\b/.test(process.argv.join(' '))) {
 var os = require('os');
 var fs = require('fs');
 var path = require('path');
-var Builds = require('./builds.js');
+var Releases = require('./transform-releases.js');
 var Installers = require('./installers.js');
 var ServeInstaller = require('./serve-installer.js');
 
@@ -49,7 +49,7 @@ var baseurl = 'https://webinstall.dev';
 var maxLen = 0;
 console.info('');
 console.info('Has the necessary files?');
-['README.md', 'install.sh', 'install.ps1']
+['README.md', 'releases.js', 'install.sh', 'install.ps1']
   .map(function (node) {
     maxLen = Math.max(maxLen, node.length);
     return node;
@@ -65,8 +65,7 @@ console.info('Has the necessary files?');
   });
 
 console.info('');
-let projName = pkgdir.split('/').filter(Boolean).pop();
-Builds.getPackage({ name: projName }).then(async function (/*projInfo*/) {
+Releases.get(path.join(process.cwd(), pkgdir)).then(async function (all) {
   var pkgname = path.basename(pkgdir.replace(/\/$/, ''));
   var nodeOs = os.platform();
   var nodeOsRelease = os.release();
@@ -83,8 +82,8 @@ Builds.getPackage({ name: projName }).then(async function (/*projInfo*/) {
   var formats = ['exe', 'xz', 'tar', 'zip', 'git'];
 
   let [rel, opts] = await ServeInstaller.helper({
-    unameAgent: `${nodeOs}/${nodeOsRelease} ${nodeArch}/unknown ${nodeLibc}`,
-    projectName: pkgname,
+    ua: `${nodeOs}/${nodeOsRelease} ${nodeArch}/unknown ${nodeLibc}`,
+    pkg: pkgname,
     tag: pkgtag || '',
     formats: formats,
     libc: nodeLibc,

_webi/transform-releases.js

@@ -2,30 +2,72 @@
 
 var Releases = module.exports;
 
-var Fs = require('node:fs/promises');
-var Os = require('node:os');
 var path = require('path');
+var request = require('@root/request');
+var _normalize = require('./normalize.js');
+
 var cache = {};
+//var staleAge = 5 * 1000;
+//var expiredAge = 15 * 1000;
+var staleAge = 5 * 60 * 1000;
+var expiredAge = 15 * 60 * 1000;
 
-var LEGACY_CACHE_DIR = path.join(Os.homedir(), '.cache/webi/legacy');
+let installerDir = path.join(__dirname, '..');
 
-// Sort releases by ext preference and libc within the same version.
-// The cache is already sorted by version (stable before beta, newest first),
-// so we only re-order within the same version string.
+Releases.get = async function (pkgdir) {
+  let get;
+  try {
+    get = require(path.join(pkgdir, 'releases.js'));
+  } catch (e) {
+    let err = new Error('no releases.js for', pkgdir.split(/[\/\\]+/).pop());
+    err.code = 'E_NO_RELEASE';
+    throw err;
+  }
+
+  let all = await get(request);
+
+  return _normalize(all);
+};
+
+// TODO needs a proper test, and more accurate (though perhaps far less simple) code
 function createFormatsSorter(formats) {
-  return function sortByExtLibc(a, b) {
-    if (a.version !== b.version) {
-      // Array.sort is stable (V8, ES2019), so returning 0 across
-      // versions preserves the cache's pre-sorted version-desc order.
-      return 0;
+  return function sortByVerExt(a, b) {
+    function lexver(semver) {
+      // v1.20.156 => 00001.00020.00156.zzzzz
+      // TODO BUG: v1.20.156-rc2 => 00001.00020.00156.rc2zz
+      var parts = semver.split(/[+\.\-]/g);
+      while (parts.length < 4) {
+        parts.push('');
+      }
+      return parts
+        .map(function (num, i) {
+          if (3 === i) {
+            return num.toString().padEnd(10, 'z');
+          }
+          return num.toString().padStart(10, '0');
+        })
+        .join('.');
+    }
+
+    var aver = lexver(a.version);
+    var bver = lexver(b.version);
+    if (aver > bver) {
+      //console.log(aver, '>', bver);
+      return -1;
+    }
+    if (aver < bver) {
+      //console.log(aver, '<', bver);
+      return 1;
     }
 
     var aExtPri = formats.indexOf(a.ext.replace(/tar\..*/, 'tar'));
     var bExtPri = formats.indexOf(b.ext.replace(/tar\..*/, 'tar'));
     if (aExtPri > bExtPri) {
+      //console.log(a.ext, aExtPri, '>', b.ext, bExtPri);
       return -1;
     }
     if (aExtPri < bExtPri) {
+      //console.log(a.ext, aExtPri, '<', b.ext, bExtPri);
       return 1;
     }
 
@@ -42,39 +84,99 @@ function createFormatsSorter(formats) {
 }
 
 async function getCachedReleases(pkg) {
-  // returns { download: '', releases: [{ version, date, os, arch, lts, channel, download}] }
+  // returns { download: '<template string>', releases: [{ version, date, os, arch, lts, channel, download}] }
 
-  if (cache[pkg]) {
-    return cache[pkg];
+  async function chainCachePromise(fn) {
+    cache[pkg].promise = cache[pkg].promise.then(fn);
+    return cache[pkg].promise;
   }
 
-  let dataFile = `${LEGACY_CACHE_DIR}/${pkg}.json`;
+  async function sleep(ms) {
+    return await new Promise(function (resolve, reject) {
+      setTimeout(resolve, ms);
+    });
+  }
 
-  let json = await Fs.readFile(dataFile, 'utf8').catch(function (err) {
-    if (err.code === 'ENOENT') {
-      return null;
+  async function putCache() {
+    var age = Date.now() - cache[pkg].updatedAt;
+    if (age < staleAge) {
+      //console.debug('NOT STALE ANYMORE - updated in previous promise');
+      return cache[pkg].all;
     }
-    throw err;
-  });
 
-  if (!json) {
-    let empty = { download: '', releases: [] };
-    cache[pkg] = empty;
-    return empty;
+    //console.debug('DOWNLOADING NEW "%s" releases', pkg);
+    var pkgdir = path.join(installerDir, pkg);
+
+    // workaround for request timeout seeming to not work
+    let complete = false;
+    await Promise.race([
+      Releases.get(pkgdir)
+        .catch(function (err) {
+          if ('E_NO_RELEASE' === err.code) {
+            let all = { _error: 'E_NO_RELEASE', download: '', releases: [] };
+            return all;
+          }
+
+          throw err;
+        })
+        .catch(function (err) {
+          let hasReleases = cache[pkg].all?.releases?.length > 1;
+          if (!hasReleases) {
+            throw err;
+          }
+
+          console.error(`Error: the BOOGEYMAN got us!`);
+          console.error(err.stack);
+
+          return cache[pkg].all;
+        })
+        .then(function (all) {
+          // Note: it is possible for slightly older data
+          // to replace slightly newer data, but this is better
+          // than being in a cycle where release updates _always_
+          // take longer than expected.
+          //console.debug('DOWNLOADED NEW "%s" releases', pkg);
+          cache[pkg].updatedAt = Date.now();
+          cache[pkg].all = all;
+          complete = true;
+        }),
+      sleep(15000).then(function () {
+        if (complete) {
+          return;
+        }
+        console.error(`request timeout waiting for '${pkg}' release info`);
+      }),
+    ]);
+
+    return cache[pkg].all;
   }
 
-  let all;
-  try {
-    all = JSON.parse(json);
-  } catch (e) {
-    console.error(`error: ${dataFile}:\n\t${e.message}`);
-    let empty = { download: '', releases: [] };
-    cache[pkg] = empty;
-    return empty;
+  if (!cache[pkg]) {
+    cache[pkg] = {
+      updatedAt: 0,
+      all: { download: '', releases: [] },
+      promise: Promise.resolve(),
+    };
   }
 
-  cache[pkg] = all;
-  return all;
+  var bgRenewal;
+  var age = Date.now() - cache[pkg].updatedAt;
+  var fresh = age < staleAge;
+  if (!fresh) {
+    bgRenewal = chainCachePromise(putCache);
+  }
+
+  var tooStale = age > expiredAge;
+  if (!tooStale) {
+    return await cache[pkg].all;
+  }
+
+  return await Promise.race([
+    bgRenewal,
+    sleep(5000).then(function () {
+      return cache[pkg].all;
+    }),
+  ]);
 }
 
 async function filterReleases(
@@ -85,22 +187,15 @@ async function filterReleases(
   // sort the most compatible format first
   // (i.e. so that we don't do .pkg on linux except on purpose)
   var rformats = formats.slice(0).reverse();
-  var sortByExtLibc = createFormatsSorter(rformats);
+  var sortByVerExt = createFormatsSorter(rformats);
   var reVer = new RegExp('^' + ver + '\\b');
 
   function selectMatches(rel) {
-    /* jshint maxcomplexity: 25 */
     if (os) {
-      // '*' = any OS (matches anything, including windows).
-      // 'posix' / 'posix_20xx' = any POSIX OS (matches linux, macos,
-      // freebsd, etc., but NOT windows).
-      let isPosix = rel.os === 'posix' || rel.os.startsWith('posix_20');
-      let osMatches =
-        rel.os === '*' ||
-        rel.os === os ||
-        (isPosix && os !== 'windows');
-      if (!osMatches) {
-        return false;
+      if (rel.os !== '*') {
+        if (rel.os !== os) {
+          return false;
+        }
       }
     }
 
@@ -112,10 +207,7 @@ async function filterReleases(
       }
     }
 
-    // libc='libc' is serve-releases.js's default when the caller
-    // didn't pin one — treat it as 'no preference', not a filter.
-    let isMeaningfulLibc = libc && libc !== 'libc' && rel.libc !== 'none';
-    if (isMeaningfulLibc) {
+    if (rel.libc !== 'none') {
       let releaseRequiresMusl = rel.libc === 'musl';
       // goal: handle non-glibc (Alpine / Docker / musl)
       let osHasMusl = libc === 'musl';
@@ -162,7 +254,7 @@ async function filterReleases(
     return true;
   }
 
-  var sortedRels = all.releases.filter(selectMatches).sort(sortByExtLibc);
+  var sortedRels = all.releases.filter(selectMatches).sort(sortByVerExt);
   //console.log(sortedRels.slice(0, 4));
 
   return sortedRels.slice(0, limit || 1000);
@@ -321,8 +413,8 @@ Releases.getReleases = function ({
 };
 
 if (require.main === module) {
-  return Releases
-    .getReleases({
+  return module
+    .exports({
       pkg: 'node',
       ver: '',
       os: 'macos',

aliasman/README.md

@@ -72,8 +72,8 @@ gc "feat: new feature"
 
 ### Common aliases
 
-Use *alias*es to make other tools you find around webi even _more_ convenient ⚡️
-(and powerful 💪).
+Use *alias*es to make other tools you find around webi even _more_ convenient
+⚡️ (and powerful 💪).
 
 ```sh
 aliasman curl 'curlie'

aliasman/releases.conf

@@ -1,2 +0,0 @@
-github_sources = BeyondCodeBootcamp/aliasman
-git_url = https://github.com/BeyondCodeBootcamp/aliasman.git

aliasman/releases.js

@@ -0,0 +1,30 @@
+'use strict';
+
+var githubSource = require('../_common/github-source.js');
+var owner = 'BeyondCodeBootcamp';
+var repo = 'aliasman';
+
+module.exports = function (request) {
+  let arches = [
+    'amd64',
+    'arm64',
+    'armv6l',
+    'armv7l',
+    'ppc64le',
+    'ppc64',
+    's390x',
+    'x86',
+  ];
+  let oses = ['freebsd', 'linux', 'macos', 'posix'];
+  return githubSource(request, owner, repo, oses, arches).then(function (all) {
+    all._names = ['aliasman', 'legacy'];
+    return all;
+  });
+};
+
+if (module === require.main) {
+  module.exports(require('@root/request')).then(function (all) {
+    all = require('../_webi/normalize.js')(all);
+    console.info(JSON.stringify(all, null, 2));
+  });
+}

arc/install.ps1

@@ -19,13 +19,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading archiver from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-if (!(Test-Path -Path "$pkg_src_cmd")) {
+IF (!(Test-Path -Path "$pkg_src_cmd")) {
     Write-Output "Installing archiver"
 
     # TODO: create package-specific temp directory

arc/releases.conf

@@ -1 +0,0 @@
-github_releases = mholt/archiver

arc/releases.js

@@ -0,0 +1,21 @@
+'use strict';
+
+var github = require('../_common/github.js');
+var owner = 'mholt';
+var repo = 'archiver';
+
+module.exports = function (request) {
+  return github(request, owner, repo).then(function (all) {
+    all._names = ['archiver', 'arc'];
+    return all;
+  });
+};
+
+if (module === require.main) {
+  module.exports(require('@root/request')).then(function (all) {
+    all = require('../_webi/normalize.js')(all);
+    // just select the first 5 for demonstration
+    all.releases = all.releases.slice(0, 5);
+    console.info(JSON.stringify(all, null, 2));
+  });
+}

archiver/install.ps1

@@ -1,5 +1,5 @@
 #!/bin/pwsh
 
 Write-Output "'archiver@$Env:WEBI_TAG' is an alias for 'arc@$Env:WEBI_VERSION'"
-if ($null -eq $Env:WEBI_HOST -or $Env:WEBI_HOST -eq "") { $Env:WEBI_HOST = "https://webinstall.dev" }
+IF ($null -eq $Env:WEBI_HOST -or $Env:WEBI_HOST -eq "") { $Env:WEBI_HOST = "https://webinstall.dev" }
 curl.exe -A MS -fsSL "$Env:WEBI_HOST/arc@$Env:WEBI_VERSION" | powershell

atomicparsley/install.ps1

@@ -20,7 +20,7 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Checking for (or Installing) MSVC Runtime..."
     & "$Env:USERPROFILE\.local\bin\webi-pwsh.ps1" vcruntime
 
@@ -29,7 +29,7 @@ if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-if (!(Test-Path -Path "$pkg_src_cmd")) {
+IF (!(Test-Path -Path "$pkg_src_cmd")) {
     Write-Output "Installing AtomicParsley"
 
     # TODO: create package-specific temp directory

atomicparsley/releases.conf

@@ -1 +0,0 @@
-github_releases = wez/atomicparsley

atomicparsley/releases.js

@@ -0,0 +1,79 @@
+'use strict';
+
+var github = require('../_common/github.js');
+var owner = 'wez';
+var repo = 'atomicparsley';
+
+let targets = {
+  x86win: {
+    os: 'windows',
+    arch: 'x86',
+    libc: 'msvc',
+  },
+  x64win: {
+    os: 'windows',
+    arch: 'amd64',
+    // https://github.com/wez/atomicparsley/issues/6#issuecomment-1364523028
+    libc: 'msvc',
+  },
+  x64mac: {
+    os: 'macos',
+    arch: 'amd64',
+  },
+  x64lin: {
+    os: 'linux',
+    arch: 'amd64',
+    libc: 'gnu',
+  },
+  x64musl: {
+    os: 'linux',
+    arch: 'amd64',
+    libc: 'musl',
+  },
+};
+
+module.exports = function (request) {
+  return github(request, owner, repo).then(function (all) {
+    for (let rel of all.releases) {
+      let windows32 = rel.name.includes('WindowsX86.');
+      if (windows32) {
+        Object.assign(rel, targets.x86win);
+        continue;
+      }
+
+      let windows64 = rel.name.includes('Windows.');
+      if (windows64) {
+        Object.assign(rel, targets.x64win);
+        continue;
+      }
+
+      let macos64 = rel.name.includes('MacOS');
+      if (macos64) {
+        Object.assign(rel, targets.x64mac);
+        continue;
+      }
+
+      let musl64 = rel.name.includes('Alpine');
+      if (musl64) {
+        Object.assign(rel, targets.x64musl);
+        continue;
+      }
+
+      let lin64 = rel.name.includes('Linux.');
+      if (lin64) {
+        Object.assign(rel, targets.x64lin);
+        continue;
+      }
+    }
+    all._names = ['AtomicParsley', 'atomicparsley'];
+    return all;
+  });
+};
+
+if (module === require.main) {
+  module.exports(require('@root/request')).then(function (all) {
+    all = require('../_webi/normalize.js')(all);
+    console.info(JSON.stringify(all));
+    //console.info(JSON.stringify(all, null, 2));
+  });
+}

awless/install.ps1

@@ -20,13 +20,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading awless from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-if (!(Test-Path -Path "$pkg_src_cmd")) {
+IF (!(Test-Path -Path "$pkg_src_cmd")) {
     Write-Output "Installing awless"
 
     # TODO: create package-specific temp directory

awless/releases.conf

@@ -1 +0,0 @@
-github_releases = wallix/awless

awless/releases.js

@@ -0,0 +1,22 @@
+'use strict';
+
+var github = require('../_common/github.js');
+var owner = 'wallix';
+var repo = 'awless';
+
+module.exports = function (request) {
+  return github(request, owner, repo).then(function (all) {
+    // remove checksums and .deb
+    all.releases = all.releases.filter(function (rel) {
+      return !/(\.txt)|(\.deb)$/i.test(rel.name);
+    });
+    return all;
+  });
+};
+
+if (module === require.main) {
+  module.exports(require('@root/request')).then(function (all) {
+    all = require('../_webi/normalize.js')(all);
+    console.info(JSON.stringify(all));
+  });
+}

bat/install.ps1

@@ -3,7 +3,7 @@
 $VERNAME = "$Env:PKG_NAME-v$Env:WEBI_VERSION.exe"
 $EXENAME = "$Env:PKG_NAME.exe"
 # Fetch archive
-if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading $Env:PKG_NAME from $Env:WEBI_PKG_URL to $Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE.part"
     & Move-Item "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE.part" "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
@@ -11,11 +11,11 @@ if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
 
 # Fetch MSVC Runtime
 Write-Output "Checking for MSVC Runtime..."
-if (-not (Test-Path "\Windows\System32\vcruntime140.dll")) {
+IF (-not (Test-Path "\Windows\System32\vcruntime140.dll")) {
     & "$Env:USERPROFILE\.local\bin\webi-pwsh.ps1" vcruntime
 }
 
-if (!(Test-Path -Path "$Env:USERPROFILE\.local\bin\$VERNAME")) {
+IF (!(Test-Path -Path "$Env:USERPROFILE\.local\bin\$VERNAME")) {
     Write-Output "Installing $Env:PKG_NAME"
     # TODO: temp directory
 

bat/releases.conf

@@ -1 +0,0 @@
-github_releases = sharkdp/bat

bat/releases.js

@@ -0,0 +1,20 @@
+'use strict';
+
+var github = require('../_common/github.js');
+var owner = 'sharkdp';
+var repo = 'bat';
+
+module.exports = function (request) {
+  return github(request, owner, repo).then(function (all) {
+    return all;
+  });
+};
+
+if (module === require.main) {
+  module.exports(require('@root/request')).then(function (all) {
+    all = require('../_webi/normalize.js')(all);
+    all.releases = all.releases.slice(0, 10);
+    //console.info(JSON.stringify(all));
+    console.info(JSON.stringify(all, null, 2));
+  });
+}

brew/brew-update-service-install

@@ -6,23 +6,15 @@ main() { (
     sed '1,/^#~\/.local\/bin\/brew-updater/d' "${0}" > ~/.local/bin/brew-update-hourly
     chmod a+x ~/.local/bin/brew-update-hourly
 
-    echo "Checking for serviceman..."
-    ~/.local/bin/webi serviceman
-    if ! command -v serviceman > /dev/null; then
-        export PATH="$HOME/.local/bin:$PATH"
-    fi
-    serviceman --version
-
-    serviceman add --agent \
+    env PATH="$PATH" serviceman add --user \
         --workdir ~/.local/opt/brew/ \
         --name sh.brew.updater -- \
         ~/.local/bin/brew-update-hourly
 ); }
 
-if ! main; then
-    exit 1
+if main; then
+    exit 0
 fi
-exit 0
 
 #~/.local/bin/brew-updater
 #!/bin/sh

bun/README.md

@@ -132,7 +132,9 @@ file)
    ```
 3. Add your project to the system launcher, running as the current user
    ```sh
-   serviceman add --name 'my-project' --daemon -- \
+   sudo env PATH="$PATH" \
+       serviceman add --path="$PATH" --system \
+           --username "$(whoami)" --name my-project -- \
        bun run ./my-project.js
    ```
 4. Restart the logging service
@@ -153,6 +155,6 @@ For **macOS**:
    ```
 3. Add your project to the system launcher, running as the current user
    ```sh
-   serviceman add --agent --name 'my-project' -- \
+   serviceman add --path="$PATH" --user --name my-project -- \
        bun run ./my-project.js
    ```

bun/releases.conf

@@ -1,5 +0,0 @@
-github_releases = oven-sh/bun
-tag_prefix = bun-
-default_x86_64 = x86_64_v3
-x86_64_v2 = baseline
-variants = profile

bun/releases.js

@@ -0,0 +1,39 @@
+'use strict';
+
+var github = require('../_common/github.js');
+var owner = 'oven-sh';
+var repo = 'bun';
+
+module.exports = function (request) {
+  return github(request, owner, repo).then(function (all) {
+    all.releases = all.releases
+      .filter(function (r) {
+        let isDebug = r.name.includes('-profile');
+        if (isDebug) {
+          return false;
+        }
+
+        let isAncient = r.name.includes('-baseline');
+        if (isAncient) {
+          return false;
+        }
+
+        return true;
+      })
+      .map(function (r) {
+        // bun-v0.5.1 => v0.5.1
+        r.version = r.version.replace(/bun-/g, '');
+        return r;
+      });
+    return all;
+  });
+};
+
+if (module === require.main) {
+  module.exports(require('@root/request')).then(function (all) {
+    all = require('../_webi/normalize.js')(all);
+    // just select the first 5 for demonstration
+    all.releases = all.releases.slice(0, 5);
+    console.info(JSON.stringify(all, null, 2));
+  });
+}

caddy/README.md

@@ -819,10 +819,10 @@ To avoid the nitty-gritty details of `launchd` plist files, you can use
 2. Use Serviceman to create a _launchd_ plist file
 
    ```sh
-   my_username="$(id -u -n)"
+   my_username="$( id -u -n )"
 
-   serviceman add --agent --name 'caddy' --workdir ./ -- \
-       caddy run --envfile ~/.config/caddy/env --config ./Caddyfile --adapter caddyfile
+   serviceman add --user --name caddy -- \
+       caddy run --config ./Caddyfile --envfile ~/.config/caddy/env
    ```
 
    (this will create `~/Library/LaunchAgents/caddy.plist`)
@@ -837,8 +837,8 @@ This process creates a _User-Level_ service in `~/Library/LaunchAgents`. To
 create a _System-Level_ service in `/Library/LaunchDaemons/` instead:
 
 ```sh
-serviceman add --name 'caddy' --workdir ./ --daemon -- \
-   caddy run --envfile ~/.config/caddy/env --config ./Caddyfile --adapter caddyfile
+sudo serviceman add --system --name caddy -- \
+   caddy run --config ./Caddyfile --envfile ~/.config/caddy/env
 ```
 
 ### How to run Caddy as a Windows Service
@@ -856,7 +856,7 @@ serviceman add --name 'caddy' --workdir ./ --daemon -- \
 3. Create a **Startup Registry Entry** with Serviceman.
    ```sh
    serviceman.exe add --name caddy -- \
-       caddy run --envfile ~/.config/caddy/env --config ./Caddyfile --adapter caddyfile
+       caddy run --config ./Caddyfile --envfile ~/.config/caddy/env
    ```
 4. You can manage the service directly with Serviceman. For example:
    ```sh
@@ -901,8 +901,10 @@ See the notes below to run as a **User Service** or use the JSON Config.
    ```
 4. Use Serviceman to create a _systemd_ config file.
    ```sh
-   serviceman add --name 'caddy' --daemon -- \
-       caddy run --envfile ~/.config/caddy/env --config ./Caddyfile --adapter caddyfile
+   my_username="$( id -u -n )"
+   sudo env PATH="$PATH" \
+       serviceman add --system --username "${my_username}" --name caddy -- \
+           caddy run --config ./Caddyfile --envfile ~/.config/caddy/env
    ```
    (this will create `/etc/systemd/system/caddy.service`)
 5. Manage the service with `systemctl` and `journalctl`:
@@ -913,10 +915,10 @@ See the notes below to run as a **User Service** or use the JSON Config.
 
 To create a **User Service** instead:
 
-- use `--agent` when running `serviceman`:
+- don't use `sudo`, but do use `--user` when running `serviceman`:
   ```sh
-  serviceman add --agent --name caddy -- \
-      caddy run --envfile ~/.config/caddy/env --config ./Caddyfile --adapter caddyfile
+  serviceman add --user --name caddy -- \
+     caddy run --config ./Caddyfile --envfile ~/.config/caddy/env
   ```
   (this will create `~/.config/systemd/user/`)
 - user the `--user` flag to manage services and logs:
@@ -1181,8 +1183,7 @@ To prevent search engine and browser confusion
 - _DO NOT_ prevent crawling via `robots.txt` \
   (counter-intuitive, but pages _must_ be crawled for links to _NOT_ be indexed)
 - _all_ domains using public TLS certs _will_ be indexed by default \
-  (they are all linked to and crawled from various Certificate Transparency
-  reports)
+  (they are all linked to and crawled from various Certificate Transparency reports)
 - follow these guidelines even if the dev sites use HTTP Basic Auth
 
 ```Caddyfile
@@ -1362,13 +1363,19 @@ See also: <https://caddyserver.com/docs/running>
 2. Generate the `service` file: \
    - JSON Config
      ```sh
-     serviceman add --name 'caddy' --daemon -- \
-         caddy run --resume --envfile ./caddy.env
+     my_app_user="$( id -u -n )"
+     sudo env PATH="${PATH}" \
+         serviceman add --system --cap-net-bind \
+             --username "${my_app_user}" --name caddy -- \
+             caddy run --resume --envfile ./caddy.env
      ```
    - Caddyfile
      ```sh
-     serviceman add --name 'caddy' --daemon -- \
-         caddy run --config ./Caddyfile --envfile ./caddy.env
+     my_app_user="$( id -u -n )"
+     sudo env PATH="${PATH}" \
+         serviceman add --system --cap-net-bind \
+             --username "${my_app_user}" --name caddy -- \
+             caddy run --config ./Caddyfile --envfile ./caddy.env
      ```
 3. Reload `systemd` config files, the logging service (it may not be started on
    a new VPS), and caddy

caddy/install.ps1

@@ -20,13 +20,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading caddy from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-if (!(Test-Path -Path "$pkg_src_cmd")) {
+IF (!(Test-Path -Path "$pkg_src_cmd")) {
     Write-Output "Installing caddy"
 
     # TODO: create package-specific temp directory

caddy/releases.conf

@@ -1 +0,0 @@
-github_releases = caddyserver/caddy

caddy/releases.js

@@ -0,0 +1,27 @@
+'use strict';
+
+var github = require('../_common/github.js');
+var owner = 'caddyserver';
+var repo = 'caddy';
+
+module.exports = function (request) {
+  return github(request, owner, repo).then(function (all) {
+    // remove checksums and .deb
+    all.releases = all.releases.filter(function (rel) {
+      let isOneOffAsset = rel.download.includes('buildable-artifact');
+      if (isOneOffAsset) {
+        return false;
+      }
+
+      return true;
+    });
+    return all;
+  });
+};
+
+if (module === require.main) {
+  module.exports(require('@root/request')).then(function (all) {
+    all = require('../_webi/normalize.js')(all);
+    console.info(JSON.stringify(all));
+  });
+}

chromedriver/install.ps1

@@ -19,13 +19,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading chromedriver from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-if (!(Test-Path -Path "$pkg_src_cmd")) {
+IF (!(Test-Path -Path "$pkg_src_cmd")) {
     Write-Output "Installing chromedriver"
 
     # TODO: create package-specific temp directory

chromedriver/releases.conf

@@ -1 +0,0 @@
-source = chromedist

chromedriver/releases.js

@@ -0,0 +1,84 @@
+'use strict';
+
+// See <https://googlechromelabs.github.io/chrome-for-testing/>
+var releaseApiUrl =
+  'https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json';
+
+// {
+//   "timestamp": "2023-11-15T21:08:56.730Z",
+//   "versions": [
+//     {
+//       "version": "121.0.6120.0",
+//       "revision": "1222902",
+//       "downloads": {
+//         "chrome": [],
+//         "chromedriver": [
+//           {
+//             "platform": "linux64",
+//             "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/121.0.6120.0/linux64/chromedriver-linux64.zip"
+//           },
+//           {
+//             "platform": "mac-arm64",
+//             "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/121.0.6120.0/mac-arm64/chromedriver-mac-arm64.zip"
+//           },
+//           {
+//             "platform": "mac-x64",
+//             "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/121.0.6120.0/mac-x64/chromedriver-mac-x64.zip"
+//           },
+//           {
+//             "platform": "win32",
+//             "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/121.0.6120.0/win32/chromedriver-win32.zip"
+//           },
+//           {
+//             "platform": "win64",
+//             "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/121.0.6120.0/win64/chromedriver-win64.zip"
+//           }
+//         ],
+//         "chrome-headless-shell": []
+//       }
+//     }
+//   ]
+// }
+
+module.exports = async function (request) {
+  let resp = await request({
+    url: releaseApiUrl,
+    json: true,
+  });
+
+  let builds = [];
+  for (let release of resp.body.versions) {
+    if (!release.downloads.chromedriver) {
+      continue;
+    }
+
+    let version = release.version;
+    for (let asset of release.downloads.chromedriver) {
+      let build = {
+        version: version,
+        download: asset.url,
+        // I' not sure that this is actually statically built but it
+        // seems to be and at worst we'll just get bug reports for Apline
+        libc: 'none',
+      };
+
+      builds.push(build);
+    }
+  }
+
+  let all = {
+    download: '',
+    releases: builds,
+  };
+
+  return all;
+};
+
+if (module === require.main) {
+  module.exports(require('@root/request')).then(function (all) {
+    all = require('../_webi/normalize.js')(all);
+    // just select the latest 5 for demonstration
+    all.releases = all.releases.slice(-20);
+    console.info(JSON.stringify(all, null, 2));
+  });
+}

cilium/README.md

@@ -1,47 +0,0 @@
----
-title: cilium
-homepage: https://github.com/cilium/cilium-cli
-tagline: |
-  cilium: manage & troubleshoot Kubernetes clusters running Cilium
----
-
-To update or switch versions, run `webi cilium@stable` (or `@v2`, `@beta`,etc).
-
-### Files
-
-These are the files / directories that are created and/or modified with this
-install:
-
-```text
-~/.config/envman/PATH.env
-~/.local/bin/cilium
-~/.local/opt/cilium/
-```
-
-## Cheat Sheet
-
-> Cilium is an open source, cloud native solution for providing, securing, and
-> observing network connectivity between workloads, fueled by the revolutionary
-> Kernel technology eBPF.
-
-Quick Start User Guide:
-
-<https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/#k8s-install-quick>
-
-To install the default version of the Cilium image:
-
-```sh
-cilium install
-```
-
-To upgrade to a specific version of the Cilium image:
-
-```sh
-cilium upgrade --version v1.15.3
-```
-
-To check the status of the current Cilium deployment:
-
-```sh
-cilium status
-```

cilium/install.ps1

@@ -1,45 +0,0 @@
-#!/usr/bin/env pwsh
-
-##################
-# Install cilium #
-##################
-
-$pkg_cmd_name = "cilium"
-
-$pkg_dst_cmd = "$Env:USERPROFILE\.local\bin\cilium.exe"
-$pkg_dst = "$pkg_dst_cmd"
-
-$pkg_src_cmd = "$Env:USERPROFILE\.local\opt\cilium-v$Env:WEBI_VERSION\bin\cilium.exe"
-$pkg_src_bin = "$Env:USERPROFILE\.local\opt\cilium-v$Env:WEBI_VERSION\bin"
-$pkg_src_dir = "$Env:USERPROFILE\.local\opt\cilium-v$Env:WEBI_VERSION"
-$pkg_src = "$pkg_src_cmd"
-
-New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
-$pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
-
-if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
-    Write-Output "Downloading cilium from $Env:WEBI_PKG_URL to $pkg_download"
-    & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
-    & Move-Item "$pkg_download.part" "$pkg_download"
-}
-
-if (!(Test-Path -Path "$pkg_src_cmd")) {
-    Write-Output "Installing cilium"
-    Push-Location .local\tmp
-
-    Remove-Item -Path ".\cilium-v*" -Recurse -ErrorAction Ignore
-    Remove-Item -Path ".\cilium.exe" -Recurse -ErrorAction Ignore
-
-    Write-Output "Unpacking $pkg_download"
-    & tar xf "$pkg_download"
-
-    Write-Output "Install Location: $pkg_src_cmd"
-    New-Item "$pkg_src_bin" -ItemType Directory -Force
-    Move-Item -Path ".\cilium-*\cilium.exe" -Destination "$pkg_src_bin"
-
-    Pop-Location
-}
-
-Write-Output "Copying into '$pkg_dst_cmd' from '$pkg_src_cmd'"
-Remove-Item -Path "$pkg_dst_cmd" -Recurse -ErrorAction Ignore
-Copy-Item -Path "$pkg_src" -Destination "$pkg_dst" -Recurse

cilium/install.sh

@@ -1,39 +0,0 @@
-#!/bin/sh
-
-__init_cilium() {
-    set -e
-    set -u
-
-    ##################
-    # Install cilium #
-    ##################
-
-    pkg_cmd_name="cilium"
-
-    pkg_dst_cmd="$HOME/.local/bin/cilium"
-    pkg_dst="$pkg_dst_cmd"
-
-    pkg_src_cmd="$HOME/.local/opt/cilium-v$WEBI_VERSION/bin/cilium"
-    pkg_src_dir="$HOME/.local/opt/cilium-v$WEBI_VERSION"
-    pkg_src="$pkg_src_cmd"
-
-    WEBI_SINGLE=true
-
-    # pkg_install must be defined by every package
-    pkg_install() {
-        # ~/.local/opt/cilium-v0.16.16/bin
-        mkdir -p "$(dirname "${pkg_src_cmd}")"
-
-        # mv ./hugo ~/.local/opt/cilium-v0.16.16/bin/
-        mv ./cilium "${pkg_src_cmd}"
-    }
-
-    pkg_get_current_version() {
-        cilium version 2> /dev/null |
-            head -n 1 |
-            cut -d ' ' -f 2
-    }
-
-}
-
-__init_cilium

cilium/releases.conf

@@ -1 +0,0 @@
-github_releases = cilium/cilium-cli

cmake/install.ps1

@@ -20,13 +20,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-if (!(Test-Path -Path "$pkg_download")) {
+IF (!(Test-Path -Path "$pkg_download")) {
     Write-Output "Downloading cmake from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-if (!(Test-Path -Path "$pkg_src_dir")) {
+IF (!(Test-Path -Path "$pkg_src_dir")) {
     Write-Output "Installing cmake"
 
     # TODO: create package-specific temp directory

cmake/releases.conf

@@ -1 +0,0 @@
-github_releases = Kitware/CMake

cmake/releases.js

@@ -0,0 +1,53 @@
+'use strict';
+
+var github = require('../_common/github.js');
+var owner = 'Kitware';
+var repo = 'CMake';
+
+module.exports = function (request) {
+  return github(request, owner, repo).then(function (all) {
+    for (let rel of all.releases) {
+      if (rel.version.startsWith('v')) {
+        rel._version = rel.version.slice(1);
+      }
+
+      {
+        let linuxRe = /(\b|_)(linux|gnu)(\b|_)/i;
+        let isLinux = linuxRe.test(rel.download) || linuxRe.test(rel.name);
+
+        if (isLinux) {
+          let muslRe = /(\b|_)(musl|alpine)(\b|_)/i;
+          let isMusl = muslRe.test(rel.download) || muslRe.test(rel.name);
+          if (isMusl) {
+            rel.libc = 'musl';
+          } else {
+            rel.libc = 'gnu';
+          }
+          continue;
+        }
+      }
+
+      {
+        let windowsRe = /(\b|_)(win\d*|windows\d*)(\b|_)/i;
+        let isWindows =
+          windowsRe.test(rel.download) || windowsRe.test(rel.name);
+
+        if (isWindows) {
+          rel.libc = 'msvc';
+          continue;
+        }
+      }
+    }
+
+    return all;
+  });
+};
+
+if (module === require.main) {
+  module.exports(require('@root/request')).then(function (all) {
+    all = require('../_webi/normalize.js')(all);
+    // just select the first 5 for demonstration
+    all.releases = all.releases.slice(0, 5);
+    console.info(JSON.stringify(all, null, 2));
+  });
+}

cmd/comparecache/main.go

@@ -1,914 +0,0 @@
-// Command comparecache compares Go-generated cache output against the
-// Node.js LIVE_cache. It identifies categorical differences in asset
-// selection — which filenames appear in one cache but not the other.
-//
-// The comparison is done at the filename level (not OS/arch/ext fields)
-// because the Node.js cache leaves those empty (normalize.js fills them
-// at serve time), while the Go pipeline classifies at write time.
-//
-// Usage:
-//
-//	go run ./cmd/comparecache -live ./LIVE_cache -go ./_cache
-//	go run ./cmd/comparecache -live ./LIVE_cache -go ./_cache bat jq
-//	go run ./cmd/comparecache -live ./LIVE_cache -go ./_cache -summary
-package main
-
-import (
-	"encoding/json"
-	"flag"
-	"fmt"
-	"log"
-	"math/rand/v2"
-	"os"
-	"path/filepath"
-	"slices"
-	"sort"
-	"strings"
-	"time"
-
-	"github.com/webinstall/webi-installers/internal/classify"
-	"github.com/webinstall/webi-installers/internal/lexver"
-)
-
-type cacheEntry struct {
-	Releases []cacheRelease `json:"releases"`
-}
-
-type cacheRelease struct {
-	Name     string `json:"name"`
-	Filename string `json:"_filename"` // Node.js uses _filename for some sources
-	Version  string `json:"version"`
-	Download string `json:"download"`
-	Channel  string `json:"channel"`
-	OS       string `json:"os"`
-	Arch     string `json:"arch"`
-	Libc     string `json:"libc"`
-	Ext      string `json:"ext"`
-}
-
-// fieldDiff records a field-level difference for an asset that exists
-// in both caches (same filename) but has different classification.
-type fieldDiff struct {
-	Filename string
-	Field    string // "os", "arch", "libc", "ext", "channel"
-	Live     string
-	Go       string
-	BothSet  bool // true when both live and go have non-empty values
-}
-
-type packageDiff struct {
-	Name         string
-	LiveCount    int
-	GoCount      int
-	OnlyInLive   []string // filenames only in Node.js cache
-	OnlyInGo     []string // filenames only in Go cache
-	FieldDiffs   []fieldDiff // classification differences on shared assets
-	VersionsLive []string    // unique versions in live
-	VersionsGo   []string    // unique versions in go
-	GoMissing    bool        // true if Go didn't produce output for this package
-	LiveMissing  bool        // true if no live cache for this package
-	Categories   []string    // categorical difference labels
-}
-
-func main() {
-	liveDir := flag.String("live", "./LIVE_cache", "path to Node.js LIVE_cache directory")
-	goDir := flag.String("go", "./_cache", "path to Go cache directory")
-	summary := flag.Bool("summary", false, "only print summary, not per-package details")
-	diffsOnly := flag.Bool("diffs", false, "only show packages with asset differences (skip matches)")
-	latest := flag.Bool("latest", false, "only compare latest version in each cache")
-	windowed := flag.Bool("windowed", false, "limit Go versions to the Node.js version range (2nd to 2nd-to-last)")
-	sample := flag.Int("sample", 0, "for each package diff, show N randomly sampled assets (implies -windowed -diffs)")
-	flag.Parse()
-	filterPkgs := flag.Args()
-
-	// -sample implies -windowed and -diffs so we focus on real classification
-	// differences, not version-depth noise.
-	if *sample > 0 {
-		*windowed = true
-		*diffsOnly = true
-	}
-
-	totalStart := time.Now()
-
-	// Find the most recent month directory in each cache.
-	liveMonth := findLatestMonth(*liveDir)
-	goMonth := findLatestMonth(*goDir)
-	if liveMonth == "" {
-		log.Fatalf("no month directories found in %s", *liveDir)
-	}
-
-	livePath := filepath.Join(*liveDir, liveMonth)
-	goPath := ""
-	if goMonth != "" {
-		goPath = filepath.Join(*goDir, goMonth)
-	}
-
-	// Discover all packages across both caches.
-	discoverStart := time.Now()
-	allPkgs := discoverPackages(livePath, goPath)
-	if len(filterPkgs) > 0 {
-		nameSet := make(map[string]bool, len(filterPkgs))
-		for _, n := range filterPkgs {
-			nameSet[n] = true
-		}
-		var filtered []string
-		for _, p := range allPkgs {
-			if nameSet[p] {
-				filtered = append(filtered, p)
-			}
-		}
-		allPkgs = filtered
-	}
-	log.Printf("discovered %d packages in %s", len(allPkgs), time.Since(discoverStart))
-
-	compareStart := time.Now()
-	var diffs []packageDiff
-	for _, pkg := range allPkgs {
-		d := compare(livePath, goPath, pkg, *latest, *windowed)
-		categorize(&d)
-		diffs = append(diffs, d)
-	}
-	log.Printf("compared %d packages in %s", len(diffs), time.Since(compareStart))
-
-	if *summary {
-		printSummary(diffs)
-	} else {
-		printDetails(diffs, *diffsOnly, *sample)
-	}
-
-	log.Printf("total: %s", time.Since(totalStart))
-}
-
-func findLatestMonth(dir string) string {
-	entries, err := os.ReadDir(dir)
-	if err != nil {
-		return ""
-	}
-	var months []string
-	for _, e := range entries {
-		if e.IsDir() && len(e.Name()) == 7 && e.Name()[4] == '-' {
-			months = append(months, e.Name())
-		}
-	}
-	if len(months) == 0 {
-		return ""
-	}
-	sort.Strings(months)
-	return months[len(months)-1]
-}
-
-func discoverPackages(livePath, goPath string) []string {
-	seen := make(map[string]bool)
-	for _, dir := range []string{livePath, goPath} {
-		if dir == "" {
-			continue
-		}
-		entries, err := os.ReadDir(dir)
-		if err != nil {
-			continue
-		}
-		for _, e := range entries {
-			name := e.Name()
-			if strings.HasSuffix(name, ".json") && !strings.HasSuffix(name, ".updated.txt") {
-				pkg := strings.TrimSuffix(name, ".json")
-				seen[pkg] = true
-			}
-		}
-	}
-	var pkgs []string
-	for p := range seen {
-		pkgs = append(pkgs, p)
-	}
-	sort.Strings(pkgs)
-	return pkgs
-}
-
-func loadCache(dir, pkg string) *cacheEntry {
-	if dir == "" {
-		return nil
-	}
-	data, err := os.ReadFile(filepath.Join(dir, pkg+".json"))
-	if err != nil {
-		return nil
-	}
-	var entry cacheEntry
-	if err := json.Unmarshal(data, &entry); err != nil {
-		return nil
-	}
-	return &entry
-}
-
-// effectiveName returns the best available filename for a release entry.
-// Node.js sometimes uses _filename (a path) instead of name.
-func effectiveName(name, filename, download string) string {
-	if name != "" {
-		return name
-	}
-	if filename != "" {
-		// _filename may be a path like "stable/macos/flutter_macos_3.41.4.zip"
-		if i := strings.LastIndex(filename, "/"); i >= 0 {
-			return filename[i+1:]
-		}
-		return filename
-	}
-	// Last resort: basename of download URL.
-	if download != "" {
-		if i := strings.LastIndex(download, "/"); i >= 0 {
-			return download[i+1:]
-		}
-	}
-	return ""
-}
-
-// versionWindow returns the 2nd and 2nd-to-last versions from a sorted
-// version list. This trims the edges where Node.js may have a newer fetch
-// or Go may have deeper history, focusing on the overlapping middle.
-func versionWindow(versions []string) (low, high string) {
-	if len(versions) <= 2 {
-		// Too few versions to window — use all.
-		if len(versions) > 0 {
-			return versions[0], versions[len(versions)-1]
-		}
-		return "", ""
-	}
-	// 2nd version (skip oldest) and 2nd-to-last (skip newest).
-	return versions[1], versions[len(versions)-2]
-}
-
-// filterVersionRange returns only the versions in sorted order that fall
-// within [low, high] inclusive (by lexver comparison).
-func filterVersionRange(vf map[string]map[string]bool, versions []string, low, high string) (map[string]bool, []string) {
-	lowV := lexver.Parse(low)
-	highV := lexver.Parse(high)
-
-	files := make(map[string]bool)
-	var kept []string
-	for _, v := range versions {
-		pv := lexver.Parse(v)
-		if lexver.Compare(pv, lowV) >= 0 && lexver.Compare(pv, highV) <= 0 {
-			kept = append(kept, v)
-			for f := range vf[v] {
-				files[f] = true
-			}
-		}
-	}
-	return files, kept
-}
-
-func compare(livePath, goPath, pkg string, latestOnly, windowed bool) packageDiff {
-	live := loadCache(livePath, pkg)
-	goCache := loadCache(goPath, pkg)
-
-	d := packageDiff{Name: pkg}
-
-	if live == nil {
-		d.LiveMissing = true
-	}
-	if goCache == nil {
-		d.GoMissing = true
-	}
-	if d.LiveMissing && d.GoMissing {
-		return d
-	}
-
-	normVersion := normalizeVersionFunc(pkg)
-
-	// Collect filenames by version. If filter is non-nil, skip filenames it rejects.
-	extractVersionFiles := func(ce *cacheEntry, filter func(string) bool) (map[string]map[string]bool, []string) {
-		vf := make(map[string]map[string]bool)
-		for _, r := range ce.Releases {
-			name := effectiveName(r.Name, r.Filename, r.Download)
-			if filter != nil && !filter(name) {
-				continue
-			}
-			ver := normVersion(r.Version)
-			if vf[ver] == nil {
-				vf[ver] = make(map[string]bool)
-			}
-			vf[ver][name] = true
-		}
-		var versions []string
-		for v := range vf {
-			versions = append(versions, v)
-		}
-		slices.SortFunc(versions, func(a, b string) int {
-			return lexver.Compare(lexver.Parse(a), lexver.Parse(b))
-		})
-		return vf, versions
-	}
-	notNoise := func(name string) bool { return !isLiveNoise(name) }
-
-	var liveFiles, goFiles map[string]bool
-
-	// Parse live cache.
-	var liveVF map[string]map[string]bool
-	var liveVersions []string
-	if live != nil {
-		liveVF, liveVersions = extractVersionFiles(live, notNoise)
-		d.VersionsLive = liveVersions
-		d.LiveCount = len(live.Releases)
-	}
-
-	// Parse Go cache.
-	var goVF map[string]map[string]bool
-	var goVersions []string
-	if goCache != nil {
-		goVF, goVersions = extractVersionFiles(goCache, notNoise)
-		d.VersionsGo = goVersions
-		d.GoCount = len(goCache.Releases)
-	}
-
-	// Determine which files to compare based on mode.
-	if latestOnly {
-		// Compare only the latest version from each cache.
-		if live != nil && len(liveVersions) > 0 {
-			liveFiles = liveVF[liveVersions[len(liveVersions)-1]]
-		}
-		if goCache != nil && len(goVersions) > 0 {
-			goFiles = goVF[goVersions[len(goVersions)-1]]
-		}
-	} else if windowed && live != nil && len(liveVersions) > 0 {
-		// Use the Node.js version range (2nd to 2nd-to-last) to establish
-		// the window. Include ALL Node.js versions in the window (so missing
-		// Go versions are visible), but exclude Go-only versions (those are
-		// just deeper history, not real gaps).
-		low, high := versionWindow(liveVersions)
-		lowV := lexver.Parse(low)
-		highV := lexver.Parse(high)
-
-		// Collect all live files in the window.
-		liveFiles = make(map[string]bool)
-		liveInWindow := make(map[string]bool)
-		for _, v := range liveVersions {
-			pv := lexver.Parse(v)
-			if lexver.Compare(pv, lowV) >= 0 && lexver.Compare(pv, highV) <= 0 {
-				liveInWindow[v] = true
-				for f := range liveVF[v] {
-					liveFiles[f] = true
-				}
-			}
-		}
-
-		// For Go, only include versions that Node.js also has in the window.
-		// Go-only versions are hidden (deeper history, not gaps).
-		goFiles = make(map[string]bool)
-		for _, v := range goVersions {
-			if !liveInWindow[v] {
-				continue
-			}
-			for f := range goVF[v] {
-				goFiles[f] = true
-			}
-		}
-	} else {
-		// Compare all versions — use pre-filtered version maps.
-		if live != nil {
-			liveFiles = make(map[string]bool)
-			for _, files := range liveVF {
-				for f := range files {
-					liveFiles[f] = true
-				}
-			}
-		}
-		if goCache != nil {
-			goFiles = make(map[string]bool)
-			for _, files := range goVF {
-				for f := range files {
-					goFiles[f] = true
-				}
-			}
-		}
-	}
-
-	if liveFiles == nil {
-		liveFiles = make(map[string]bool)
-	}
-	if goFiles == nil {
-		goFiles = make(map[string]bool)
-	}
-
-	for f := range liveFiles {
-		if !goFiles[f] {
-			d.OnlyInLive = append(d.OnlyInLive, f)
-		}
-	}
-	for f := range goFiles {
-		if !liveFiles[f] {
-			d.OnlyInGo = append(d.OnlyInGo, f)
-		}
-	}
-	sort.Strings(d.OnlyInLive)
-	sort.Strings(d.OnlyInGo)
-
-	// Field-level comparison on assets that exist in both caches.
-	// Build version+filename → fields maps from each cache.
-	if live != nil && goCache != nil {
-		type assetKey struct {
-			version  string
-			filename string
-		}
-		liveByKey := make(map[assetKey]cacheRelease)
-		for _, r := range live.Releases {
-			name := effectiveName(r.Name, r.Filename, r.Download)
-			ver := normVersion(r.Version)
-			liveByKey[assetKey{ver, name}] = r
-		}
-
-		for _, r := range goCache.Releases {
-			name := effectiveName(r.Name, r.Filename, r.Download)
-			ver := normVersion(r.Version)
-			lr, ok := liveByKey[assetKey{ver, name}]
-			if !ok {
-				continue
-			}
-
-			// Compare classification fields.
-			// Use equivalence checks for os/arch/ext so naming
-			// convention differences don't mask real classification bugs.
-			for _, cmp := range []struct {
-				field string
-				live  string
-				go_   string
-				equiv bool
-			}{
-				{"os", lr.OS, r.OS, equivOS(lr.OS, r.OS)},
-				{"arch", lr.Arch, r.Arch, equivArch(lr.Arch, r.Arch)},
-				{"libc", lr.Libc, r.Libc, lr.Libc == r.Libc},
-				{"ext", lr.Ext, r.Ext, equivExt(lr.Ext, r.Ext)},
-				{"channel", lr.Channel, r.Channel, lr.Channel == r.Channel},
-			} {
-				if cmp.equiv {
-					continue
-				}
-				d.FieldDiffs = append(d.FieldDiffs, fieldDiff{
-					Filename: name,
-					Field:    cmp.field,
-					Live:     cmp.live,
-					Go:       cmp.go_,
-					BothSet:  cmp.live != "" && cmp.go_ != "",
-				})
-			}
-		}
-		sort.Slice(d.FieldDiffs, func(i, j int) bool {
-			if d.FieldDiffs[i].Field != d.FieldDiffs[j].Field {
-				return d.FieldDiffs[i].Field < d.FieldDiffs[j].Field
-			}
-			return d.FieldDiffs[i].Filename < d.FieldDiffs[j].Filename
-		})
-	}
-
-	return d
-}
-
-// equivOS returns true if two OS values are equivalent across naming conventions.
-func equivOS(a, b string) bool {
-	return a == b || canonicalOS(a) == canonicalOS(b)
-}
-
-func canonicalOS(s string) string {
-	switch strings.ToLower(s) {
-	case "darwin", "macos", "mac", "osx":
-		return "darwin"
-	case "win", "windows":
-		return "windows"
-	default:
-		return strings.ToLower(s)
-	}
-}
-
-// equivArch returns true if two arch values are equivalent.
-func equivArch(a, b string) bool {
-	return a == b || canonicalArch(a) == canonicalArch(b)
-}
-
-func canonicalArch(s string) string {
-	switch strings.ToLower(s) {
-	case "x86_64", "amd64", "x64":
-		return "x86_64"
-	case "aarch64", "arm64":
-		return "aarch64"
-	case "armv7", "armv7l":
-		return "armv7"
-	case "armv6", "armv6l":
-		return "armv6"
-	case "x86", "i386", "i686", "386":
-		return "x86"
-	default:
-		return strings.ToLower(s)
-	}
-}
-
-// equivExt returns true if two extension values are equivalent.
-func equivExt(a, b string) bool {
-	// Normalize: strip leading dot, handle common aliases.
-	return a == b || canonicalExt(a) == canonicalExt(b)
-}
-
-func canonicalExt(s string) string {
-	s = strings.TrimPrefix(s, ".")
-	switch s {
-	case "tgz":
-		return "tar.gz"
-	default:
-		return s
-	}
-}
-
-func categorize(d *packageDiff) {
-	if d.GoMissing {
-		d.Categories = append(d.Categories, "go-missing")
-		return
-	}
-	if d.LiveMissing {
-		d.Categories = append(d.Categories, "live-missing")
-		return
-	}
-
-	if len(d.OnlyInLive) == 0 && len(d.OnlyInGo) == 0 && len(d.FieldDiffs) == 0 {
-		d.Categories = append(d.Categories, "match")
-		return
-	}
-	if len(d.OnlyInLive) == 0 && len(d.OnlyInGo) == 0 && len(d.FieldDiffs) > 0 {
-		d.Categories = append(d.Categories, "fields-only")
-	}
-
-	// Check if differences are only version depth (Go has more history).
-	liveVersionSet := make(map[string]bool, len(d.VersionsLive))
-	for _, v := range d.VersionsLive {
-		liveVersionSet[v] = true
-	}
-	goVersionSet := make(map[string]bool, len(d.VersionsGo))
-	for _, v := range d.VersionsGo {
-		goVersionSet[v] = true
-	}
-
-	goExtraVersions := 0
-	for _, v := range d.VersionsGo {
-		if !liveVersionSet[v] {
-			goExtraVersions++
-		}
-	}
-	liveExtraVersions := 0
-	for _, v := range d.VersionsLive {
-		if !goVersionSet[v] {
-			liveExtraVersions++
-		}
-	}
-
-	if goExtraVersions > 0 {
-		d.Categories = append(d.Categories, fmt.Sprintf("go-extra-versions(%d)", goExtraVersions))
-	}
-	if liveExtraVersions > 0 {
-		d.Categories = append(d.Categories, fmt.Sprintf("live-extra-versions(%d)", liveExtraVersions))
-	}
-
-	// Check for meta-asset filtering differences.
-	metaOnlyInLive := 0
-	nonMetaOnlyInLive := 0
-	for _, f := range d.OnlyInLive {
-		if classify.IsMetaAsset(f) {
-			metaOnlyInLive++
-		} else {
-			nonMetaOnlyInLive++
-		}
-	}
-	metaOnlyInGo := 0
-	nonMetaOnlyInGo := 0
-	for _, f := range d.OnlyInGo {
-		if classify.IsMetaAsset(f) {
-			metaOnlyInGo++
-		} else {
-			nonMetaOnlyInGo++
-		}
-	}
-
-	if metaOnlyInLive > 0 {
-		d.Categories = append(d.Categories, fmt.Sprintf("live-has-meta(%d)", metaOnlyInLive))
-	}
-	if metaOnlyInGo > 0 {
-		d.Categories = append(d.Categories, fmt.Sprintf("go-has-meta(%d)", metaOnlyInGo))
-	}
-
-	// Check for source tarball differences.
-	srcOnlyInGo := 0
-	for _, f := range d.OnlyInGo {
-		if strings.HasSuffix(f, ".tar.gz") || strings.HasSuffix(f, ".zip") {
-			if strings.HasPrefix(f, "v") || strings.HasPrefix(f, "refs/") {
-				srcOnlyInGo++
-			}
-		}
-	}
-	if srcOnlyInGo > 0 {
-		d.Categories = append(d.Categories, fmt.Sprintf("go-has-source-tarballs(%d)", srcOnlyInGo))
-	}
-
-	if nonMetaOnlyInLive > 0 {
-		d.Categories = append(d.Categories, fmt.Sprintf("live-extra-assets(%d)", nonMetaOnlyInLive))
-	}
-	if nonMetaOnlyInGo > 0 {
-		d.Categories = append(d.Categories, fmt.Sprintf("go-extra-assets(%d)", nonMetaOnlyInGo))
-	}
-
-	// Count field diffs by field name, separating real disagreements
-	// from expected "live empty, Go classified" differences.
-	type fieldCount struct {
-		bothSet  int // both caches have a value but they disagree
-		oneEmpty int // one side is empty (typically live — normalize.js fills at serve time)
-	}
-	fieldCounts := make(map[string]fieldCount)
-	for _, fd := range d.FieldDiffs {
-		fc := fieldCounts[fd.Field]
-		if fd.BothSet {
-			fc.bothSet++
-		} else {
-			fc.oneEmpty++
-		}
-		fieldCounts[fd.Field] = fc
-	}
-	for _, field := range []string{"os", "arch", "libc", "ext", "channel"} {
-		fc := fieldCounts[field]
-		if fc.bothSet > 0 {
-			d.Categories = append(d.Categories, fmt.Sprintf("diff-%s(%d)", field, fc.bothSet))
-		}
-		if fc.oneEmpty > 0 {
-			d.Categories = append(d.Categories, fmt.Sprintf("fill-%s(%d)", field, fc.oneEmpty))
-		}
-	}
-}
-
-// isLiveNoise returns true for filenames that the Node.js cache keeps
-// but Go intentionally filters out. Pre-filtering these from the live
-// side prevents them from appearing as live-extra-assets noise.
-//
-// This includes everything classify.IsMetaAsset catches plus formats
-// that Go's legacy export strips (.deb, .rpm, etc.).
-func isLiveNoise(name string) bool {
-	if classify.IsMetaAsset(name) {
-		return true
-	}
-
-	lower := strings.ToLower(name)
-
-	// Formats Go filters from legacy export but Node.js keeps.
-	for _, suffix := range []string{
-		".deb", ".rpm", ".gpg",
-	} {
-		if strings.HasSuffix(lower, suffix) {
-			return true
-		}
-	}
-
-	// Source tarballs (e.g. gitea-src-1.25.4.tar.gz, caddy_2.10.0_src.tar.gz, go1.26.1.src.tar.gz).
-	if strings.Contains(lower, "-src-") || strings.Contains(lower, "_src.") || strings.Contains(lower, ".src.") || strings.HasPrefix(lower, "src-") {
-		return true
-	}
-
-	// Docs tarballs (e.g. gitea-docs-1.22.3.tar.gz).
-	if strings.Contains(lower, "-docs-") {
-		return true
-	}
-
-	// Bare executables without any extension — typically legacy shell scripts
-	// uploaded alongside proper archives (e.g. kubectx, kubens).
-	if !strings.Contains(name, ".") {
-		return true
-	}
-
-	// GPU accelerator / hardware variants that Go tags as variants
-	// but Node.js keeps with special arch names.
-	for _, v := range []string{"-rocm", "-jetpack"} {
-		if strings.Contains(lower, v) {
-			return true
-		}
-	}
-
-	// Linux binaries for packages where Node.js only kept macOS .app.zip.
-	// Go correctly includes these as installable on Linux.
-	if strings.HasPrefix(lower, "fish-") && strings.Contains(lower, "-linux-") {
-		return true
-	}
-
-	return false
-}
-
-// normalizeVersionFunc returns a version normalizer for a given package.
-// Most packages return the identity function. Some (like git) need
-// version string normalization to match across Go and Node.js caches.
-func normalizeVersionFunc(pkg string) func(string) string {
-	switch pkg {
-	case "git":
-		return func(v string) string {
-			// Git for Windows: v2.53.0.windows.1 → v2.53.0
-			//                  v2.53.0.windows.2 → v2.53.0.2
-			idx := strings.Index(v, ".windows.")
-			if idx < 0 {
-				return v
-			}
-			suffix := v[idx+len(".windows."):]
-			base := v[:idx]
-			if suffix == "1" {
-				return base
-			}
-			return base + "." + suffix
-		}
-	case "lf":
-		return func(v string) string {
-			// lf: r21 → 0.21.0
-			if strings.HasPrefix(v, "r") {
-				return "0." + v[1:] + ".0"
-			}
-			return v
-		}
-	case "bun":
-		return func(v string) string {
-			// bun: bun-v1.3.9 → v1.3.9
-			return strings.TrimPrefix(v, "bun-")
-		}
-	case "watchexec":
-		return func(v string) string {
-			// watchexec monorepo: cli-v1.20.5 → v1.20.5
-			return strings.TrimPrefix(v, "cli-")
-		}
-	case "go":
-		return func(v string) string {
-			// Go: go1.10 → 1.10.0 (pad to 3 parts)
-			v = strings.TrimPrefix(v, "go")
-			parts := strings.SplitN(v, ".", 3)
-			for len(parts) < 3 {
-				parts = append(parts, "0")
-			}
-			return strings.Join(parts, ".")
-		}
-	default:
-		return func(v string) string { return v }
-	}
-}
-
-func printSummary(diffs []packageDiff) {
-	// Count by category.
-	categoryCounts := make(map[string]int)
-	for _, d := range diffs {
-		for _, c := range d.Categories {
-			// Strip the count suffix for grouping.
-			base := c
-			if idx := strings.Index(c, "("); idx != -1 {
-				base = c[:idx]
-			}
-			categoryCounts[base]++
-		}
-	}
-
-	fmt.Println("=== COMPARISON SUMMARY ===")
-	fmt.Printf("Total packages: %d\n\n", len(diffs))
-
-	var cats []string
-	for c := range categoryCounts {
-		cats = append(cats, c)
-	}
-	sort.Strings(cats)
-	for _, c := range cats {
-		fmt.Printf("  %-30s %d\n", c, categoryCounts[c])
-	}
-
-	fmt.Println("\n=== PER-PACKAGE CATEGORIES ===")
-	for _, d := range diffs {
-		fmt.Printf("%-25s %s\n", d.Name, strings.Join(d.Categories, ", "))
-	}
-}
-
-func printDetails(diffs []packageDiff, diffsOnly bool, sampleN int) {
-	for _, d := range diffs {
-		if diffsOnly && len(d.OnlyInLive) == 0 && len(d.OnlyInGo) == 0 && len(d.FieldDiffs) == 0 {
-			continue
-		}
-
-		fmt.Printf("=== %s ===\n", d.Name)
-		fmt.Printf("  Categories: %s\n", strings.Join(d.Categories, ", "))
-		fmt.Printf("  Live: %d assets, %d versions  |  Go: %d assets, %d versions\n",
-			d.LiveCount, len(d.VersionsLive), d.GoCount, len(d.VersionsGo))
-
-		printAssetList("Only in LIVE", d.OnlyInLive, sampleN)
-		printAssetList("Only in Go", d.OnlyInGo, sampleN)
-		printFieldDiffs(d.FieldDiffs, sampleN)
-
-		fmt.Println()
-	}
-}
-
-// printFieldDiffs shows classification differences on shared assets.
-// Shows "real" diffs (both sides non-empty) first, then "fill" diffs
-// (one side empty) as a summary count only.
-func printFieldDiffs(diffs []fieldDiff, sampleN int) {
-	if len(diffs) == 0 {
-		return
-	}
-
-	// Separate real disagreements from fill diffs.
-	var real, fill []fieldDiff
-	for _, fd := range diffs {
-		if fd.BothSet {
-			real = append(real, fd)
-		} else {
-			fill = append(fill, fd)
-		}
-	}
-
-	// Show real disagreements in detail.
-	if len(real) > 0 {
-		byField := make(map[string][]fieldDiff)
-		for _, fd := range real {
-			byField[fd.Field] = append(byField[fd.Field], fd)
-		}
-
-		for _, field := range []string{"os", "arch", "libc", "ext", "channel"} {
-			fds := byField[field]
-			if len(fds) == 0 {
-				continue
-			}
-
-			fmt.Printf("  DISAGREE %s (%d):\n", field, len(fds))
-			printFieldDiffItems(fds, sampleN)
-		}
-	}
-
-	// Summarize fill diffs (live empty, Go classified) as counts.
-	if len(fill) > 0 {
-		byField := make(map[string]int)
-		for _, fd := range fill {
-			byField[fd.Field]++
-		}
-		var parts []string
-		for _, field := range []string{"os", "arch", "libc", "ext", "channel"} {
-			if n := byField[field]; n > 0 {
-				parts = append(parts, fmt.Sprintf("%s(%d)", field, n))
-			}
-		}
-		if len(parts) > 0 {
-			fmt.Printf("  Go fills empty: %s\n", strings.Join(parts, ", "))
-		}
-	}
-}
-
-func printFieldDiffItems(fds []fieldDiff, sampleN int) {
-	items := fds
-	if sampleN > 0 && len(items) > sampleN {
-		sampled := make([]fieldDiff, len(items))
-		copy(sampled, items)
-		rand.Shuffle(len(sampled), func(i, j int) {
-			sampled[i], sampled[j] = sampled[j], sampled[i]
-		})
-		items = sampled[:sampleN]
-		sort.Slice(items, func(i, j int) bool {
-			return items[i].Filename < items[j].Filename
-		})
-	}
-
-	limit := 20
-	for i, fd := range items {
-		if sampleN == 0 && i >= limit {
-			fmt.Printf("    ... and %d more\n", len(fds)-limit)
-			break
-		}
-		fmt.Printf("    - %s: live=%q go=%q\n", fd.Filename, fd.Live, fd.Go)
-	}
-	if sampleN > 0 && len(fds) > sampleN {
-		fmt.Printf("    ... sampled %d of %d\n", sampleN, len(fds))
-	}
-}
-
-// printAssetList prints a list of asset filenames, optionally sampling N at
-// random. When sampleN > 0 and the list is longer, it picks N random items
-// so you can spot classification bugs across the full range instead of only
-// seeing the first alphabetical entries.
-func printAssetList(label string, items []string, sampleN int) {
-	if len(items) == 0 {
-		return
-	}
-
-	fmt.Printf("  %s (%d):\n", label, len(items))
-
-	if sampleN > 0 && len(items) > sampleN {
-		// Shuffle a copy, take first N, then sort for readable output.
-		sampled := make([]string, len(items))
-		copy(sampled, items)
-		rand.Shuffle(len(sampled), func(i, j int) {
-			sampled[i], sampled[j] = sampled[j], sampled[i]
-		})
-		picked := sampled[:sampleN]
-		sort.Strings(picked)
-		for _, f := range picked {
-			fmt.Printf("    - %s\n", f)
-		}
-		fmt.Printf("    ... sampled %d of %d (run again for different sample)\n", sampleN, len(items))
-		return
-	}
-
-	limit := 20
-	for i, f := range items {
-		if i >= limit {
-			fmt.Printf("    ... and %d more\n", len(items)-limit)
-			break
-		}
-		fmt.Printf("    - %s\n", f)
-	}
-}

cmd/e2etest/main.go

@@ -1,846 +0,0 @@
-// Command e2etest runs the full release pipeline for selected packages
-// and compares results against the live webi.sh API.
-//
-// It fetches from upstream, classifies assets, resolves the best match
-// for a set of test queries, then fetches the same queries from the live
-// API and reports any differences.
-//
-// Usage:
-//
-//	go run ./cmd/e2etest
-//	go run ./cmd/e2etest -packages goreleaser,ollama,node
-//	go run ./cmd/e2etest -cache ./_cache/raw    # reuse existing cache
-package main
-
-import (
-	"bufio"
-	"context"
-	"encoding/json"
-	"flag"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"github.com/webinstall/webi-installers/internal/buildmeta"
-	"github.com/webinstall/webi-installers/internal/installerconf"
-	"github.com/webinstall/webi-installers/internal/lexver"
-	"github.com/webinstall/webi-installers/internal/rawcache"
-	"github.com/webinstall/webi-installers/internal/releases/github"
-	"github.com/webinstall/webi-installers/internal/releases/githubish"
-	"github.com/webinstall/webi-installers/internal/releases/nodedist"
-	"github.com/webinstall/webi-installers/internal/resolve"
-)
-
-// testCase is one query to resolve and compare against the live API.
-type testCase struct {
-	Name    string
-	Package string
-	OS      buildmeta.OS
-	Arch    buildmeta.Arch
-	Libc    buildmeta.Libc
-	Formats []string
-	UA      string // User-Agent for live API query
-}
-
-// liveResult holds parsed fields from the live webi API response.
-type liveResult struct {
-	Version  string
-	OS       string
-	Arch     string
-	Libc     string
-	Ext      string
-	PkgURL   string
-	PkgFile  string
-	Channel  string
-	Stable   string
-	Latest   string
-	Oses     string
-	Arches   string
-	Libcs    string
-	Formats  string
-}
-
-// UA format from webi.sh bootstrap: "curl {uname -s}/{uname -r} {uname -m}/unknown {libc}"
-// libc is "gnu", "musl", or "libc" (for darwin/other)
-var cases = []testCase{
-	{
-		Name: "goreleaser/linux/x86_64", Package: "goreleaser",
-		OS: buildmeta.OSLinux, Arch: buildmeta.ArchAMD64, Libc: buildmeta.LibcGNU,
-		Formats: []string{".tar.gz", ".tar.xz", ".zip"},
-		UA:      "curl Linux/6.6.123 x86_64/unknown gnu",
-	},
-	{
-		Name: "goreleaser/darwin/arm64", Package: "goreleaser",
-		OS: buildmeta.OSDarwin, Arch: buildmeta.ArchARM64, Libc: "",
-		Formats: []string{".tar.gz", ".tar.xz", ".zip"},
-		UA:      "curl Darwin/25.2.0 arm64/unknown libc",
-	},
-	{
-		Name: "goreleaser/windows/x86_64", Package: "goreleaser",
-		OS: buildmeta.OSWindows, Arch: buildmeta.ArchAMD64, Libc: "",
-		Formats: []string{".zip", ".exe"},
-		UA:      "PowerShell/7.0 Windows/10.0 x86_64/unknown msvc",
-	},
-	{
-		Name: "ollama/linux/x86_64", Package: "ollama",
-		OS: buildmeta.OSLinux, Arch: buildmeta.ArchAMD64, Libc: buildmeta.LibcGNU,
-		Formats: []string{".tar.gz", ".tar.xz", ".tar.zst", ".zip"},
-		UA:      "curl Linux/6.6.123 x86_64/unknown gnu",
-	},
-	{
-		Name: "ollama/darwin/arm64", Package: "ollama",
-		OS: buildmeta.OSDarwin, Arch: buildmeta.ArchARM64, Libc: "",
-		Formats: []string{".tar.gz", ".tar.xz", ".tar.zst", ".zip", ".dmg"},
-		UA:      "curl Darwin/25.2.0 arm64/unknown libc",
-	},
-	{
-		Name: "ollama/linux/arm64", Package: "ollama",
-		OS: buildmeta.OSLinux, Arch: buildmeta.ArchARM64, Libc: buildmeta.LibcGNU,
-		Formats: []string{".tar.gz", ".tar.xz", ".tar.zst", ".zip"},
-		UA:      "curl Linux/6.6.123 aarch64/unknown gnu",
-	},
-	{
-		Name: "node/linux/x86_64", Package: "node",
-		OS: buildmeta.OSLinux, Arch: buildmeta.ArchAMD64, Libc: buildmeta.LibcGNU,
-		Formats: []string{".tar.xz", ".tar.gz", ".zip"},
-		UA:      "curl Linux/6.6.123 x86_64/unknown gnu",
-	},
-	{
-		Name: "node/darwin/arm64", Package: "node",
-		OS: buildmeta.OSDarwin, Arch: buildmeta.ArchARM64, Libc: "",
-		Formats: []string{".tar.xz", ".tar.gz", ".zip"},
-		UA:      "curl Darwin/25.2.0 arm64/unknown libc",
-	},
-	{
-		Name: "node/linux/arm64", Package: "node",
-		OS: buildmeta.OSLinux, Arch: buildmeta.ArchARM64, Libc: buildmeta.LibcGNU,
-		Formats: []string{".tar.xz", ".tar.gz", ".zip"},
-		UA:      "curl Linux/6.6.123 aarch64/unknown gnu",
-	},
-}
-
-func main() {
-	cacheDir := flag.String("cache", "_cache/raw", "root directory for raw cache")
-	confDir := flag.String("conf", ".", "root directory containing {pkg}/releases.conf files")
-	token := flag.String("token", os.Getenv("GITHUB_TOKEN"), "GitHub API token")
-	skipFetch := flag.Bool("skip-fetch", false, "skip fetching, use existing cache")
-	skipLive := flag.Bool("skip-live", false, "skip live API comparison")
-	packages := flag.String("packages", "goreleaser,ollama,node", "comma-separated packages to test")
-	flag.Parse()
-
-	pkgList := strings.Split(*packages, ",")
-	pkgSet := make(map[string]bool, len(pkgList))
-	for _, p := range pkgList {
-		pkgSet[strings.TrimSpace(p)] = true
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
-	defer cancel()
-
-	client := &http.Client{Timeout: 30 * time.Second}
-	var auth *githubish.Auth
-	if *token != "" {
-		auth = &githubish.Auth{Token: *token}
-	}
-
-	// Step 1: Fetch raw releases.
-	if !*skipFetch {
-		log.Println("=== Step 1: Fetching releases ===")
-		for _, pkg := range pkgList {
-			if err := fetchPackage(ctx, client, *cacheDir, *confDir, pkg, auth); err != nil {
-				log.Fatalf("fetch %s: %v", pkg, err)
-			}
-		}
-	} else {
-		log.Println("=== Step 1: Skipping fetch (using cache) ===")
-	}
-
-	// Step 2: Classify releases.
-	log.Println("=== Step 2: Classifying releases ===")
-	allDists := make(map[string][]resolve.Dist)
-	for _, pkg := range pkgList {
-		conf, err := installerconf.Read(filepath.Join(*confDir, pkg, "releases.conf"))
-		if err != nil {
-			log.Fatalf("read conf %s: %v", pkg, err)
-		}
-		d, err := rawcache.Open(filepath.Join(*cacheDir, pkg))
-		if err != nil {
-			log.Fatalf("open cache %s: %v", pkg, err)
-		}
-		dists, err := classifyFromCache(pkg, conf, d)
-		if err != nil {
-			log.Fatalf("classify %s: %v", pkg, err)
-		}
-		allDists[pkg] = dists
-		log.Printf("  %s: %d distributables", pkg, len(dists))
-
-		// Show catalog.
-		cat := resolve.Survey(dists)
-		log.Printf("    oses=%v arches=%v libcs=%v formats=%v", cat.OSes, cat.Arches, cat.Libcs, cat.Formats)
-		log.Printf("    latest=%s stable=%s", cat.Latest, cat.Stable)
-	}
-
-	// Step 3: Resolve best match for each test case.
-	log.Println("=== Step 3: Resolving best matches ===")
-	type result struct {
-		tc    testCase
-		match *resolve.Match
-		live  *liveResult
-	}
-	var results []result
-	for _, tc := range cases {
-		if !pkgSet[tc.Package] {
-			continue
-		}
-		dists := allDists[tc.Package]
-		q := resolve.Query{
-			OS:      tc.OS,
-			Arch:    tc.Arch,
-			Libc:    tc.Libc,
-			Formats: tc.Formats,
-			Channel: "stable",
-		}
-		m := resolve.Best(dists, q)
-		results = append(results, result{tc: tc, match: m})
-	}
-
-	// Step 4: Compare with live API.
-	if !*skipLive {
-		log.Println("=== Step 4: Comparing with live API ===")
-		for i := range results {
-			tc := results[i].tc
-			live, err := queryLiveAPI(client, tc)
-			if err != nil {
-				log.Printf("  %s: live API error: %v", tc.Name, err)
-				continue
-			}
-			results[i].live = live
-		}
-	}
-
-	// Step 5: Report.
-	log.Println("")
-	log.Println("=== Results ===")
-	log.Println("")
-
-	pass, fail, warn := 0, 0, 0
-	for _, r := range results {
-		tc := r.tc
-		m := r.match
-		live := r.live
-
-		if m == nil {
-			log.Printf("FAIL %s: no match found", tc.Name)
-			fail++
-			continue
-		}
-
-		log.Printf("--- %s ---", tc.Name)
-		log.Printf("  Go:   version=%s file=%s ext=%s url=%s", m.Version, m.Filename, m.Format, m.Download)
-
-		if live != nil {
-			log.Printf("  Live: version=%s file=%s ext=%s url=%s", live.Version, live.PkgFile, live.Ext, live.PkgURL)
-
-			if live.Version == "0.0.0" {
-				log.Printf("  WARN: live API returned error (no match)")
-				warn++
-			} else if m.Version == live.Version && m.Filename == live.PkgFile {
-				log.Printf("  PASS: exact match")
-				pass++
-			} else if m.Version == live.Version && m.Download == live.PkgURL {
-				log.Printf("  PASS: same URL (filename display differs: go=%s live=%s)", m.Filename, live.PkgFile)
-				pass++
-			} else if m.Version == live.Version {
-				log.Printf("  WARN: same version, different file (go=%s live=%s)", m.Filename, live.PkgFile)
-				warn++
-			} else {
-				log.Printf("  DIFF: version mismatch (go=%s live=%s)", m.Version, live.Version)
-				fail++
-			}
-		} else {
-			log.Printf("  (no live comparison)")
-			pass++
-		}
-	}
-
-	log.Println("")
-	log.Printf("Summary: %d pass, %d fail, %d warn (live API errors)", pass, fail, warn)
-
-	if fail > 0 {
-		os.Exit(1)
-	}
-}
-
-// fetchPackage fetches raw releases for one package.
-func fetchPackage(ctx context.Context, client *http.Client, cacheRoot, confDir, pkg string, auth *githubish.Auth) error {
-	conf, err := installerconf.Read(filepath.Join(confDir, pkg, "releases.conf"))
-	if err != nil {
-		return fmt.Errorf("read conf: %w", err)
-	}
-
-	source := conf.Source
-	log.Printf("  %s: source=%s", pkg, source)
-
-	switch source {
-	case "github":
-		return fetchGitHub(ctx, client, cacheRoot, pkg, conf, auth)
-	case "nodedist":
-		return fetchNodeDist(ctx, client, cacheRoot, pkg, conf)
-	default:
-		return fmt.Errorf("unsupported source %q (only github and nodedist for e2e test)", source)
-	}
-}
-
-func fetchGitHub(ctx context.Context, client *http.Client, cacheRoot, pkg string, conf *installerconf.Conf, auth *githubish.Auth) error {
-	owner := conf.Owner
-	repo := conf.Repo
-	tagPrefix := conf.TagPrefix
-
-	d, err := rawcache.Open(filepath.Join(cacheRoot, pkg))
-	if err != nil {
-		return err
-	}
-
-	var added, changed, skipped int
-	var latest string
-	for batch, err := range github.Fetch(ctx, client, owner, repo, auth) {
-		if err != nil {
-			return fmt.Errorf("github %s/%s: %w", owner, repo, err)
-		}
-		for _, rel := range batch {
-			if rel.Draft {
-				continue
-			}
-
-			tag := rel.TagName
-			if tagPrefix != "" {
-				if !strings.HasPrefix(tag, tagPrefix) {
-					continue
-				}
-				tag = strings.TrimPrefix(tag, tagPrefix)
-			}
-
-			data, err := json.Marshal(rel)
-			if err != nil {
-				return fmt.Errorf("marshal %s: %w", tag, err)
-			}
-
-			action, err := d.Merge(tag, data)
-			if err != nil {
-				return err
-			}
-			switch action {
-			case "added":
-				added++
-			case "changed":
-				changed++
-			default:
-				skipped++
-			}
-
-			if latest == "" && !rel.Prerelease {
-				latest = tag
-			}
-		}
-	}
-
-	if latest != "" {
-		current := d.Latest()
-		if current == "" || lexver.Compare(lexver.Parse(latest), lexver.Parse(current)) > 0 {
-			d.SetLatest(latest)
-		}
-	}
-
-	log.Printf("    +%d ~%d =%d latest=%s", added, changed, skipped, d.Latest())
-	return nil
-}
-
-func fetchNodeDist(ctx context.Context, client *http.Client, cacheRoot, pkg string, conf *installerconf.Conf) error {
-	baseURL := conf.BaseURL
-	d, err := rawcache.Open(filepath.Join(cacheRoot, pkg))
-	if err != nil {
-		return err
-	}
-
-	var added, changed, skipped int
-	var latest string
-	for batch, err := range nodedist.Fetch(ctx, client, baseURL) {
-		if err != nil {
-			return fmt.Errorf("nodedist: %w", err)
-		}
-		for _, entry := range batch {
-			tag := entry.Version
-			data, err := json.Marshal(entry)
-			if err != nil {
-				return fmt.Errorf("marshal %s: %w", tag, err)
-			}
-
-			action, err := d.Merge(tag, data)
-			if err != nil {
-				return err
-			}
-			switch action {
-			case "added":
-				added++
-			case "changed":
-				changed++
-			default:
-				skipped++
-			}
-
-			if latest == "" {
-				latest = tag
-			}
-		}
-	}
-
-	if latest != "" {
-		current := d.Latest()
-		if current == "" || lexver.Compare(lexver.Parse(latest), lexver.Parse(current)) > 0 {
-			d.SetLatest(latest)
-		}
-	}
-
-	log.Printf("    +%d ~%d =%d latest=%s", added, changed, skipped, d.Latest())
-	return nil
-}
-
-// classifyFromCache reads the raw cache and produces classified dists.
-func classifyFromCache(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]resolve.Dist, error) {
-	source := conf.Source
-	switch source {
-	case "github":
-		return classifyGitHub(pkg, conf, d)
-	case "nodedist":
-		return classifyNodeDist(pkg, conf, d)
-	default:
-		return nil, fmt.Errorf("unsupported source %q", source)
-	}
-}
-
-func classifyGitHub(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]resolve.Dist, error) {
-	tagPrefix := conf.TagPrefix
-	releases, err := readAllReleases(d)
-	if err != nil {
-		return nil, err
-	}
-
-	var dists []resolve.Dist
-	for _, data := range releases {
-		var rel struct {
-			TagName     string `json:"tag_name"`
-			Prerelease  bool   `json:"prerelease"`
-			Draft       bool   `json:"draft"`
-			PublishedAt string `json:"published_at"`
-			Assets      []struct {
-				Name               string `json:"name"`
-				BrowserDownloadURL string `json:"browser_download_url"`
-				Size               int64  `json:"size"`
-			} `json:"assets"`
-		}
-		if err := json.Unmarshal(data, &rel); err != nil {
-			continue
-		}
-		if rel.Draft {
-			continue
-		}
-
-		version := rel.TagName
-		if tagPrefix != "" {
-			version = strings.TrimPrefix(version, tagPrefix)
-		}
-		// Strip leading "v" for version normalization.
-		version = strings.TrimPrefix(version, "v")
-
-		channel := "stable"
-		if rel.Prerelease {
-			channel = "beta"
-		}
-
-		date := ""
-		if len(rel.PublishedAt) >= 10 {
-			date = rel.PublishedAt[:10]
-		}
-
-		for _, asset := range rel.Assets {
-			if isMetaAsset(asset.Name) {
-				continue
-			}
-
-			r := classifyFilename(asset.Name)
-			extra := detectExtra(asset.Name)
-			dists = append(dists, resolve.Dist{
-				Package:  pkg,
-				Version:  version,
-				Channel:  channel,
-				OS:       r.os,
-				Arch:     r.arch,
-				Libc:     r.libc,
-				Format:   r.format,
-				Download: asset.BrowserDownloadURL,
-				Filename: asset.Name,
-				Size:     asset.Size,
-				Date:     date,
-				Extra:    extra,
-			})
-		}
-	}
-	return dists, nil
-}
-
-func classifyNodeDist(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]resolve.Dist, error) {
-	baseURL := conf.BaseURL
-	releases, err := readAllReleases(d)
-	if err != nil {
-		return nil, err
-	}
-
-	var dists []resolve.Dist
-	for _, data := range releases {
-		var entry struct {
-			Version  string          `json:"version"`
-			Date     string          `json:"date"`
-			Files    []string        `json:"files"`
-			LTS      json.RawMessage `json:"lts"`
-			Security bool            `json:"security"`
-		}
-		if err := json.Unmarshal(data, &entry); err != nil {
-			continue
-		}
-
-		lts := string(entry.LTS) != "false" && string(entry.LTS) != ""
-		version := strings.TrimPrefix(entry.Version, "v")
-
-		// Webi treats even major versions as "stable" (LTS-eligible).
-		channel := "stable"
-		parts := strings.SplitN(version, ".", 2)
-		if len(parts) > 0 {
-			var major int
-			fmt.Sscanf(parts[0], "%d", &major)
-			if major%2 != 0 {
-				channel = "beta"
-			}
-		}
-
-		for _, file := range entry.Files {
-			if file == "src" || file == "headers" {
-				continue
-			}
-			fileDists := expandNodeFile(pkg, entry.Version, version, channel, entry.Date, lts, baseURL, file)
-			dists = append(dists, fileDists...)
-		}
-	}
-	return dists, nil
-}
-
-func expandNodeFile(pkg, rawVersion, version, channel, date string, lts bool, baseURL, file string) []resolve.Dist {
-	parts := strings.Split(file, "-")
-	if len(parts) < 2 {
-		return nil
-	}
-
-	osMap := map[string]string{
-		"osx": "darwin", "linux": "linux", "win": "windows",
-		"sunos": "sunos", "aix": "aix",
-	}
-	archMap := map[string]string{
-		"x64": "x86_64", "x86": "x86", "arm64": "aarch64",
-		"armv7l": "armv7", "armv6l": "armv6",
-		"ppc64": "ppc64", "ppc64le": "ppc64le", "s390x": "s390x",
-		"loong64": "loong64", "riscv64": "riscv64",
-	}
-
-	os_ := osMap[parts[0]]
-	arch := archMap[parts[1]]
-	if os_ == "" || arch == "" {
-		return nil
-	}
-
-	libc := ""
-	pkgType := ""
-	if len(parts) > 2 {
-		pkgType = parts[2]
-	}
-
-	var formats []string
-	switch pkgType {
-	case "musl":
-		libc = "musl"
-		formats = []string{".tar.gz", ".tar.xz"}
-	case "tar":
-		formats = []string{".tar.gz", ".tar.xz"}
-	case "zip":
-		formats = []string{".zip"}
-	case "7z":
-		formats = []string{".7z"}
-	case "pkg":
-		formats = []string{".pkg"}
-	case "msi":
-		formats = []string{".msi"}
-	case "exe":
-		formats = []string{".exe"}
-	case "":
-		formats = []string{".tar.gz", ".tar.xz"}
-	default:
-		return nil
-	}
-
-	if libc == "" && os_ == "linux" {
-		libc = "gnu"
-	}
-
-	osPart := parts[0]
-	if osPart == "osx" {
-		osPart = "darwin"
-	}
-	archPart := parts[1]
-	muslExtra := ""
-	if libc == "musl" {
-		muslExtra = "-musl"
-	}
-
-	var dists []resolve.Dist
-	for _, format := range formats {
-		var filename string
-		if format == ".msi" {
-			filename = fmt.Sprintf("node-%s-%s%s%s", rawVersion, archPart, muslExtra, format)
-		} else {
-			filename = fmt.Sprintf("node-%s-%s-%s%s%s", rawVersion, osPart, archPart, muslExtra, format)
-		}
-
-		dists = append(dists, resolve.Dist{
-			Package:  pkg,
-			Version:  version,
-			Channel:  channel,
-			OS:       os_,
-			Arch:     arch,
-			Libc:     libc,
-			Format:   format,
-			Download: fmt.Sprintf("%s/%s/%s", baseURL, rawVersion, filename),
-			Filename: filename,
-			LTS:      lts,
-			Date:     date,
-		})
-	}
-	return dists
-}
-
-// queryLiveAPI queries the live webi.sh API and parses the response header.
-func queryLiveAPI(client *http.Client, tc testCase) (*liveResult, error) {
-	// Build format string matching what the webi.sh bootstrap sends.
-	// Order: tar,exe,zip,xz,dmg,git (least to most favorable in bootstrap,
-	// but the API doesn't care about order).
-	fmtParam := "tar,exe,zip,xz,dmg"
-
-	url := fmt.Sprintf("https://webi.sh/api/installers/%s@stable.sh?formats=%s", tc.Package, fmtParam)
-
-	req, err := http.NewRequest("GET", url, nil)
-	if err != nil {
-		return nil, err
-	}
-	req.Header.Set("User-Agent", tc.UA)
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
-	}
-
-	return parseLiveResponse(string(body)), nil
-}
-
-// parseLiveResponse extracts WEBI_* and PKG_* variables from the shell script.
-func parseLiveResponse(body string) *liveResult {
-	vars := make(map[string]string)
-	scanner := bufio.NewScanner(strings.NewReader(body))
-	for scanner.Scan() {
-		line := strings.TrimSpace(scanner.Text())
-		for _, prefix := range []string{"WEBI_", "PKG_"} {
-			if strings.HasPrefix(line, prefix) {
-				if eq := strings.IndexByte(line, '='); eq > 0 {
-					key := line[:eq]
-					val := line[eq+1:]
-					val = strings.Trim(val, "'\"")
-					vars[key] = val
-				}
-			}
-		}
-	}
-
-	return &liveResult{
-		Version: vars["WEBI_VERSION"],
-		OS:      vars["WEBI_OS"],
-		Arch:    vars["WEBI_ARCH"],
-		Libc:    vars["WEBI_LIBC"],
-		Ext:     vars["WEBI_EXT"],
-		PkgURL:  vars["WEBI_PKG_URL"],
-		PkgFile: vars["WEBI_PKG_FILE"],
-		Channel: vars["WEBI_CHANNEL"],
-		Stable:  vars["PKG_STABLE"],
-		Latest:  vars["PKG_LATEST"],
-		Oses:    vars["PKG_OSES"],
-		Arches:  vars["PKG_ARCHES"],
-		Libcs:   vars["PKG_LIBCS"],
-		Formats: vars["PKG_FORMATS"],
-	}
-}
-
-// readAllReleases reads all cached release files.
-func readAllReleases(d *rawcache.Dir) (map[string][]byte, error) {
-	active, err := d.ActivePath()
-	if err != nil {
-		return nil, err
-	}
-	entries, err := os.ReadDir(active)
-	if err != nil {
-		return nil, err
-	}
-	result := make(map[string][]byte, len(entries))
-	for _, e := range entries {
-		if e.IsDir() || strings.HasPrefix(e.Name(), "_") {
-			continue
-		}
-		data, err := os.ReadFile(filepath.Join(active, e.Name()))
-		if err != nil {
-			return nil, err
-		}
-		result[e.Name()] = data
-	}
-	return result, nil
-}
-
-type classResult struct {
-	os, arch, libc, format string
-}
-
-func classifyFilename(name string) classResult {
-	// Use the classify package.
-	// Import it indirectly to avoid circular deps — inline the logic
-	// we need for the e2e test.
-	lower := strings.ToLower(name)
-
-	var r classResult
-	r.format = detectFormat(name)
-
-	// OS detection
-	switch {
-	case strings.Contains(lower, "linux"):
-		r.os = "linux"
-	case strings.Contains(lower, "darwin") || strings.Contains(lower, "macos") || strings.Contains(lower, "apple"):
-		r.os = "darwin"
-	case strings.Contains(lower, "windows") || strings.Contains(lower, "win64") || strings.Contains(lower, "win32"):
-		r.os = "windows"
-	case strings.HasSuffix(lower, ".dmg") || strings.HasSuffix(lower, ".app.zip"):
-		r.os = "darwin"
-	case strings.HasSuffix(lower, ".exe") || strings.HasSuffix(lower, ".msi"):
-		r.os = "windows"
-	case strings.Contains(lower, "freebsd"):
-		r.os = "freebsd"
-	}
-
-	// Arch detection
-	switch {
-	case strings.Contains(lower, "x86_64") || strings.Contains(lower, "amd64") || strings.Contains(lower, "x64"):
-		r.arch = "x86_64"
-	case strings.Contains(lower, "aarch64") || strings.Contains(lower, "arm64"):
-		r.arch = "aarch64"
-	case strings.Contains(lower, "armv7") || strings.Contains(lower, "armhf"):
-		r.arch = "armv7"
-	case strings.Contains(lower, "armv6"):
-		r.arch = "armv6"
-	case strings.Contains(lower, "i686") || strings.Contains(lower, "i386") || strings.Contains(lower, "x86") || strings.Contains(lower, "386"):
-		r.arch = "x86"
-	case strings.Contains(lower, "ppc64le") || strings.Contains(lower, "powerpc64le"):
-		r.arch = "ppc64le"
-	case strings.Contains(lower, "ppc64") || strings.Contains(lower, "powerpc64"):
-		r.arch = "ppc64"
-	case strings.Contains(lower, "riscv64"):
-		r.arch = "riscv64"
-	case strings.Contains(lower, "s390x"):
-		r.arch = "s390x"
-	case strings.Contains(lower, "loong64"):
-		r.arch = "loong64"
-	}
-
-	// Libc detection
-	switch {
-	case strings.Contains(lower, "musl"):
-		r.libc = "musl"
-	case strings.Contains(lower, "gnu"):
-		r.libc = "gnu"
-	case strings.Contains(lower, "msvc"):
-		r.libc = "msvc"
-	}
-
-	return r
-}
-
-func detectFormat(name string) string {
-	lower := strings.ToLower(name)
-	for _, ext := range []string{".tar.gz", ".tar.xz", ".tar.bz2", ".tar.zst", ".exe.xz", ".app.zip"} {
-		if strings.HasSuffix(lower, ext) {
-			return ext
-		}
-	}
-	// .tgz is a common alias for .tar.gz
-	if strings.HasSuffix(lower, ".tgz") {
-		return ".tar.gz"
-	}
-	return filepath.Ext(lower)
-}
-
-// detectExtra identifies GPU/vendor-specific variant suffixes in filenames
-// like "ollama-linux-amd64-rocm.tar.zst" or "ollama-linux-arm64-jetpack5.tar.zst".
-func detectExtra(name string) string {
-	lower := strings.ToLower(name)
-	for _, variant := range []string{
-		"-rocm", "-jetpack", "-cuda", "-vulkan", "-metal",
-		"-extended", "-static", "-debug", "-nightly",
-	} {
-		if strings.Contains(lower, variant) {
-			return strings.TrimPrefix(variant, "-")
-		}
-	}
-	return ""
-}
-
-func isMetaAsset(name string) bool {
-	lower := strings.ToLower(name)
-	for _, suffix := range []string{
-		".sha256", ".sha256sum", ".sha512", ".sha512sum",
-		".md5", ".md5sum", ".sig", ".asc", ".pem",
-		"checksums.txt", "sha256sums", "sha512sums",
-		".sbom", ".spdx", ".json.sig", ".sigstore",
-		".d.ts", ".pub",
-	} {
-		if strings.HasSuffix(lower, suffix) {
-			return true
-		}
-	}
-	for _, contains := range []string{
-		"checksums", "sha256sum", "sha512sum",
-		"buildable-artifact",
-	} {
-		if strings.Contains(lower, contains) {
-			return true
-		}
-	}
-	for _, exact := range []string{
-		"install.sh", "install.ps1", "compat.json",
-	} {
-		if lower == exact {
-			return true
-		}
-	}
-	return false
-}

cmd/fetchraw/main.go

@@ -1,862 +0,0 @@
-// Command fetchraw fetches release histories from upstream APIs and
-// merges them into rawcache. Safe to run repeatedly — unchanged releases
-// are skipped, new/changed ones are recorded in the audit log.
-//
-// Reads releases.conf files from package directories to discover what
-// to fetch. Adding a new package is just creating a conf file.
-//
-// Usage:
-//
-//	go run ./cmd/fetchraw -cache ./_cache/raw
-//	go run ./cmd/fetchraw -cache ./_cache/raw hugo caddy
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"flag"
-	"fmt"
-	"log"
-	"net/http"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-	"time"
-
-	"github.com/webinstall/webi-installers/internal/installerconf"
-	"github.com/webinstall/webi-installers/internal/lexver"
-	"github.com/webinstall/webi-installers/internal/rawcache"
-	"github.com/webinstall/webi-installers/internal/releases/chromedist"
-	"github.com/webinstall/webi-installers/internal/releases/flutterdist"
-	"github.com/webinstall/webi-installers/internal/releases/gitea"
-	"github.com/webinstall/webi-installers/internal/releases/github"
-	"github.com/webinstall/webi-installers/internal/releases/githubish"
-	"github.com/webinstall/webi-installers/internal/releases/gittag"
-	"github.com/webinstall/webi-installers/internal/releases/golang"
-	"github.com/webinstall/webi-installers/internal/releases/gpgdist"
-	"github.com/webinstall/webi-installers/internal/releases/hashicorp"
-	"github.com/webinstall/webi-installers/internal/releases/iterm2dist"
-	"github.com/webinstall/webi-installers/internal/releases/juliadist"
-	"github.com/webinstall/webi-installers/internal/releases/mariadbdist"
-	"github.com/webinstall/webi-installers/internal/releases/nodedist"
-	"github.com/webinstall/webi-installers/internal/releases/zigdist"
-)
-
-func main() {
-	cacheDir := flag.String("cache", "_cache/raw", "root directory for raw cache")
-	confDir := flag.String("conf", ".", "root directory containing {pkg}/releases.conf files")
-	token := flag.String("token", os.Getenv("GITHUB_TOKEN"), "GitHub API token")
-	flag.Parse()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
-	defer cancel()
-
-	client := &http.Client{Timeout: 30 * time.Second}
-	var auth *githubish.Auth
-	if *token != "" {
-		auth = &githubish.Auth{Token: *token}
-	}
-
-	// Discover packages from releases.conf files.
-	packages, err := discover(*confDir)
-	if err != nil {
-		log.Fatalf("discover: %v", err)
-	}
-
-	// Filter to requested packages if args given.
-	args := flag.Args()
-	if len(args) > 0 {
-		nameSet := make(map[string]bool, len(args))
-		for _, a := range args {
-			nameSet[a] = true
-		}
-		var filtered []pkgConf
-		for _, p := range packages {
-			if nameSet[p.name] {
-				filtered = append(filtered, p)
-			}
-		}
-		packages = filtered
-	}
-
-	log.Printf("found %d packages", len(packages))
-
-	for _, pkg := range packages {
-		// Aliases share cache with their target — skip fetching.
-		if alias := pkg.conf.Extra["alias_of"]; alias != "" {
-			log.Printf("  %s: alias of %s, skipping", pkg.name, alias)
-			continue
-		}
-
-		log.Printf("fetching %s...", pkg.name)
-		var err error
-		switch pkg.conf.Source {
-		case "github":
-			err = fetchGitHub(ctx, client, *cacheDir, pkg.name, pkg.conf, auth)
-		case "nodedist":
-			err = fetchNodeDist(ctx, client, *cacheDir, pkg.name, pkg.conf)
-		case "golang":
-			err = fetchGolang(ctx, client, *cacheDir, pkg.name)
-		case "zigdist":
-			err = fetchZig(ctx, client, *cacheDir, pkg.name)
-		case "flutterdist":
-			err = fetchFlutter(ctx, client, *cacheDir, pkg.name)
-		case "iterm2dist":
-			err = fetchITerm2(ctx, client, *cacheDir, pkg.name)
-		case "hashicorp":
-			err = fetchHashiCorp(ctx, client, *cacheDir, pkg.name, pkg.conf)
-		case "juliadist":
-			err = fetchJulia(ctx, client, *cacheDir, pkg.name)
-		case "gittag":
-			err = fetchGitTag(ctx, *cacheDir, pkg.name, pkg.conf)
-		case "gitea":
-			err = fetchGitea(ctx, client, *cacheDir, pkg.name, pkg.conf)
-		case "chromedist":
-			err = fetchChrome(ctx, client, *cacheDir, pkg.name)
-		case "gpgdist":
-			err = fetchGPG(ctx, client, *cacheDir, pkg.name)
-		case "mariadbdist":
-			err = fetchMariaDB(ctx, client, *cacheDir, pkg.name)
-		default:
-			log.Printf("  %s: unknown source %q, skipping", pkg.name, pkg.conf.Source)
-			continue
-		}
-		if err != nil {
-			log.Printf("  ERROR: %s: %v", pkg.name, err)
-		}
-	}
-}
-
-type pkgConf struct {
-	name string
-	conf *installerconf.Conf
-}
-
-// discover finds all {dir}/*/releases.conf files and returns them sorted.
-func discover(dir string) ([]pkgConf, error) {
-	pattern := filepath.Join(dir, "*", "releases.conf")
-	matches, err := filepath.Glob(pattern)
-	if err != nil {
-		return nil, err
-	}
-
-	var packages []pkgConf
-	for _, path := range matches {
-		name := filepath.Base(filepath.Dir(path))
-		// Skip infrastructure dirs (_example, _webi, _common, etc.)
-		if strings.HasPrefix(name, "_") {
-			continue
-		}
-		conf, err := installerconf.Read(path)
-		if err != nil {
-			log.Printf("warning: %s: %v", path, err)
-			continue
-		}
-		packages = append(packages, pkgConf{name: name, conf: conf})
-	}
-
-	sort.Slice(packages, func(i, j int) bool {
-		return packages[i].name < packages[j].name
-	})
-
-	return packages, nil
-}
-
-func fetchNodeDist(ctx context.Context, client *http.Client, cacheRoot, pkgName string, conf *installerconf.Conf) error {
-	baseURL := conf.BaseURL
-	if baseURL == "" {
-		return fmt.Errorf("missing url in releases.conf")
-	}
-
-	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
-	if err != nil {
-		return err
-	}
-
-	var added, changed, skipped int
-	var latest string
-	for batch, err := range nodedist.Fetch(ctx, client, baseURL) {
-		if err != nil {
-			return fmt.Errorf("%s fetch: %w", pkgName, err)
-		}
-		for _, entry := range batch {
-			tag := entry.Version
-			data, err := json.Marshal(entry)
-			if err != nil {
-				return fmt.Errorf("%s marshal %s: %w", pkgName, tag, err)
-			}
-
-			action, err := d.Merge(tag, data)
-			if err != nil {
-				return err
-			}
-			switch action {
-			case "added":
-				added++
-			case "changed":
-				changed++
-			default:
-				skipped++
-			}
-
-			if latest == "" {
-				latest = tag
-			}
-		}
-	}
-
-	if err := updateLatest(d, latest); err != nil {
-		return err
-	}
-
-	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
-	return nil
-}
-
-func fetchGitHub(ctx context.Context, client *http.Client, cacheRoot, pkgName string, conf *installerconf.Conf, auth *githubish.Auth) error {
-	owner := conf.Owner
-	repo := conf.Repo
-	tagPrefix := conf.TagPrefix
-
-	if owner == "" || repo == "" {
-		return fmt.Errorf("missing owner or repo in releases.conf")
-	}
-
-	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
-	if err != nil {
-		return err
-	}
-
-	var added, changed, skipped int
-	var latest string
-	for batch, err := range github.Fetch(ctx, client, owner, repo, auth) {
-		if err != nil {
-			return fmt.Errorf("github %s/%s: %w", owner, repo, err)
-		}
-		for _, rel := range batch {
-			if rel.Draft {
-				continue
-			}
-
-			tag := rel.TagName
-
-			if tagPrefix != "" {
-				if !strings.HasPrefix(tag, tagPrefix) {
-					continue
-				}
-				tag = strings.TrimPrefix(tag, tagPrefix)
-			}
-
-			data, err := json.Marshal(rel)
-			if err != nil {
-				return fmt.Errorf("marshal %s: %w", tag, err)
-			}
-
-			action, err := d.Merge(tag, data)
-			if err != nil {
-				return err
-			}
-			switch action {
-			case "added":
-				added++
-			case "changed":
-				changed++
-			default:
-				skipped++
-			}
-
-			if latest == "" && !rel.Prerelease {
-				latest = tag
-			}
-		}
-	}
-
-	if err := updateLatest(d, latest); err != nil {
-		return err
-	}
-
-	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
-	return nil
-}
-
-func updateLatest(d *rawcache.Dir, candidate string) error {
-	if candidate == "" {
-		return nil
-	}
-	current := d.Latest()
-	if current == "" || lexver.Compare(lexver.Parse(candidate), lexver.Parse(current)) > 0 {
-		return d.SetLatest(candidate)
-	}
-	return nil
-}
-
-func fetchGolang(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
-	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
-	if err != nil {
-		return err
-	}
-
-	var added, changed, skipped int
-	var latest string
-	for batch, err := range golang.Fetch(ctx, client) {
-		if err != nil {
-			return fmt.Errorf("golang: %w", err)
-		}
-		for _, rel := range batch {
-			tag := rel.Version // "go1.24.1"
-			data, err := json.Marshal(rel)
-			if err != nil {
-				return fmt.Errorf("golang marshal %s: %w", tag, err)
-			}
-
-			action, err := d.Merge(tag, data)
-			if err != nil {
-				return err
-			}
-			switch action {
-			case "added":
-				added++
-			case "changed":
-				changed++
-			default:
-				skipped++
-			}
-
-			if latest == "" && rel.Stable {
-				latest = tag
-			}
-		}
-	}
-
-	if err := updateLatest(d, latest); err != nil {
-		return err
-	}
-
-	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
-	return nil
-}
-
-func fetchZig(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
-	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
-	if err != nil {
-		return err
-	}
-
-	var added, changed, skipped int
-	var latest string
-	for batch, err := range zigdist.Fetch(ctx, client) {
-		if err != nil {
-			return fmt.Errorf("zigdist: %w", err)
-		}
-		for _, rel := range batch {
-			tag := rel.Version
-			data, err := json.Marshal(rel)
-			if err != nil {
-				return fmt.Errorf("zigdist marshal %s: %w", tag, err)
-			}
-
-			action, err := d.Merge(tag, data)
-			if err != nil {
-				return err
-			}
-			switch action {
-			case "added":
-				added++
-			case "changed":
-				changed++
-			default:
-				skipped++
-			}
-
-			// Stable versions have dots and no dev/pre markers.
-			isStable := strings.Contains(tag, ".") && !strings.ContainsAny(tag, "+-")
-			if isStable {
-				if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
-					latest = tag
-				}
-			}
-		}
-	}
-
-	if err := updateLatest(d, latest); err != nil {
-		return err
-	}
-
-	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
-	return nil
-}
-
-func fetchFlutter(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
-	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
-	if err != nil {
-		return err
-	}
-
-	var added, changed, skipped int
-	var latest string
-	for batch, err := range flutterdist.Fetch(ctx, client) {
-		if err != nil {
-			return fmt.Errorf("flutterdist: %w", err)
-		}
-		for _, rel := range batch {
-			// Use version+channel+os+arch as the tag. The arch is embedded
-			// in the archive path (e.g. flutter_macos_arm64_3.0.0-stable.zip
-			// vs flutter_macos_3.0.0-stable.zip for universal/x64).
-			arch := ""
-			base := filepath.Base(rel.Archive)
-			prefix := "flutter_" + rel.OS + "_"
-			if after, ok := strings.CutPrefix(base, prefix); ok {
-				if !strings.HasPrefix(after, rel.Version) {
-					// There's an arch segment between OS and version.
-					if idx := strings.Index(after, "_"); idx > 0 {
-						arch = after[:idx]
-					}
-				}
-			}
-			tag := fmt.Sprintf("%s-%s-%s", rel.Version, rel.Channel, rel.OS)
-			if arch != "" {
-				tag += "-" + arch
-			}
-			data, err := json.Marshal(rel)
-			if err != nil {
-				return fmt.Errorf("flutterdist marshal %s: %w", tag, err)
-			}
-
-			action, err := d.Merge(tag, data)
-			if err != nil {
-				return err
-			}
-			switch action {
-			case "added":
-				added++
-			case "changed":
-				changed++
-			default:
-				skipped++
-			}
-
-			if latest == "" && rel.Channel == "stable" {
-				latest = tag
-			}
-		}
-	}
-
-	if err := updateLatest(d, latest); err != nil {
-		return err
-	}
-
-	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
-	return nil
-}
-
-func fetchITerm2(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
-	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
-	if err != nil {
-		return err
-	}
-
-	var added, changed, skipped int
-	var latest string
-	for batch, err := range iterm2dist.Fetch(ctx, client) {
-		if err != nil {
-			return fmt.Errorf("iterm2dist: %w", err)
-		}
-		for _, entry := range batch {
-			tag := entry.Version
-			if tag == "" {
-				continue
-			}
-			data, err := json.Marshal(entry)
-			if err != nil {
-				return fmt.Errorf("iterm2dist marshal %s: %w", tag, err)
-			}
-
-			action, err := d.Merge(tag, data)
-			if err != nil {
-				return err
-			}
-			switch action {
-			case "added":
-				added++
-			case "changed":
-				changed++
-			default:
-				skipped++
-			}
-
-			if latest == "" && entry.Channel == "stable" {
-				latest = tag
-			}
-		}
-	}
-
-	if err := updateLatest(d, latest); err != nil {
-		return err
-	}
-
-	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
-	return nil
-}
-
-func fetchHashiCorp(ctx context.Context, client *http.Client, cacheRoot, pkgName string, conf *installerconf.Conf) error {
-	product := conf.Extra["product"]
-	if product == "" {
-		return fmt.Errorf("missing product in releases.conf")
-	}
-
-	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
-	if err != nil {
-		return err
-	}
-
-	var added, changed, skipped int
-	var latest string
-	for idx, err := range hashicorp.Fetch(ctx, client, product) {
-		if err != nil {
-			return fmt.Errorf("hashicorp %s: %w", product, err)
-		}
-		for tag, ver := range idx.Versions {
-			data, err := json.Marshal(ver)
-			if err != nil {
-				return fmt.Errorf("hashicorp marshal %s: %w", tag, err)
-			}
-
-			action, err := d.Merge(tag, data)
-			if err != nil {
-				return err
-			}
-			switch action {
-			case "added":
-				added++
-			case "changed":
-				changed++
-			default:
-				skipped++
-			}
-
-			// Stable = no prerelease markers. Compare all to find highest.
-			isStable := !strings.ContainsAny(tag, "-+")
-			if isStable {
-				if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
-					latest = tag
-				}
-			}
-		}
-	}
-
-	if err := updateLatest(d, latest); err != nil {
-		return err
-	}
-
-	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
-	return nil
-}
-
-func fetchJulia(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
-	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
-	if err != nil {
-		return err
-	}
-
-	var added, changed, skipped int
-	var latest string
-	for batch, err := range juliadist.Fetch(ctx, client) {
-		if err != nil {
-			return fmt.Errorf("juliadist: %w", err)
-		}
-		for _, rel := range batch {
-			tag := rel.Version
-			data, err := json.Marshal(rel)
-			if err != nil {
-				return fmt.Errorf("juliadist marshal %s: %w", tag, err)
-			}
-
-			action, err := d.Merge(tag, data)
-			if err != nil {
-				return err
-			}
-			switch action {
-			case "added":
-				added++
-			case "changed":
-				changed++
-			default:
-				skipped++
-			}
-
-			if rel.Stable {
-				if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
-					latest = tag
-				}
-			}
-		}
-	}
-
-	if err := updateLatest(d, latest); err != nil {
-		return err
-	}
-
-	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
-	return nil
-}
-
-func fetchGitTag(ctx context.Context, cacheRoot, pkgName string, conf *installerconf.Conf) error {
-	gitURL := conf.BaseURL
-	if gitURL == "" {
-		return fmt.Errorf("missing url in releases.conf")
-	}
-
-	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
-	if err != nil {
-		return err
-	}
-
-	repoDir := filepath.Join(cacheRoot, "_repos")
-	if err := os.MkdirAll(repoDir, 0o755); err != nil {
-		return err
-	}
-
-	var added, changed, skipped int
-	var latest string
-	for batch, err := range gittag.Fetch(ctx, gitURL, repoDir) {
-		if err != nil {
-			return fmt.Errorf("gittag %s: %w", pkgName, err)
-		}
-		for _, entry := range batch {
-			tag := entry.Version
-			if tag == "" {
-				tag = "HEAD-" + entry.CommitHash
-			}
-			data, err := json.Marshal(entry)
-			if err != nil {
-				return fmt.Errorf("gittag marshal %s: %w", tag, err)
-			}
-
-			action, err := d.Merge(tag, data)
-			if err != nil {
-				return err
-			}
-			switch action {
-			case "added":
-				added++
-			case "changed":
-				changed++
-			default:
-				skipped++
-			}
-
-			if entry.GitTag != "" && entry.GitTag != "HEAD" {
-				if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
-					latest = tag
-				}
-			}
-		}
-	}
-
-	if err := updateLatest(d, latest); err != nil {
-		return err
-	}
-
-	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
-	return nil
-}
-
-func fetchGitea(ctx context.Context, client *http.Client, cacheRoot, pkgName string, conf *installerconf.Conf) error {
-	baseURL := conf.BaseURL
-	owner := conf.Owner
-	repo := conf.Repo
-
-	if baseURL == "" || owner == "" || repo == "" {
-		return fmt.Errorf("missing base_url, owner, or repo in releases.conf")
-	}
-
-	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
-	if err != nil {
-		return err
-	}
-
-	var added, changed, skipped int
-	var latest string
-	for batch, err := range gitea.Fetch(ctx, client, baseURL, owner, repo, nil) {
-		if err != nil {
-			return fmt.Errorf("gitea %s/%s: %w", owner, repo, err)
-		}
-		for _, rel := range batch {
-			if rel.Draft {
-				continue
-			}
-
-			tag := rel.TagName
-			data, err := json.Marshal(rel)
-			if err != nil {
-				return fmt.Errorf("gitea marshal %s: %w", tag, err)
-			}
-
-			action, err := d.Merge(tag, data)
-			if err != nil {
-				return err
-			}
-			switch action {
-			case "added":
-				added++
-			case "changed":
-				changed++
-			default:
-				skipped++
-			}
-
-			if latest == "" && !rel.Prerelease {
-				latest = tag
-			}
-		}
-	}
-
-	if err := updateLatest(d, latest); err != nil {
-		return err
-	}
-
-	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
-	return nil
-}
-
-func fetchChrome(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
-	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
-	if err != nil {
-		return err
-	}
-
-	var added, changed, skipped int
-	var latest string
-	for batch, err := range chromedist.Fetch(ctx, client) {
-		if err != nil {
-			return fmt.Errorf("chromedist: %w", err)
-		}
-		for _, ver := range batch {
-			tag := ver.Version
-			data, err := json.Marshal(ver)
-			if err != nil {
-				return fmt.Errorf("chromedist marshal %s: %w", tag, err)
-			}
-
-			action, err := d.Merge(tag, data)
-			if err != nil {
-				return err
-			}
-			switch action {
-			case "added":
-				added++
-			case "changed":
-				changed++
-			default:
-				skipped++
-			}
-
-			if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
-				latest = tag
-			}
-		}
-	}
-
-	if err := updateLatest(d, latest); err != nil {
-		return err
-	}
-
-	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
-	return nil
-}
-
-func fetchGPG(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
-	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
-	if err != nil {
-		return err
-	}
-
-	var added, changed, skipped int
-	var latest string
-	for batch, err := range gpgdist.Fetch(ctx, client) {
-		if err != nil {
-			return fmt.Errorf("gpgdist: %w", err)
-		}
-		for _, entry := range batch {
-			tag := entry.Version
-			data, err := json.Marshal(entry)
-			if err != nil {
-				return fmt.Errorf("gpgdist marshal %s: %w", tag, err)
-			}
-
-			action, err := d.Merge(tag, data)
-			if err != nil {
-				return err
-			}
-			switch action {
-			case "added":
-				added++
-			case "changed":
-				changed++
-			default:
-				skipped++
-			}
-
-			if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
-				latest = tag
-			}
-		}
-	}
-
-	if err := updateLatest(d, latest); err != nil {
-		return err
-	}
-
-	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
-	return nil
-}
-
-func fetchMariaDB(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
-	d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
-	if err != nil {
-		return err
-	}
-
-	var added, changed, skipped int
-	var latest string
-	for batch, err := range mariadbdist.Fetch(ctx, client) {
-		if err != nil {
-			return fmt.Errorf("mariadbdist: %w", err)
-		}
-		for _, rel := range batch {
-			tag := rel.ReleaseID
-			data, err := json.Marshal(rel)
-			if err != nil {
-				return fmt.Errorf("mariadbdist marshal %s: %w", tag, err)
-			}
-
-			action, err := d.Merge(tag, data)
-			if err != nil {
-				return err
-			}
-			switch action {
-			case "added":
-				added++
-			case "changed":
-				changed++
-			default:
-				skipped++
-			}
-
-			isStable := rel.MajorStatus == "Stable"
-			if isStable {
-				if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
-					latest = tag
-				}
-			}
-		}
-	}
-
-	if err := updateLatest(d, latest); err != nil {
-		return err
-	}
-
-	log.Printf("  %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
-	return nil
-}

cmd/inspect/main.go

@@ -1,625 +0,0 @@
-// Command inspect downloads release archives, unpacks them, and reports
-// their internal structure. This helps discover how packages are laid out
-// and whether the layout changes across versions.
-//
-// Usage:
-//
-//	go run ./cmd/inspect -csv distributables.csv -cache ./_cache/downloads ollama sd
-package main
-
-import (
-	"context"
-	"encoding/csv"
-	"flag"
-	"fmt"
-	"io"
-	"log"
-	"mime"
-	"net/http"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"sort"
-	"strings"
-	"time"
-
-	"github.com/webinstall/webi-installers/internal/httpclient"
-)
-
-// Row is one CSV row from distributables.csv.
-type Row struct {
-	Package  string
-	Version  string
-	Channel  string
-	Date     string
-	OS       string
-	Arch     string
-	Libc     string
-	Format   string
-	Download string
-	Filename string
-	Extra    string
-}
-
-// archiveFormats are the formats we download and unpack.
-var archiveFormats = map[string]bool{
-	".tar.gz":  true,
-	".tar.xz":  true,
-	".tar.bz2": true,
-	".tar.zst": true,
-	".zip":     true,
-	".dmg":     true,
-	".gz":      true,
-	".xz":      true,
-}
-
-// inspectOSes are the OSes we inspect.
-var inspectOSes = map[string]bool{
-	"linux":   true,
-	"darwin":  true,
-	"windows": true,
-	"":        true, // source-only packages
-}
-
-// preferredArch picks one arch per OS to download.
-func preferredArch(os_ string) string {
-	switch os_ {
-	case "darwin":
-		return "aarch64"
-	default:
-		return "x86_64"
-	}
-}
-
-func main() {
-	csvFile := flag.String("csv", "distributables.csv", "path to distributables CSV")
-	cacheDir := flag.String("cache", "_cache/downloads", "download cache directory")
-	flag.Parse()
-
-	packages := flag.Args()
-	if len(packages) == 0 {
-		log.Fatal("usage: inspect [-csv FILE] [-cache DIR] PACKAGE [PACKAGE...]")
-	}
-
-	rows, err := readCSV(*csvFile)
-	if err != nil {
-		log.Fatalf("read csv: %v", err)
-	}
-
-	client := httpclient.New()
-	// Override timeout for large downloads.
-	client.Timeout = 10 * time.Minute
-
-	for _, pkg := range packages {
-		log.Printf("=== %s ===", pkg)
-		if err := inspectPackage(client, rows, pkg, *cacheDir); err != nil {
-			log.Printf("ERROR: %s: %v", pkg, err)
-		}
-	}
-}
-
-func readCSV(path string) ([]Row, error) {
-	f, err := os.Open(path)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-
-	r := csv.NewReader(f)
-	header, err := r.Read()
-	if err != nil {
-		return nil, err
-	}
-
-	// Build column index.
-	idx := make(map[string]int, len(header))
-	for i, col := range header {
-		idx[col] = i
-	}
-
-	var rows []Row
-	for {
-		record, err := r.Read()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return nil, err
-		}
-
-		get := func(col string) string {
-			if i, ok := idx[col]; ok && i < len(record) {
-				return record[i]
-			}
-			return ""
-		}
-
-		rows = append(rows, Row{
-			Package:  get("package"),
-			Version:  get("version"),
-			Channel:  get("channel"),
-			Date:     get("date"),
-			OS:       get("os"),
-			Arch:     get("arch"),
-			Libc:     get("libc"),
-			Format:   get("format"),
-			Download: get("download"),
-			Filename: get("filename"),
-			Extra:    get("extra"),
-		})
-	}
-	return rows, nil
-}
-
-func inspectPackage(client *http.Client, allRows []Row, pkg, cacheDir string) error {
-	// Filter rows for this package.
-	var pkgRows []Row
-	for _, r := range allRows {
-		if r.Package == pkg {
-			pkgRows = append(pkgRows, r)
-		}
-	}
-	if len(pkgRows) == 0 {
-		return fmt.Errorf("no rows found")
-	}
-
-	// Find latest stable version, fall back to any version.
-	versions := findVersionsByDate(pkgRows)
-	if len(versions) == 0 {
-		return fmt.Errorf("no versions found")
-	}
-
-	latestVer := versions[0]
-	log.Printf("  latest version: %s", latestVer)
-
-	// Check if latest has assets uploaded (more than just source tarballs).
-	latestRows := filterVersion(pkgRows, latestVer)
-	hasRealAssets := false
-	for _, r := range latestRows {
-		if r.Extra != "source" && archiveFormats[r.Format] {
-			hasRealAssets = true
-			break
-		}
-	}
-
-	// If latest looks empty, step back one version.
-	if !hasRealAssets && len(versions) > 1 {
-		latestVer = versions[1]
-		latestRows = filterVersion(pkgRows, latestVer)
-		log.Printf("  latest has no assets, using: %s", latestVer)
-	}
-
-	// Inspect the latest version.
-	if err := inspectVersion(client, pkg, latestVer, latestRows, cacheDir); err != nil {
-		return err
-	}
-
-	// Find versions roughly a year apart going back.
-	yearVersions := findYearlyVersions(pkgRows, latestVer)
-	for _, v := range yearVersions {
-		log.Printf("  --- checking %s ---", v)
-		vRows := filterVersion(pkgRows, v)
-		if err := inspectVersion(client, pkg, v, vRows, cacheDir); err != nil {
-			log.Printf("    ERROR: %v", err)
-		}
-	}
-
-	return nil
-}
-
-// findVersionsByDate returns versions sorted newest first, preferring stable.
-func findVersionsByDate(rows []Row) []string {
-	type vInfo struct {
-		version string
-		date    string
-		stable  bool
-	}
-	seen := map[string]*vInfo{}
-	for _, r := range rows {
-		if _, ok := seen[r.Version]; !ok {
-			seen[r.Version] = &vInfo{
-				version: r.Version,
-				date:    r.Date,
-				stable:  r.Channel == "stable",
-			}
-		}
-	}
-
-	var vs []*vInfo
-	for _, v := range seen {
-		vs = append(vs, v)
-	}
-
-	// Sort: stable first, then by date descending, then version descending.
-	sort.Slice(vs, func(i, j int) bool {
-		if vs[i].stable != vs[j].stable {
-			return vs[i].stable
-		}
-		if vs[i].date != vs[j].date {
-			return vs[i].date > vs[j].date
-		}
-		return vs[i].version > vs[j].version
-	})
-
-	result := make([]string, len(vs))
-	for i, v := range vs {
-		result[i] = v.version
-	}
-	return result
-}
-
-// findYearlyVersions picks versions roughly a year apart before the given version.
-func findYearlyVersions(rows []Row, latestVer string) []string {
-	// Find the date of latest.
-	var latestDate string
-	for _, r := range rows {
-		if r.Version == latestVer && r.Date != "" {
-			latestDate = r.Date
-			break
-		}
-	}
-	if latestDate == "" {
-		return nil
-	}
-
-	latestTime, err := time.Parse("2006-01-02", latestDate)
-	if err != nil {
-		return nil
-	}
-
-	// Collect all stable versions with dates.
-	type vd struct {
-		version string
-		date    time.Time
-	}
-	seen := map[string]bool{}
-	var all []vd
-	for _, r := range rows {
-		if seen[r.Version] || r.Date == "" || r.Channel != "stable" {
-			continue
-		}
-		seen[r.Version] = true
-		t, err := time.Parse("2006-01-02", r.Date)
-		if err != nil {
-			continue
-		}
-		if t.Before(latestTime) {
-			all = append(all, vd{r.Version, t})
-		}
-	}
-
-	sort.Slice(all, func(i, j int) bool {
-		return all[i].date.After(all[j].date)
-	})
-
-	// Pick versions roughly a year apart.
-	var result []string
-	nextTarget := latestTime.AddDate(-1, 0, 0)
-	for _, v := range all {
-		if v.date.Before(nextTarget) || v.date.Equal(nextTarget) {
-			result = append(result, v.version)
-			nextTarget = v.date.AddDate(-1, 0, 0)
-		}
-	}
-
-	return result
-}
-
-func filterVersion(rows []Row, version string) []Row {
-	var result []Row
-	for _, r := range rows {
-		if r.Version == version {
-			result = append(result, r)
-		}
-	}
-	return result
-}
-
-// inspectVersion downloads and inspects archives for one version.
-func inspectVersion(client *http.Client, pkg, version string, rows []Row, cacheDir string) error {
-	// Group by OS, pick one arch per OS, pick distinct formats.
-	type dlKey struct {
-		os_    string
-		format string
-	}
-	selected := map[dlKey]*Row{}
-
-	for i := range rows {
-		r := &rows[i]
-		if !inspectOSes[r.OS] {
-			continue
-		}
-		if !archiveFormats[r.Format] {
-			continue
-		}
-
-		key := dlKey{r.OS, r.Format}
-		existing := selected[key]
-		if existing == nil {
-			selected[key] = r
-			continue
-		}
-
-		// Prefer the preferred arch.
-		pref := preferredArch(r.OS)
-		if r.Arch == pref && existing.Arch != pref {
-			selected[key] = r
-		}
-		// Skip rocm/jetpack variants.
-		if strings.Contains(r.Filename, "rocm") || strings.Contains(r.Filename, "jetpack") {
-			if !strings.Contains(existing.Filename, "rocm") && !strings.Contains(existing.Filename, "jetpack") {
-				continue // keep existing non-special variant
-			}
-		}
-	}
-
-	if len(selected) == 0 {
-		log.Printf("  %s: no downloadable archives", version)
-		return nil
-	}
-
-	// Sort keys for deterministic output.
-	var keys []dlKey
-	for k := range selected {
-		keys = append(keys, k)
-	}
-	sort.Slice(keys, func(i, j int) bool {
-		if keys[i].os_ != keys[j].os_ {
-			return keys[i].os_ < keys[j].os_
-		}
-		return keys[i].format < keys[j].format
-	})
-
-	for _, key := range keys {
-		r := selected[key]
-		os_ := r.OS
-		if os_ == "" {
-			os_ = "any"
-		}
-		log.Printf("  [%s] %s %s → %s", version, os_, r.Format, r.Filename)
-
-		dlPath, err := download(client, r.Download, r.Filename, filepath.Join(cacheDir, pkg, version))
-		if err != nil {
-			log.Printf("    download error: %v", err)
-			continue
-		}
-
-		contents, err := unpackAndList(dlPath, r.Format)
-		if err != nil {
-			log.Printf("    unpack error: %v", err)
-			continue
-		}
-
-		printContents(contents)
-	}
-
-	return nil
-}
-
-// download fetches a URL to the cache dir. Returns the path to the cached file.
-// Skips download if the file already exists.
-func download(client *http.Client, url, hintFilename, dir string) (string, error) {
-	// Check if already cached by hint filename.
-	cached := filepath.Join(dir, hintFilename)
-	if _, err := os.Stat(cached); err == nil {
-		return cached, nil
-	}
-
-	if err := os.MkdirAll(dir, 0o755); err != nil {
-		return "", err
-	}
-
-	ctx := context.Background()
-	resp, err := httpclient.Get(ctx, client, url)
-	if err != nil {
-		return "", fmt.Errorf("GET %s: %w", url, err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("GET %s: %s", url, resp.Status)
-	}
-
-	// Determine filename from Content-Disposition or hint.
-	filename := hintFilename
-	if cd := resp.Header.Get("Content-Disposition"); cd != "" {
-		_, params, err := mime.ParseMediaType(cd)
-		if err == nil {
-			if fn, ok := params["filename"]; ok && fn != "" {
-				filename = fn
-			}
-		}
-	}
-
-	outPath := filepath.Join(dir, filename)
-
-	// Atomic write: temp file + rename.
-	tmp := outPath + ".tmp"
-	f, err := os.Create(tmp)
-	if err != nil {
-		return "", err
-	}
-
-	n, err := io.Copy(f, resp.Body)
-	if closeErr := f.Close(); closeErr != nil && err == nil {
-		err = closeErr
-	}
-	if err != nil {
-		os.Remove(tmp)
-		return "", fmt.Errorf("download %s: %w", url, err)
-	}
-
-	if err := os.Rename(tmp, outPath); err != nil {
-		os.Remove(tmp)
-		return "", err
-	}
-
-	log.Printf("    downloaded %s (%d bytes)", filename, n)
-	return outPath, nil
-}
-
-// FileEntry describes one file inside an archive.
-type FileEntry struct {
-	Path       string
-	Size       int64
-	Mode       os.FileMode
-	IsDir      bool
-	IsExec     bool
-	IsSymlink  bool
-	LinkTarget string
-}
-
-// unpackAndList extracts an archive to a temp dir and lists contents.
-func unpackAndList(archivePath, format string) ([]FileEntry, error) {
-	tmpDir, err := os.MkdirTemp("", "webi-inspect-*")
-	if err != nil {
-		return nil, err
-	}
-	defer os.RemoveAll(tmpDir)
-
-	switch format {
-	case ".tar.gz":
-		err = run("tar", "xzf", archivePath, "-C", tmpDir)
-	case ".tar.xz":
-		err = run("tar", "xJf", archivePath, "-C", tmpDir)
-	case ".tar.bz2":
-		err = run("tar", "xjf", archivePath, "-C", tmpDir)
-	case ".tar.zst":
-		err = run("tar", "--zstd", "-xf", archivePath, "-C", tmpDir)
-	case ".zip":
-		err = run("unzip", "-q", "-o", archivePath, "-d", tmpDir)
-	case ".dmg":
-		err = extractDMG(archivePath, tmpDir)
-	case ".gz":
-		// Single file gzip.
-		base := filepath.Base(archivePath)
-		base = strings.TrimSuffix(base, ".gz")
-		outPath := filepath.Join(tmpDir, base)
-		err = run("sh", "-c", fmt.Sprintf("gunzip -c %q > %q", archivePath, outPath))
-	case ".xz":
-		base := filepath.Base(archivePath)
-		base = strings.TrimSuffix(base, ".xz")
-		outPath := filepath.Join(tmpDir, base)
-		err = run("sh", "-c", fmt.Sprintf("xz -dc %q > %q", archivePath, outPath))
-	default:
-		return nil, fmt.Errorf("unsupported format: %s", format)
-	}
-	if err != nil {
-		return nil, fmt.Errorf("extract %s: %w", format, err)
-	}
-
-	return listDir(tmpDir, "")
-}
-
-func extractDMG(dmgPath, outDir string) error {
-	// Try 7z first (doesn't require mounting).
-	if _, err := exec.LookPath("7z"); err == nil {
-		return run("7z", "x", "-o"+outDir, dmgPath)
-	}
-
-	// Fall back to hdiutil mount + copy + unmount.
-	mountPoint, err := os.MkdirTemp("", "webi-dmg-*")
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(mountPoint)
-
-	if err := run("hdiutil", "attach", dmgPath, "-mountpoint", mountPoint, "-nobrowse", "-quiet"); err != nil {
-		return fmt.Errorf("mount dmg: %w", err)
-	}
-	defer run("hdiutil", "detach", mountPoint, "-quiet")
-
-	// Copy contents.
-	return run("cp", "-R", mountPoint+"/.", outDir)
-}
-
-func run(name string, args ...string) error {
-	cmd := exec.Command(name, args...)
-	cmd.Stderr = os.Stderr
-	return cmd.Run()
-}
-
-func listDir(root, prefix string) ([]FileEntry, error) {
-	entries, err := os.ReadDir(filepath.Join(root, prefix))
-	if err != nil {
-		return nil, err
-	}
-
-	var result []FileEntry
-	for _, e := range entries {
-		relPath := filepath.Join(prefix, e.Name())
-		fullPath := filepath.Join(root, relPath)
-
-		info, err := e.Info()
-		if err != nil {
-			continue
-		}
-
-		entry := FileEntry{
-			Path:  relPath,
-			Size:  info.Size(),
-			Mode:  info.Mode(),
-			IsDir: e.IsDir(),
-		}
-
-		if info.Mode()&os.ModeSymlink != 0 {
-			entry.IsSymlink = true
-			target, _ := os.Readlink(fullPath)
-			entry.LinkTarget = target
-		}
-
-		if !e.IsDir() && info.Mode()&0o111 != 0 {
-			entry.IsExec = true
-		}
-
-		result = append(result, entry)
-
-		if e.IsDir() {
-			sub, err := listDir(root, relPath)
-			if err != nil {
-				continue
-			}
-			result = append(result, sub...)
-		}
-	}
-	return result, nil
-}
-
-func printContents(entries []FileEntry) {
-	for _, e := range entries {
-		marker := "  "
-		if e.IsExec {
-			marker = "* "
-		}
-		if e.IsDir {
-			marker = "d "
-		}
-		if e.IsSymlink {
-			marker = "→ "
-		}
-
-		size := ""
-		if !e.IsDir {
-			size = formatSize(e.Size)
-		}
-
-		line := fmt.Sprintf("    %s%-50s %8s", marker, e.Path, size)
-		if e.IsSymlink {
-			line += " → " + e.LinkTarget
-		}
-		log.Print(line)
-	}
-}
-
-func formatSize(n int64) string {
-	switch {
-	case n >= 1<<30:
-		return fmt.Sprintf("%.1fG", float64(n)/float64(1<<30))
-	case n >= 1<<20:
-		return fmt.Sprintf("%.1fM", float64(n)/float64(1<<20))
-	case n >= 1<<10:
-		return fmt.Sprintf("%.1fK", float64(n)/float64(1<<10))
-	default:
-		return fmt.Sprintf("%dB", n)
-	}
-}

cmd/uaparse/main.go

@@ -1,356 +0,0 @@
-// Command uaparse analyzes User-Agent strings from webi.sh logs.
-//
-// It reads UA strings (one per line) from stdin or a file, parses each
-// through uadetect, and produces summary output showing:
-//   - unique platform tuples (os, arch, libc) with counts
-//   - platform hints extracted from kernel version strings (cloud provider,
-//     container runtime, device info)
-//   - detection failures and malformed UAs
-//
-// Usage:
-//
-//	uaparse < LIVE-UAS.txt
-//	uaparse LIVE-UAS.txt
-//	uaparse -json LIVE-UAS.txt
-//	uaparse -fixtures LIVE-UAS.txt    # output Go test fixtures
-package main
-
-import (
-	"bufio"
-	"encoding/json"
-	"flag"
-	"fmt"
-	"os"
-	"regexp"
-	"sort"
-	"strings"
-
-	"github.com/webinstall/webi-installers/internal/uadetect"
-)
-
-// PlatformKey is the resolution-relevant tuple — everything else is noise
-// for artifact selection.
-type PlatformKey struct {
-	OS   string `json:"os"`
-	Arch string `json:"arch"`
-	Libc string `json:"libc"`
-}
-
-func (k PlatformKey) String() string {
-	return fmt.Sprintf("%-10s %-10s %s", k.OS, k.Arch, k.Libc)
-}
-
-// PlatformEntry holds a unique platform and its metadata.
-type PlatformEntry struct {
-	Key      PlatformKey `json:"key"`
-	Count    int         `json:"count"`
-	Examples []string    `json:"examples"` // up to 3 representative UAs
-	Hints    []string    `json:"hints"`    // unique platform hints seen
-}
-
-// UAIssue records a malformed or undetectable UA.
-type UAIssue struct {
-	Line   int    `json:"line"`
-	UA     string `json:"ua"`
-	Reason string `json:"reason"`
-}
-
-// Hint is a platform detail extracted from the kernel version string.
-type Hint struct {
-	Tag   string // short label: "amzn", "azure", "gcp", "wsl", etc.
-	Count int
-}
-
-func main() {
-	jsonOut := flag.Bool("json", false, "output as JSON")
-	fixtures := flag.Bool("fixtures", false, "output Go test fixture table")
-	flag.Parse()
-
-	var scanner *bufio.Scanner
-	if flag.NArg() > 0 {
-		f, err := os.Open(flag.Arg(0))
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "uaparse: %v\n", err)
-			os.Exit(1)
-		}
-		defer f.Close()
-		scanner = bufio.NewScanner(f)
-	} else {
-		scanner = bufio.NewScanner(os.Stdin)
-	}
-
-	platforms := make(map[PlatformKey]*PlatformEntry)
-	hints := make(map[string]int)
-	var issues []UAIssue
-	lineNum := 0
-
-	for scanner.Scan() {
-		lineNum++
-		ua := strings.TrimSpace(scanner.Text())
-		if ua == "" {
-			continue
-		}
-
-		// Detect corruption: truncated/double-pasted lines.
-		if isMalformed(ua) {
-			issues = append(issues, UAIssue{
-				Line:   lineNum,
-				UA:     ua,
-				Reason: "malformed (truncated or corrupted)",
-			})
-			continue
-		}
-
-		// Parse through uadetect.
-		result := uadetect.Parse(ua)
-
-		// Check for detection failures.
-		if result.OS == "" {
-			issues = append(issues, UAIssue{
-				Line:   lineNum,
-				UA:     ua,
-				Reason: "OS not detected",
-			})
-		}
-		if result.Arch == "" {
-			issues = append(issues, UAIssue{
-				Line:   lineNum,
-				UA:     ua,
-				Reason: "arch not detected",
-			})
-		}
-
-		key := PlatformKey{
-			OS:   string(result.OS),
-			Arch: string(result.Arch),
-			Libc: string(result.Libc),
-		}
-
-		entry, ok := platforms[key]
-		if !ok {
-			entry = &PlatformEntry{Key: key}
-			platforms[key] = entry
-		}
-		entry.Count++
-		if len(entry.Examples) < 3 {
-			entry.Examples = append(entry.Examples, ua)
-		}
-
-		// Extract platform hints from kernel version.
-		for _, h := range extractHints(ua) {
-			if !containsStr(entry.Hints, h) {
-				entry.Hints = append(entry.Hints, h)
-			}
-			hints[h]++
-		}
-	}
-
-	if err := scanner.Err(); err != nil {
-		fmt.Fprintf(os.Stderr, "uaparse: read error: %v\n", err)
-		os.Exit(1)
-	}
-
-	// Sort platforms by count descending.
-	entries := make([]*PlatformEntry, 0, len(platforms))
-	for _, e := range platforms {
-		entries = append(entries, e)
-	}
-	sort.Slice(entries, func(i, j int) bool {
-		return entries[i].Count > entries[j].Count
-	})
-
-	if *jsonOut {
-		outputJSON(entries, issues, hints)
-	} else if *fixtures {
-		outputFixtures(entries)
-	} else {
-		outputTable(entries, issues, hints, lineNum)
-	}
-}
-
-func outputTable(entries []*PlatformEntry, issues []UAIssue, hints map[string]int, total int) {
-	fmt.Printf("=== UA Analysis: %d lines → %d unique platforms ===\n\n", total, len(entries))
-
-	fmt.Printf("%-10s %-10s %-6s %6s  %s\n", "OS", "ARCH", "LIBC", "COUNT", "HINTS")
-	fmt.Println(strings.Repeat("-", 72))
-	for _, e := range entries {
-		hintStr := ""
-		if len(e.Hints) > 0 {
-			hintStr = strings.Join(e.Hints, ", ")
-		}
-		fmt.Printf("%-10s %-10s %-6s %6d  %s\n",
-			displayOS(e.Key.OS), e.Key.Arch, displayLibc(e.Key.Libc),
-			e.Count, hintStr)
-	}
-
-	if len(hints) > 0 {
-		fmt.Printf("\n=== Platform Hints (environment signals from kernel strings) ===\n\n")
-		sortedHints := make([]Hint, 0, len(hints))
-		for tag, count := range hints {
-			sortedHints = append(sortedHints, Hint{tag, count})
-		}
-		sort.Slice(sortedHints, func(i, j int) bool {
-			return sortedHints[i].Count > sortedHints[j].Count
-		})
-		for _, h := range sortedHints {
-			fmt.Printf("  %-20s %d\n", h.Tag, h.Count)
-		}
-	}
-
-	if len(issues) > 0 {
-		fmt.Printf("\n=== Issues (%d) ===\n\n", len(issues))
-		for _, iss := range issues {
-			fmt.Printf("  line %d: %s\n    %s\n", iss.Line, iss.Reason, iss.UA)
-		}
-	}
-}
-
-func outputJSON(entries []*PlatformEntry, issues []UAIssue, hints map[string]int) {
-	out := struct {
-		Platforms []*PlatformEntry `json:"platforms"`
-		Issues    []UAIssue        `json:"issues"`
-		Hints     map[string]int   `json:"hints"`
-	}{entries, issues, hints}
-
-	enc := json.NewEncoder(os.Stdout)
-	enc.SetIndent("", "  ")
-	_ = enc.Encode(out)
-}
-
-func outputFixtures(entries []*PlatformEntry) {
-	fmt.Println("// Generated by cmd/uaparse from live UA data.")
-	fmt.Println("// Each entry represents a unique (os, arch, libc) platform seen in production.")
-	fmt.Println("var liveUAPlatforms = []struct {")
-	fmt.Println("\tua   string")
-	fmt.Println("\tos   buildmeta.OS")
-	fmt.Println("\tarch buildmeta.Arch")
-	fmt.Println("\tlibc buildmeta.Libc")
-	fmt.Println("}{")
-
-	for _, e := range entries {
-		if e.Key.OS == "" || e.Key.Arch == "" {
-			continue // skip undetectable
-		}
-		ua := e.Examples[0]
-		fmt.Printf("\t{%q, %s, %s, %s},\n",
-			ua, goConst("OS", e.Key.OS), goConst("Arch", e.Key.Arch), goConst("Libc", e.Key.Libc))
-	}
-
-	fmt.Println("}")
-}
-
-// isMalformed checks for genuinely corrupted UA strings (network truncation).
-func isMalformed(ua string) bool {
-	// Extremely short (less than 10 chars) suggests truncation.
-	if len(ua) < 10 {
-		return true
-	}
-
-	return false
-}
-
-// extractHints finds environment signals in a UA string.
-func extractHints(ua string) []string {
-	lower := strings.ToLower(ua)
-	var out []string
-
-	patterns := []struct {
-		substr string
-		tag    string
-	}{
-		{"amzn", "amzn"},         // Amazon Linux
-		{"-azure", "azure"},     // Azure VM
-		{"-gcp", "gcp"},         // Google Cloud
-		{"-aws", "aws"},         // AWS kernel
-		{"-oracle", "oracle"},   // Oracle Cloud
-		{"el7", "rhel7"},        // RHEL/CentOS 7
-		{"el8", "rhel8"},        // RHEL/CentOS 8
-		{"el9", "rhel9"},        // RHEL/CentOS 9
-		{".fc", "fedora"},       // Fedora
-		{"+deb", "debian"},      // Debian
-		{"-generic", "ubuntu"},  // Ubuntu generic kernel
-		{"-pve", "proxmox"},     // Proxmox VE
-		{"linuxkit", "docker"},  // Docker Desktop / linuxkit
-		{"orbstack", "orbstack"},
-		{"microsoft-standard-wsl", "wsl"},
-		{"android", "android"},
-		{"+rpt-rpi", "rpi"},     // Raspberry Pi
-		{"cygwin", "cygwin"},
-		{"mingw", "mingw"},
-		{"msys", "msys"},
-		{"freebsd", "freebsd"},
-		{"-nvidia", "nvidia"},
-		{"gentoo", "gentoo"},
-		{"coreweave", "coreweave"},
-	}
-
-	for _, p := range patterns {
-		if strings.Contains(lower, p.substr) {
-			out = append(out, p.tag)
-		}
-	}
-
-	return out
-}
-
-// androidDeviceRe extracts device/build info from Android kernel strings.
-var androidDeviceRe = regexp.MustCompile(`ab[A-Z0-9]+`)
-
-func displayOS(os string) string {
-	if os == "" {
-		return "(none)"
-	}
-	return os
-}
-
-func displayLibc(libc string) string {
-	if libc == "" {
-		return "(none)"
-	}
-	return libc
-}
-
-func goConst(prefix, val string) string {
-	m := map[string]map[string]string{
-		"OS": {
-			"darwin":  "buildmeta.OSDarwin",
-			"linux":   "buildmeta.OSLinux",
-			"windows": "buildmeta.OSWindows",
-			"freebsd": "buildmeta.OSFreeBSD",
-			"android": "buildmeta.OSAndroid",
-			"":        `""`,
-		},
-		"Arch": {
-			"aarch64": "buildmeta.ArchARM64",
-			"x86_64":  "buildmeta.ArchAMD64",
-			"armv7":   "buildmeta.ArchARMv7",
-			"armv6":   "buildmeta.ArchARMv6",
-			"x86":     "buildmeta.ArchX86",
-			"ppc64le": "buildmeta.ArchPPC64LE",
-			"ppc64":   "buildmeta.ArchPPC64",
-			"s390x":   "buildmeta.ArchS390X",
-			"riscv64": "buildmeta.ArchRISCV64",
-			"":        `""`,
-		},
-		"Libc": {
-			"gnu":  "buildmeta.LibcGNU",
-			"musl": "buildmeta.LibcMusl",
-			"msvc": "buildmeta.LibcMSVC",
-			"none": "buildmeta.LibcNone",
-			"":     `""`,
-		},
-	}
-	if v, ok := m[prefix][val]; ok {
-		return v
-	}
-	return fmt.Sprintf("%q /* unmapped */", val)
-}
-
-func containsStr(ss []string, s string) bool {
-	for _, v := range ss {
-		if v == s {
-			return true
-		}
-	}
-	return false
-}

cmd/webicached/main.go

@@ -1,899 +0,0 @@
-// Command webicached is the release cache daemon. It fetches releases
-// from upstream sources, classifies build assets, and writes them to
-// the _cache/ directory in the format the Node.js server expects.
-//
-// This is the Go replacement for the Node.js release-fetching pipeline.
-// It reads releases.conf files to discover packages, fetches from the
-// configured source, classifies assets, and writes to fsstore.
-//
-// Default mode: classify all from existing rawcache on startup, then
-// fetch+refresh one package per tick (round-robin, 15m default).
-//
-// Usage:
-//
-//	go run ./cmd/webicached                # default: round-robin, one per tick
-//	go run ./cmd/webicached -eager         # fetch all packages on startup
-//	go run ./cmd/webicached -once -no-fetch # classify from rawcache and exit
-//	go run ./cmd/webicached bat goreleaser # only these packages
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"log"
-	"math/rand/v2"
-	"net/http"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-	"time"
-
-	"github.com/joho/godotenv"
-	"github.com/webinstall/webi-installers/internal/classifypkg"
-	"github.com/webinstall/webi-installers/internal/installerconf"
-	"github.com/webinstall/webi-installers/internal/rawcache"
-	"github.com/webinstall/webi-installers/internal/releases/chromedist"
-	"github.com/webinstall/webi-installers/internal/releases/flutterdist"
-	"github.com/webinstall/webi-installers/internal/releases/gitea"
-	"github.com/webinstall/webi-installers/internal/releases/github"
-	"github.com/webinstall/webi-installers/internal/releases/githubish"
-	"github.com/webinstall/webi-installers/internal/releases/gittag"
-	"github.com/webinstall/webi-installers/internal/releases/golang"
-	"github.com/webinstall/webi-installers/internal/releases/gpgdist"
-	"github.com/webinstall/webi-installers/internal/releases/hashicorp"
-	"github.com/webinstall/webi-installers/internal/releases/iterm2dist"
-	"github.com/webinstall/webi-installers/internal/releases/juliadist"
-	"github.com/webinstall/webi-installers/internal/releases/mariadbdist"
-	"github.com/webinstall/webi-installers/internal/releases/nodedist"
-	"github.com/webinstall/webi-installers/internal/releases/servicemandist"
-	"github.com/webinstall/webi-installers/internal/releases/zigdist"
-	"github.com/webinstall/webi-installers/internal/storage"
-	"github.com/webinstall/webi-installers/internal/storage/fsstore"
-	"github.com/webinstall/webi-installers/internal/storage/pgstore"
-)
-
-var (
-	name         = "webicached"
-	version      = "0.0.0-dev"
-	commit       = "0000000"
-	date         = "0001-01-01"
-	licenseYear  = "2024"
-	licenseOwner = "AJ ONeal"
-	licenseType  = "MPL-2.0"
-)
-
-func printVersion(w io.Writer) {
-	b_ver := strings.TrimPrefix(version, "v")
-	_, _ = fmt.Fprintf(w, "%s v%s %s (%s)\n", name, b_ver, commit[:7], date)
-	_, _ = fmt.Fprintf(w, "Copyright (C) %s %s\n", licenseYear, licenseOwner)
-	_, _ = fmt.Fprintf(w, "Licensed under %s\n", licenseType)
-}
-
-type MainConfig struct {
-	envFile   string
-	confDir   string
-	cacheDir  string
-	pgDSN     string
-	rawDir    string
-	token     string
-	once      bool
-	noFetch   bool
-	shallow   bool
-	eager     bool
-	interval  time.Duration
-	pageDelay time.Duration
-}
-
-// WebiCache holds the configuration for the cache daemon.
-type WebiCache struct {
-	ConfDir   string          // root directory with {pkg}/releases.conf files
-	Store     storage.Store   // classified asset storage (fsstore or pgstore)
-	RawDir    string          // raw upstream response cache
-	Client    *http.Client    // HTTP client for upstream calls
-	Auth      *githubish.Auth // GitHub API auth (optional)
-	Shallow   bool            // fetch only the first page of releases
-	NoFetch   bool            // skip fetching, classify from existing raw data only
-	PageDelay time.Duration   // delay between paginated API requests
-}
-
-// delayTransport wraps an http.RoundTripper to add a delay between requests.
-type delayTransport struct {
-	base  http.RoundTripper
-	delay time.Duration
-	last  time.Time
-}
-
-func (t *delayTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	if !t.last.IsZero() && t.delay > 0 {
-		if wait := t.delay - time.Since(t.last); wait > 0 {
-			time.Sleep(wait)
-		}
-	}
-	t.last = time.Now()
-	return t.base.RoundTrip(req)
-}
-
-func main() {
-	if len(os.Args) > 1 {
-		switch os.Args[1] {
-		case "-V", "-version", "--version", "version":
-			printVersion(os.Stdout)
-			os.Exit(0)
-		case "help", "-help", "--help":
-			printVersion(os.Stdout)
-			fmt.Fprintln(os.Stdout, "")
-			fs := flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
-			fs.SetOutput(os.Stdout)
-			registerFlags(fs, &MainConfig{})
-			fs.Usage()
-			os.Exit(0)
-		}
-	}
-
-	cfg := MainConfig{}
-	fs := flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
-	registerFlags(fs, &cfg)
-	if err := fs.Parse(os.Args[1:]); err != nil {
-		if errors.Is(err, flag.ErrHelp) {
-			os.Exit(0)
-		}
-		os.Exit(1)
-	}
-
-	cfg.cacheDir = expandHome(cfg.cacheDir)
-	cfg.rawDir = expandHome(cfg.rawDir)
-
-	if cfg.envFile != "" {
-		if err := godotenv.Load(cfg.envFile); err != nil {
-			log.Fatalf("envfile: %v", err)
-		}
-	}
-	if cfg.token == "" {
-		cfg.token = os.Getenv("GITHUB_TOKEN")
-	}
-
-	var store storage.Store
-	if cfg.pgDSN != "" {
-		pg, err := pgstore.New(context.Background(), cfg.pgDSN)
-		if err != nil {
-			log.Fatalf("pgstore: %v", err)
-		}
-		store = pg
-	} else {
-		fs, err := fsstore.New(cfg.cacheDir)
-		if err != nil {
-			log.Fatalf("fsstore: %v", err)
-		}
-		store = fs
-	}
-
-	var auth *githubish.Auth
-	if cfg.token != "" {
-		auth = &githubish.Auth{Token: cfg.token}
-	}
-
-	client := &http.Client{Timeout: 30 * time.Second}
-	if cfg.pageDelay > 0 {
-		client.Transport = &delayTransport{
-			base:  http.DefaultTransport,
-			delay: cfg.pageDelay,
-		}
-	}
-
-	wc := &WebiCache{
-		ConfDir:   cfg.confDir,
-		Store:     store,
-		RawDir:    cfg.rawDir,
-		Client:    client,
-		Auth:      auth,
-		Shallow:   cfg.shallow,
-		NoFetch:   cfg.noFetch,
-		PageDelay: cfg.pageDelay,
-	}
-
-	filterPkgs := fs.Args()
-
-	if cfg.eager {
-		wc.Run(filterPkgs)
-		if cfg.once {
-			return
-		}
-	} else if cfg.once {
-		wc.Run(filterPkgs)
-		return
-	} else {
-		saved := wc.NoFetch
-		wc.NoFetch = true
-		wc.Run(filterPkgs)
-		wc.NoFetch = saved
-	}
-
-	packages, err := discover(wc.ConfDir)
-	if err != nil {
-		log.Fatalf("discover: %v", err)
-	}
-	if len(filterPkgs) > 0 {
-		nameSet := make(map[string]bool, len(filterPkgs))
-		for _, a := range filterPkgs {
-			nameSet[a] = true
-		}
-		var filtered []pkgConf
-		for _, p := range packages {
-			if nameSet[p.name] {
-				filtered = append(filtered, p)
-			}
-		}
-		packages = filtered
-	}
-
-	var real []pkgConf
-	for _, pkg := range packages {
-		if pkg.conf.AliasOf == "" {
-			real = append(real, pkg)
-		}
-	}
-
-	log.Printf("refreshing %d packages, interval %s, batch size 20 (ctrl-c to stop)", len(real), cfg.interval)
-	for {
-		stale := wc.stalest(real)
-		if len(stale) == 0 {
-			log.Printf("all packages fresh, sleeping %s", cfg.interval)
-			time.Sleep(cfg.interval)
-			continue
-		}
-
-		batch := stale
-		if len(batch) > 20 {
-			batch = batch[:20]
-		}
-		rand.Shuffle(len(batch), func(i, j int) {
-			batch[i], batch[j] = batch[j], batch[i]
-		})
-
-		log.Printf("batch: %d stale, refreshing %d (most stale first)", len(stale), len(batch))
-		for _, pkg := range batch {
-			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
-			if err := wc.refreshPackage(ctx, pkg); err != nil {
-				log.Printf("  ERROR %s: %v", pkg.name, err)
-			}
-			cancel()
-			time.Sleep(cfg.interval)
-		}
-	}
-}
-
-func registerFlags(fs *flag.FlagSet, cfg *MainConfig) {
-	fs.StringVar(&cfg.envFile, "envfile", "", "path to .env file to load before running")
-	fs.StringVar(&cfg.confDir, "conf", ".", "root directory containing {pkg}/releases.conf files")
-	fs.StringVar(&cfg.cacheDir, "legacy", "~/.cache/webi/legacy", "legacy cache directory (fsstore root)")
-	fs.StringVar(&cfg.pgDSN, "pg", "", "PostgreSQL DSN (enables pgstore; mutually exclusive with -legacy)")
-	fs.StringVar(&cfg.rawDir, "raw", "~/.cache/webi/raw", "raw cache directory for upstream responses")
-	fs.StringVar(&cfg.token, "token", "", "GitHub API token (or set $GITHUB_TOKEN)")
-	fs.BoolVar(&cfg.once, "once", false, "run once then exit (no periodic refresh)")
-	fs.BoolVar(&cfg.noFetch, "no-fetch", false, "skip fetching, classify from existing raw data only")
-	fs.BoolVar(&cfg.shallow, "shallow", false, "fetch only the first page of releases (latest)")
-	fs.BoolVar(&cfg.eager, "eager", false, "fetch all packages on startup (default: one per tick)")
-	fs.DurationVar(&cfg.interval, "interval", 9*time.Second, "delay between individual package fetches")
-	fs.DurationVar(&cfg.pageDelay, "page-delay", 2*time.Second, "delay between paginated API requests")
-}
-
-func expandHome(path string) string {
-	if !strings.HasPrefix(path, "~/") {
-		return path
-	}
-	home, err := os.UserHomeDir()
-	if err != nil {
-		return path
-	}
-	return filepath.Join(home, path[2:])
-}
-
-// stalest returns packages sorted by most stale first (oldest UpdatedAt).
-// Packages with no cache entry or empty assets are considered most stale.
-func (wc *WebiCache) stalest(packages []pkgConf) []pkgConf {
-	type stamped struct {
-		pkg       pkgConf
-		updatedAt time.Time
-	}
-
-	var stale []stamped
-	ctx := context.Background()
-	for _, pkg := range packages {
-		data, err := wc.Store.Load(ctx, pkg.name)
-		var t time.Time
-		hasAssets := false
-		if err == nil && data != nil {
-			t = data.UpdatedAt
-			hasAssets = len(data.Assets) > 0
-		}
-		// Never fetched, or has no assets despite having a timestamp
-		// (e.g. classified from empty rawcache), or older than 10 minutes.
-		if t.IsZero() || !hasAssets || time.Since(t) > 10*time.Minute {
-			stale = append(stale, stamped{pkg: pkg, updatedAt: t})
-		}
-	}
-
-	sort.SliceStable(stale, func(i, j int) bool {
-		ti, tj := stale[i].updatedAt, stale[j].updatedAt
-		if ti.Equal(tj) {
-			return stale[i].pkg.name < stale[j].pkg.name
-		}
-		return ti.Before(tj)
-	})
-
-	result := make([]pkgConf, len(stale))
-	for i, s := range stale {
-		result[i] = s.pkg
-	}
-	return result
-}
-
-// Run discovers packages and refreshes each one.
-func (wc *WebiCache) Run(filterPkgs []string) {
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
-	defer cancel()
-
-	packages, err := discover(wc.ConfDir)
-	if err != nil {
-		log.Printf("discover: %v", err)
-		return
-	}
-
-	if len(filterPkgs) > 0 {
-		nameSet := make(map[string]bool, len(filterPkgs))
-		for _, a := range filterPkgs {
-			nameSet[a] = true
-		}
-		var filtered []pkgConf
-		for _, p := range packages {
-			if nameSet[p.name] {
-				filtered = append(filtered, p)
-			}
-		}
-		packages = filtered
-	}
-
-	var real []pkgConf
-	for _, pkg := range packages {
-		if pkg.conf.AliasOf != "" {
-			continue
-		}
-		real = append(real, pkg)
-	}
-
-	log.Printf("refreshing %d packages", len(real))
-	runStart := time.Now()
-
-	for _, pkg := range real {
-		if err := wc.refreshPackage(ctx, pkg); err != nil {
-			log.Printf("  ERROR %s: %v", pkg.name, err)
-		}
-	}
-
-	log.Printf("refreshed %d packages in %s", len(real), time.Since(runStart))
-}
-
-type pkgConf struct {
-	name string
-	conf *installerconf.Conf
-}
-
-func discover(dir string) ([]pkgConf, error) {
-	pattern := filepath.Join(dir, "*", "releases.conf")
-	matches, err := filepath.Glob(pattern)
-	if err != nil {
-		return nil, err
-	}
-
-	var packages []pkgConf
-	for _, path := range matches {
-		pkgDir := filepath.Dir(path)
-		name := filepath.Base(pkgDir)
-		if strings.HasPrefix(name, "_") {
-			continue
-		}
-
-		// If the package directory is a symlink, treat it as an alias
-		// of the symlink target (e.g. rust.vim → vim-rust).
-		fi, err := os.Lstat(filepath.Join(dir, name))
-		if err != nil {
-			log.Printf("warning: %s: %v", name, err)
-			continue
-		}
-		if fi.Mode()&os.ModeSymlink != 0 {
-			target, err := os.Readlink(filepath.Join(dir, name))
-			if err != nil {
-				log.Printf("warning: readlink %s: %v", name, err)
-				continue
-			}
-			packages = append(packages, pkgConf{
-				name: name,
-				conf: &installerconf.Conf{AliasOf: target},
-			})
-			continue
-		}
-
-		conf, err := installerconf.Read(path)
-		if err != nil {
-			log.Printf("warning: %s: %v", path, err)
-			continue
-		}
-		packages = append(packages, pkgConf{name: name, conf: conf})
-	}
-
-	sort.Slice(packages, func(i, j int) bool {
-		return packages[i].name < packages[j].name
-	})
-	return packages, nil
-}
-
-// refreshPackage does the full pipeline for one package:
-// fetch raw → classify → write to fsstore.
-func (wc *WebiCache) refreshPackage(ctx context.Context, pkg pkgConf) error {
-	pkgStart := time.Now()
-	name := pkg.name
-	conf := pkg.conf
-
-	// Step 1: Fetch raw upstream data to rawcache (unless -no-fetch).
-	if !wc.NoFetch {
-		shallow := wc.Shallow
-		if !shallow {
-			d, err := rawcache.Open(filepath.Join(wc.RawDir, name))
-			if err == nil && d.Populated() {
-				shallow = true
-			}
-		}
-		fetchStart := time.Now()
-		if err := wc.fetchRaw(ctx, pkg, shallow); err != nil {
-			return fmt.Errorf("fetch: %w", err)
-		}
-		log.Printf("  %s: fetch %s", name, time.Since(fetchStart))
-	}
-
-	// Step 2: Classify raw data into assets, tag variants, apply config.
-	classifyStart := time.Now()
-	d, err := rawcache.Open(filepath.Join(wc.RawDir, name))
-	if err != nil {
-		return fmt.Errorf("rawcache open: %w", err)
-	}
-
-	// Open supplementary gittag raw cache if available (for packages with
-	// git_url that use a non-gittag source type like servicemandist).
-	var gitTagDir *rawcache.Dir
-	if conf.GitURL != "" && conf.Source != "gittag" {
-		gd, gdErr := rawcache.Open(filepath.Join(wc.RawDir, "_gittag", name))
-		if gdErr == nil && gd.Populated() {
-			gitTagDir = gd
-		}
-	}
-
-	assets, err := classifypkg.Package(name, conf, d, gitTagDir)
-	if err != nil {
-		return fmt.Errorf("classify: %w", err)
-	}
-	classifyDur := time.Since(classifyStart)
-
-	// Step 3: Write to fsstore.
-	writeStart := time.Now()
-	tx, err := wc.Store.BeginRefresh(ctx, name)
-	if err != nil {
-		return fmt.Errorf("begin refresh: %w", err)
-	}
-	if err := tx.Put(assets); err != nil {
-		tx.Rollback()
-		return fmt.Errorf("put: %w", err)
-	}
-	if err := tx.Commit(ctx); err != nil {
-		return fmt.Errorf("commit: %w", err)
-	}
-	writeDur := time.Since(writeStart)
-
-	log.Printf("  %s: %d assets (classify %s, write %s, total %s)",
-		name, len(assets), classifyDur, writeDur, time.Since(pkgStart))
-	return nil
-}
-
-// --- Fetch raw ---
-
-func (wc *WebiCache) fetchRaw(ctx context.Context, pkg pkgConf, shallow bool) error {
-	switch pkg.conf.Source {
-	case "github", "githubsource":
-		if err := wc.fetchGitHub(ctx, pkg.name, pkg.conf, shallow); err != nil {
-			return err
-		}
-	case "nodedist":
-		return wc.fetchNodeDist(ctx, pkg.name, pkg.conf)
-	case "gittag":
-		return wc.fetchGitTag(ctx, pkg.name, pkg.conf, shallow)
-	case "gitea":
-		return wc.fetchGitea(ctx, pkg.name, pkg.conf, shallow)
-	case "chromedist":
-		return fetchChromeDist(ctx, wc.Client, wc.RawDir, pkg.name)
-	case "flutterdist":
-		return fetchFlutterDist(ctx, wc.Client, wc.RawDir, pkg.name)
-	case "golang":
-		return fetchGolang(ctx, wc.Client, wc.RawDir, pkg.name)
-	case "gpgdist":
-		return fetchGPGDist(ctx, wc.Client, wc.RawDir, pkg.name)
-	case "hashicorp":
-		return fetchHashiCorp(ctx, wc.Client, wc.RawDir, pkg.name, pkg.conf)
-	case "iterm2dist":
-		return fetchITerm2Dist(ctx, wc.Client, wc.RawDir, pkg.name)
-	case "juliadist":
-		return fetchJuliaDist(ctx, wc.Client, wc.RawDir, pkg.name)
-	case "mariadbdist":
-		return fetchMariaDBDist(ctx, wc.Client, wc.RawDir, pkg.name)
-	case "servicemandist":
-		if err := servicemandist.Fetch(ctx, wc.Client, wc.RawDir, pkg.name, wc.Auth, shallow); err != nil {
-			return err
-		}
-	case "zigdist":
-		return fetchZigDist(ctx, wc.Client, wc.RawDir, pkg.name)
-	default:
-		log.Printf("  %s: source %q not yet supported, skipping", pkg.name, pkg.conf.Source)
-		return nil
-	}
-
-	// For non-gittag sources with a git_url, also clone the repo to get
-	// commit hashes. Git entries are classified from this data in
-	// refreshPackage, not from the main raw cache.
-	if pkg.conf.GitURL != "" && pkg.conf.Source != "gittag" {
-		gitShallow := shallow
-		if !wc.Shallow {
-			gd, gdErr := rawcache.Open(filepath.Join(wc.RawDir, "_gittag", pkg.name))
-			if gdErr == nil && !gd.Populated() {
-				gitShallow = false
-			}
-		}
-		if err := wc.fetchGitTagSupplementary(ctx, pkg.name, pkg.conf.GitURL, gitShallow); err != nil {
-			log.Printf("  %s: supplementary gittag fetch: %v", pkg.name, err)
-		}
-	}
-	return nil
-}
-
-// fetchGitTagSupplementary clones a git repo to get commit hashes for
-// packages that use a non-gittag source type (servicemandist, githubsource)
-// but also have a git_url for source installs.
-func (wc *WebiCache) fetchGitTagSupplementary(ctx context.Context, pkgName, gitURL string, shallow bool) error {
-	d, err := rawcache.Open(filepath.Join(wc.RawDir, "_gittag", pkgName))
-	if err != nil {
-		return err
-	}
-
-	repoDir := filepath.Join(wc.RawDir, "_repos")
-	os.MkdirAll(repoDir, 0o755)
-
-	for batch, err := range gittag.Fetch(ctx, gitURL, repoDir) {
-		if err != nil {
-			return err
-		}
-		for _, entry := range batch {
-			tag := entry.Version
-			if tag == "" {
-				tag = "HEAD-" + entry.CommitHash
-			}
-			data, _ := json.Marshal(entry)
-			d.Merge(tag, data)
-		}
-		if shallow {
-			break
-		}
-	}
-	return nil
-}
-
-func (wc *WebiCache) fetchGitHub(ctx context.Context, pkgName string, conf *installerconf.Conf, shallow bool) error {
-	owner, repo := conf.Owner, conf.Repo
-	if owner == "" || repo == "" {
-		return fmt.Errorf("missing owner or repo")
-	}
-
-	d, err := rawcache.Open(filepath.Join(wc.RawDir, pkgName))
-	if err != nil {
-		return err
-	}
-
-	tagPrefix := conf.TagPrefix
-	for batch, err := range github.Fetch(ctx, wc.Client, owner, repo, wc.Auth) {
-		if err != nil {
-			return fmt.Errorf("github %s/%s: %w", owner, repo, err)
-		}
-		for _, rel := range batch {
-			if rel.Draft {
-				continue
-			}
-			tag := rel.TagName
-			if tagPrefix != "" && !strings.HasPrefix(tag, tagPrefix) {
-				continue
-			}
-			data, _ := json.Marshal(rel)
-			d.Merge(tag, data)
-		}
-		if shallow {
-			break
-		}
-	}
-	return nil
-}
-
-func (wc *WebiCache) fetchNodeDist(ctx context.Context, pkgName string, conf *installerconf.Conf) error {
-	baseURL := conf.BaseURL
-	if baseURL == "" {
-		return fmt.Errorf("missing url")
-	}
-
-	d, err := rawcache.Open(filepath.Join(wc.RawDir, pkgName))
-	if err != nil {
-		return err
-	}
-
-	// Fetch from primary URL. Tag with "official/" prefix so unofficial
-	// entries for the same version don't overwrite.
-	for batch, err := range nodedist.Fetch(ctx, wc.Client, baseURL) {
-		if err != nil {
-			return err
-		}
-		for _, entry := range batch {
-			data, _ := json.Marshal(entry)
-			d.Merge("official/"+entry.Version, data)
-		}
-	}
-
-	// Fetch from unofficial URL if configured (e.g. Node.js unofficial builds
-	// which add musl, riscv64, loong64 targets).
-	if unofficialURL := conf.Extra["unofficial_url"]; unofficialURL != "" {
-		for batch, err := range nodedist.Fetch(ctx, wc.Client, unofficialURL) {
-			if err != nil {
-				log.Printf("warning: %s unofficial fetch: %v", pkgName, err)
-				break
-			}
-			for _, entry := range batch {
-				data, _ := json.Marshal(entry)
-				d.Merge("unofficial/"+entry.Version, data)
-			}
-		}
-	}
-
-	return nil
-}
-
-func (wc *WebiCache) fetchGitTag(ctx context.Context, pkgName string, conf *installerconf.Conf, shallow bool) error {
-	gitURL := conf.BaseURL
-	if gitURL == "" {
-		return fmt.Errorf("missing url")
-	}
-
-	d, err := rawcache.Open(filepath.Join(wc.RawDir, pkgName))
-	if err != nil {
-		return err
-	}
-
-	repoDir := filepath.Join(wc.RawDir, "_repos")
-	os.MkdirAll(repoDir, 0o755)
-
-	for batch, err := range gittag.Fetch(ctx, gitURL, repoDir) {
-		if err != nil {
-			return err
-		}
-		for _, entry := range batch {
-			tag := entry.Version
-			if tag == "" {
-				tag = "HEAD-" + entry.CommitHash
-			}
-			data, _ := json.Marshal(entry)
-			d.Merge(tag, data)
-		}
-		if shallow {
-			break
-		}
-	}
-	return nil
-}
-
-func (wc *WebiCache) fetchGitea(ctx context.Context, pkgName string, conf *installerconf.Conf, shallow bool) error {
-	baseURL, owner, repo := conf.BaseURL, conf.Owner, conf.Repo
-	if baseURL == "" || owner == "" || repo == "" {
-		return fmt.Errorf("missing base_url, owner, or repo")
-	}
-
-	d, err := rawcache.Open(filepath.Join(wc.RawDir, pkgName))
-	if err != nil {
-		return err
-	}
-
-	for batch, err := range gitea.Fetch(ctx, wc.Client, baseURL, owner, repo, nil) {
-		if err != nil {
-			return err
-		}
-		for _, rel := range batch {
-			if rel.Draft {
-				continue
-			}
-			data, _ := json.Marshal(rel)
-			d.Merge(rel.TagName, data)
-		}
-		if shallow {
-			break
-		}
-	}
-	return nil
-}
-
-func fetchChromeDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
-	d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
-	if err != nil {
-		return err
-	}
-
-	for batch, err := range chromedist.Fetch(ctx, client) {
-		if err != nil {
-			return fmt.Errorf("chromedist: %w", err)
-		}
-		for _, ver := range batch {
-			data, _ := json.Marshal(ver)
-			d.Merge(ver.Version, data)
-		}
-	}
-	return nil
-}
-
-func fetchFlutterDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
-	d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
-	if err != nil {
-		return err
-	}
-
-	for batch, err := range flutterdist.Fetch(ctx, client) {
-		if err != nil {
-			return fmt.Errorf("flutterdist: %w", err)
-		}
-		for _, rel := range batch {
-			// Key by version+channel+os for uniqueness.
-			key := rel.Version + "-" + rel.Channel + "-" + rel.OS
-			data, _ := json.Marshal(rel)
-			d.Merge(key, data)
-		}
-	}
-	return nil
-}
-
-func fetchGolang(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
-	d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
-	if err != nil {
-		return err
-	}
-
-	for batch, err := range golang.Fetch(ctx, client) {
-		if err != nil {
-			return fmt.Errorf("golang: %w", err)
-		}
-		for _, rel := range batch {
-			data, _ := json.Marshal(rel)
-			d.Merge(rel.Version, data)
-		}
-	}
-	return nil
-}
-
-func fetchGPGDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
-	d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
-	if err != nil {
-		return err
-	}
-
-	for batch, err := range gpgdist.Fetch(ctx, client) {
-		if err != nil {
-			return fmt.Errorf("gpgdist: %w", err)
-		}
-		for _, entry := range batch {
-			data, _ := json.Marshal(entry)
-			d.Merge(entry.Version, data)
-		}
-	}
-	return nil
-}
-
-func fetchHashiCorp(ctx context.Context, client *http.Client, rawDir, pkgName string, conf *installerconf.Conf) error {
-	product := conf.Repo
-	if product == "" {
-		product = pkgName
-	}
-
-	d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
-	if err != nil {
-		return err
-	}
-
-	for idx, err := range hashicorp.Fetch(ctx, client, product) {
-		if err != nil {
-			return fmt.Errorf("hashicorp %s: %w", product, err)
-		}
-		for ver, vdata := range idx.Versions {
-			data, _ := json.Marshal(vdata)
-			d.Merge(ver, data)
-		}
-	}
-	return nil
-}
-
-func fetchITerm2Dist(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
-	d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
-	if err != nil {
-		return err
-	}
-
-	for batch, err := range iterm2dist.Fetch(ctx, client) {
-		if err != nil {
-			return fmt.Errorf("iterm2dist: %w", err)
-		}
-		for _, entry := range batch {
-			key := entry.Version
-			if entry.Channel == "beta" {
-				key += "-beta"
-			}
-			data, _ := json.Marshal(entry)
-			d.Merge(key, data)
-		}
-	}
-	return nil
-}
-
-func fetchJuliaDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
-	d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
-	if err != nil {
-		return err
-	}
-
-	for batch, err := range juliadist.Fetch(ctx, client) {
-		if err != nil {
-			return fmt.Errorf("juliadist: %w", err)
-		}
-		for _, rel := range batch {
-			data, _ := json.Marshal(rel)
-			d.Merge(rel.Version, data)
-		}
-	}
-	return nil
-}
-
-func fetchMariaDBDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
-	d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
-	if err != nil {
-		return err
-	}
-
-	for batch, err := range mariadbdist.Fetch(ctx, client) {
-		if err != nil {
-			return fmt.Errorf("mariadbdist: %w", err)
-		}
-		for _, rel := range batch {
-			data, _ := json.Marshal(rel)
-			d.Merge(rel.ReleaseID, data)
-		}
-	}
-	return nil
-}
-
-func fetchZigDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
-	d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
-	if err != nil {
-		return err
-	}
-
-	for batch, err := range zigdist.Fetch(ctx, client) {
-		if err != nil {
-			return fmt.Errorf("zigdist: %w", err)
-		}
-		for _, rel := range batch {
-			data, _ := json.Marshal(rel)
-			d.Merge(rel.Version, data)
-		}
-	}
-	return nil
-}

commandlinetools/README.md

@@ -1,55 +0,0 @@
----
-title: XCode Command Line Tools
-homepage: https://webinstall.dev/commandlinetools
-tagline: |
-  The XCode Command Line Tools include git, swift, make, clang, and other developer tools
----
-
-## Cheat Sheet
-
-> The developer tools provided by Apple for macOS.
-
-- git
-- swift
-- make
-- clang
-- etc
-
-This is also part of [webi-essentials](../webi-essentials/).
-
-## Table of Contents
-
-- Files
-- Manual Install
-  - macOS
-  - Linux
-  - Alpine
-  - Windows
-
-### Files
-
-These are the files / directories that are created and/or modified with this
-install:
-
-```sh
-/Library/Developer/CommandLineTools/
-```
-
-### How to Install Manually
-
-It's very easy to start the installer:
-
-```sh
-xcode-select --install
-```
-
-The trick is to also have a mechanism to know when it has finished:
-
-```sh
-while ! test -x /Library/Developer/CommandLineTools/usr/bin/git ||
-    ! test -x /Library/Developer/CommandLineTools/usr/bin/make; do
-    sleep 0.25
-done
-
-echo "Command Line Tools Installed"
-```

commandlinetools/install.sh

@@ -1,62 +0,0 @@
-#!/bin/sh
-set -e
-set -u
-
-fn_install_xcode_commandlinetools() { (
-    b_os="$(uname -s)"
-    if test "${b_os}" != 'Darwin'; then
-        echo >&2 'XCode Command Line Tools are for macOS only'
-        return 1
-    fi
-
-    # streamline the output to be pretty
-    fn_check_pkg '/Library/Developer/CommandLineTools/usr/bin/clang' 'clang'
-    fn_check_pkg '/Library/Developer/CommandLineTools/usr/bin/git' 'git'
-    fn_check_pkg '/Library/Developer/CommandLineTools/usr/bin/make' 'make'
-    echo >&2 ""
-
-    # git
-    if xcode-select -p > /dev/null 2> /dev/null; then
-        echo ""
-        return 0
-    fi
-
-    cmd_xcode_cli_tools_install="xcode-select --install"
-    echo "    Running $(t_cmd "${cmd_xcode_cli_tools_install}")"
-    $cmd_xcode_cli_tools_install 2> /dev/null
-    echo ""
-    echo ">>> $(t_attn 'ACTION REQUIRED') <<<"
-    echo ""
-    echo "        $(t_attn "Click") '$(t_bold 'Install')' $(t_attn "in the pop-up")"
-    echo "        (it may appear $(t_em 'under') this window)"
-    echo ""
-    echo "^^^ $(t_attn 'ACTION REQUIRED') ^^^"
-    echo ""
-    printf "    waiting %s to finish installing Command Line Developer Tools ..." "$(t_em 'for you')"
-    while ! test -x /Library/Developer/CommandLineTools/usr/bin/git ||
-        ! test -x /Library/Developer/CommandLineTools/usr/bin/make; do
-        sleep 0.25
-    done
-    echo " $(t_info 'OK')"
-    echo "    Installed to $(t_path '/Library/Developer/CommandLineTools/')"
-    sleep 1
-); }
-
-fn_check_pkg() { (
-    a_pkg="${1}"
-    a_pkgname="${2:-$a_pkg}"
-
-    printf >&2 '    %s %s %s' \
-        "$(t_dim "Checking for")" \
-        "$(t_pkg "${a_pkgname}")" \
-        "$(t_dim "...")"
-
-    if command -v "${a_pkg}" > /dev/null; then
-        echo >&2 " $(t_dim 'OK')"
-        return 0
-    fi
-
-    echo >&2 ' missing'
-); }
-
-fn_install_xcode_commandlinetools

comrak/install.ps1

@@ -3,13 +3,13 @@
 $VERNAME = "$Env:PKG_NAME-v$Env:WEBI_VERSION.exe"
 $EXENAME = "$Env:PKG_NAME.exe"
 # Fetch archive
-if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading $Env:PKG_NAME from $Env:WEBI_PKG_URL to $Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE.part"
     & Move-Item "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE.part" "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 }
 
-if (!(Test-Path -Path "$Env:USERPROFILE\.local\opt\$Env:PKG_NAME-v$Env:WEBI_VERSION\bin\$VERNAME")) {
+IF (!(Test-Path -Path "$Env:USERPROFILE\.local\opt\$Env:PKG_NAME-v$Env:WEBI_VERSION\bin\$VERNAME")) {
     Write-Output "Installing $Env:PKG_NAME"
     # TODO: temp directory
 

comrak/releases.conf

@@ -1 +0,0 @@
-github_releases = kivikakk/comrak

comrak/releases.js

@@ -0,0 +1,40 @@
+'use strict';
+
+var github = require('../_common/github.js');
+var owner = 'kivikakk';
+var repo = 'comrak';
+
+var ODDITIES = ['-musleabihf.1-'];
+
+module.exports = function (request) {
+  return github(request, owner, repo).then(function (all) {
+    let builds = [];
+
+    loopBuilds: for (let build of all.releases) {
+      let isOddity;
+      for (let oddity of ODDITIES) {
+        isOddity = build.name.includes(oddity);
+        if (isOddity) {
+          break;
+        }
+      }
+      if (isOddity) {
+        continue;
+      }
+
+      builds.push(build);
+    }
+
+    all.releases = builds;
+    return all;
+  });
+};
+
+if (module === require.main) {
+  module.exports(require('@root/request')).then(function (all) {
+    all = require('../_webi/normalize.js')(all);
+    all.releases = all.releases.slice(0, 10);
+    //console.info(JSON.stringify(all));
+    console.info(JSON.stringify(all, null, 2));
+  });
+}

crabz/install.ps1

@@ -20,13 +20,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-if (!(Test-Path -Path "$pkg_download")) {
+IF (!(Test-Path -Path "$pkg_download")) {
     Write-Output "Downloading crabz from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-if (!(Test-Path -Path "$pkg_src_cmd")) {
+IF (!(Test-Path -Path "$pkg_src_cmd")) {
     Write-Output "Installing crabz"
 
     # TODO: create package-specific temp directory

crabz/releases.conf

@@ -1 +0,0 @@
-github_releases = sstadick/crabz

crabz/releases.js

@@ -0,0 +1,31 @@
+'use strict';
+
+var github = require('../_common/github.js');
+var owner = 'sstadick';
+var repo = 'crabz';
+
+module.exports = async function (request) {
+  let all = await github(request, owner, repo);
+
+  let releases = [];
+  for (let rel of all.releases) {
+    let isSrc = rel.download.includes('-src.');
+    if (isSrc) {
+      continue;
+    }
+
+    releases.push(rel);
+  }
+  all.releases = releases;
+
+  return all;
+};
+
+if (module === require.main) {
+  module.exports(require('@root/request')).then(function (all) {
+    all = require('../_webi/normalize.js')(all);
+    // just select the first 5 for demonstration
+    all.releases = all.releases.slice(0, 5);
+    console.info(JSON.stringify(all, null, 2));
+  });
+}

curlie/install.ps1

@@ -3,13 +3,13 @@
 $VERNAME = "$Env:PKG_NAME-v$Env:WEBI_VERSION.exe"
 $EXENAME = "$Env:PKG_NAME.exe"
 # Fetch archive
-if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading $Env:PKG_NAME from $Env:WEBI_PKG_URL to $Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE.part"
     & Move-Item "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE.part" "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 }
 
-if (!(Test-Path -Path "$Env:USERPROFILE\.local\bin\$VERNAME")) {
+IF (!(Test-Path -Path "$Env:USERPROFILE\.local\bin\$VERNAME")) {
     Write-Output "Installing $Env:PKG_NAME"
     # TODO: temp directory
 

curlie/releases.conf

@@ -1 +0,0 @@
-github_releases = rs/curlie

curlie/releases.js

@@ -0,0 +1,21 @@
+'use strict';
+
+var github = require('../_common/github.js');
+var owner = 'rs';
+var repo = 'curlie';
+
+module.exports = function (request) {
+  return github(request, owner, repo).then(function (all) {
+    all._names = ['curlie', 'curl-httpie'];
+    return all;
+  });
+};
+
+if (module === require.main) {
+  module.exports(require('@root/request')).then(function (all) {
+    all = require('../_webi/normalize.js')(all);
+    all.releases = all.releases.slice(0, 10);
+    //console.info(JSON.stringify(all));
+    console.info(JSON.stringify(all, null, 2));
+  });
+}

dashcore-utils/README.md

@@ -100,7 +100,8 @@ mkdir -p ~/.dashcore/wallets/
 mkdir -p /mnt/slc1_vol_100g/dashcore/_data
 mkdir -p /mnt/slc1_vol_100g/dashcore/_caches
 
-serviceman add --name 'dashd' --daemon -- \
+sudo env PATH="$PATH" serviceman add \
+        --system --user "$my_user" --path "$PATH" --name dashd --force -- \
     dashd \
         -usehd \
         -conf="$HOME/.dashcore/dash.conf" \

dashcore-utils/dash.example.conf

@@ -1,14 +1,7 @@
-# for exposing RPCs, building APIs
 txindex=1
 addressindex=1
 timestampindex=1
 spentindex=1
-# listen as server (explicit default)
-listen=1
-# because its already run as a service (systemd, openrc)
-daemon=0
-# for evonodes
-#server=1
 
 [main]
 rpcuser=RPCUSER_MAIN

dashcore-utils/dashd-hd-service-install

@@ -17,7 +17,7 @@ fn_usage() { (
 ); }
 
 fn_datadir_help() { (
-    my_vol="${1}"
+    my_vol="${1:-}"
     my_user="$(
         id -n -u
     )"
@@ -25,7 +25,6 @@ fn_datadir_help() { (
         id -n -g
     )"
 
-    my_mount="$(dirname "${my_vol}")"
     echo >&2 ""
     echo >&2 "ERROR"
     echo >&2 "        '${my_vol}' is not writable"
@@ -33,8 +32,8 @@ fn_datadir_help() { (
     echo >&2 "SOLUTION"
     echo >&2 "        1. Mount a large (50gb+) volume"
     echo >&2 ""
-    echo >&2 "              sudo mkdir -p ${my_mount}"
-    echo >&2 "              sudo mount /dev/sdx1 ${my_mount}"
+    echo >&2 "              sudo mkdir -p /mnt/EXAMPLE"
+    echo >&2 "              sudo mount /dev/sdx1 /mnt/EXAMPLE"
     echo >&2 ""
     echo >&2 "        2. Create a 'dashcore' inside of it"
     echo >&2 ""
@@ -84,8 +83,20 @@ fn_srv_install() { (
         my_name="dashd-${my_netname}"
     fi
 
+    my_system_args=""
+    my_kernel="$(
+        uname -s
+    )"
+    if test "Darwin" != "${my_kernel}"; then
+        my_user="$(
+            id -u -n
+        )"
+        my_system_args="--system --username ${my_user}"
+    fi
+
     # shellcheck disable=SC2016,SC1090
-    echo "serviceman add --name \"${my_name}\" --" \
+    echo 'sudo env PATH="$PATH"' \
+        "serviceman add ${my_system_args} --path \"\$PATH\" --name \"${my_name}\" --force --" \
         "dashd " \
         "${my_net_flag}" \
         -usehd \
@@ -95,16 +106,16 @@ fn_srv_install() { (
         "-datadir=\"${my_datadir}\"" \
         "-blocksdir=\"${my_blocksdir}\""
 
-    echo ""
-    echo "Installing latest 'serviceman'..."
-    echo ""
-    "$HOME/.local/bin/webi" serviceman > /dev/null
     if ! command -v serviceman > /dev/null; then
-        export PATH="$HOME/.local/bin:$PATH"
-    fi
-    serviceman --version
-    if ! command -v dashd > /dev/null; then
-        export PATH="$HOME/.local/opt/dashcore/bin:$PATH"
+        echo ""
+        echo "Installing 'serviceman'..."
+        echo ""
+        {
+            curl -fsSL "${WEBI_HOST}/serviceman" | sh
+        } > /dev/null
+
+        # shellcheck disable=SC1090
+        . ~/.config/envman/PATH.env || true
     fi
 
     mkdir -p "$HOME/.dashcore/wallets/"
@@ -119,7 +130,8 @@ fn_srv_install() { (
     cd "${my_vol}" || return 1
     # leave options unquoted so they're interpreted separately
     # shellcheck disable=SC2086
-    serviceman add --name "${my_name}" -- \
+    sudo env PATH="${PATH}" \
+        serviceman add ${my_system_args} --path "${PATH}" --name "${my_name}" --force -- \
         dashd \
         ${my_net_flag} \
         -usehd \

dashcore-utils/install.sh

@@ -5,38 +5,32 @@ set -u
 __install_dashcore_utils() {
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dash-qt-hd" \
-        "$HOME/.local/bin/dash-qt-hd" \
-        "dash-qt-hd"
+        "$HOME/.local/bin/dash-qt-hd"
     chmod a+x "$HOME/.local/bin/dash-qt-hd"
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dash-qt-testnet" \
-        "$HOME/.local/bin/dash-qt-testnet" \
-        "dash-qt-testnet"
+        "$HOME/.local/bin/dash-qt-testnet"
     chmod a+x "$HOME/.local/bin/dash-qt-testnet"
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dashd-hd" \
-        "$HOME/.local/bin/dashd-hd" \
-        "dashd-hd"
+        "$HOME/.local/bin/dashd-hd"
     chmod a+x "$HOME/.local/bin/dashd-hd"
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dashd-testnet" \
-        "$HOME/.local/bin/dashd-testnet" \
-        "dashd-testnet"
+        "$HOME/.local/bin/dashd-testnet"
     chmod a+x "$HOME/.local/bin/dashd-testnet"
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dashd-hd-service-install" \
-        "$HOME/.local/bin/dashd-hd-service-install" \
-        "dashd-hd-service-install"
+        "$HOME/.local/bin/dashd-hd-service-install"
     chmod a+x "$HOME/.local/bin/dashd-hd-service-install"
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dashd-testnet-service-install" \
-        "$HOME/.local/bin/dashd-testnet-service-install" \
-        "dashd-testnet-service-install"
+        "$HOME/.local/bin/dashd-testnet-service-install"
     chmod a+x "$HOME/.local/bin/dashd-testnet-service-install"
 
     if ! test -e "${HOME}/.dashcore"; then
@@ -50,8 +44,7 @@ __install_dashcore_utils() {
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dash.example.conf" \
-        "$HOME/.dashcore/dash.example.conf" \
-        "dash.example.conf"
+        "$HOME/.dashcore/dash.example.conf"
 
     if ! grep -q rpcuser ~/.dashcore/dash.conf; then
         cat ~/.dashcore/dash.example.conf >> ~/.dashcore/dash.conf

dashcore/README.md

@@ -147,8 +147,8 @@ dash-qt \
 CoinJoin aids in preventing some bad actors and malicious observers being able
 to easily reconstruct details about your transactions from the publicly
 available data by creating many excess transactions. \
-(be aware, however, that dedicated bad actors can use sophisticated software
-that will reveal much of the same information over time)
+(be aware, however, that dedicated bad actors can use sophisticated software that
+will reveal much of the same information over time)
 
 `dash-qt` does not enable CoinJoin mixing by default.
 

dashcore/install.ps1

@@ -20,13 +20,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading dashcore from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-if (!(Test-Path -Path "$pkg_src_dir")) {
+IF (!(Test-Path -Path "$pkg_src_dir")) {
     Write-Output "Installing dashcore"
 
     # TODO: create package-specific temp directory

dashcore/releases.conf

@@ -1 +0,0 @@
-github_releases = dashpay/dash

dashcore/releases.js

@@ -0,0 +1,31 @@
+'use strict';
+
+var github = require('../_common/github.js');
+var owner = 'dashpay';
+var repo = 'dash';
+
+module.exports = function (request) {
+  return github(request, owner, repo).then(function (all) {
+    all.releases.forEach(function (rel) {
+      if (rel.name.includes('osx64')) {
+        rel.os = 'macos';
+      }
+
+      if (rel.version.startsWith('v')) {
+        rel._version = rel.version.slice(1);
+      }
+    });
+
+    all._names = ['dashd', 'dashcore'];
+    return all;
+  });
+};
+
+if (module === require.main) {
+  module.exports(require('@root/request')).then(function (all) {
+    all = require('../_webi/normalize.js')(all);
+    // just select the first 5 for demonstration
+    all.releases = all.releases.slice(0, 5);
+    console.info(JSON.stringify(all, null, 2));
+  });
+}

dashd/README.md

@@ -24,7 +24,7 @@ install:
 ~/.config/envman/PATH.env
 ~/.dashcore/dash.conf
 ~/.dashcore/wallets/
-~/.local/bin/dashd-hd-service-install
+~/.local/bin/bin/dashd-hd-service-install
 ~/.local/opt/dashcore/
 /mnt/<BLK_VOL>/dashcore/
 ```
@@ -219,7 +219,14 @@ You can use [`serviceman`](../serviceman/):
 **Linux**
 
 ```sh
-serviceman add --name 'dashd' -- \
+sudo env PATH="$PATH" \
+    serviceman add \
+        --system \
+        --username "$(id -n -u)" \
+        --path "$PATH" \
+        --name dashd \
+        --force \
+        -- \
     dashd \
         -usehd \
         -conf="$HOME/.dashcore/dash.conf" \
@@ -232,7 +239,11 @@ serviceman add --name 'dashd' -- \
 **Mac**
 
 ```sh
-serviceman add --name 'dashd' -- \
+serviceman add \
+    --path "$PATH" \
+    --name dashd \
+    --force \
+    -- \
     dashd \
         -usehd \
         -conf="$HOME/.dashcore/dash.conf" \

dashd/install.ps1

@@ -20,13 +20,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading dashd from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-if (!(Test-Path -Path "$pkg_src_dir")) {
+IF (!(Test-Path -Path "$pkg_src_dir")) {
     Write-Output "Installing dashd"
 
     # TODO: create package-specific temp directory

dashd/releases.conf

@@ -1 +0,0 @@
-alias_of = dashcore

dashd/releases.js

@@ -0,0 +1,3 @@
+'use strict';
+
+module.exports = require('../dashcore/releases.js');

dashmsg/install.ps1

@@ -19,13 +19,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading dashmsg from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-if (!(Test-Path -Path "$pkg_src_cmd")) {
+IF (!(Test-Path -Path "$pkg_src_cmd")) {
     Write-Output "Installing dashmsg"
 
     # TODO: create package-specific temp directory