test: add API compatibility test against live webinstall.dev

Compares local cache-only output against golden data fetched from
the live API. Tests OS/arch/ext vocabulary, version format, and
filtered query results for bat, go, node, rg, jq, caddy.

PROD_NOTES.md

@@ -36,6 +36,16 @@ pre-classified fields and re-normalize from filenames to match production output
 For the installer endpoint (`builds-cacher.js` + `serve-installer.js`), the Go
 naming is used directly — the build-classifier handles its own mappings.
 
+### Known Intentional Differences from Production
+
+The Go cache filters releases more aggressively than the old Node normalize.js:
+- **Excluded**: `.deb`, `.rpm`, `.sha256`, `.sig`, `.pem`, `.sbom`, `.txt` files
+  (non-installable metadata/package-manager assets)
+- **OS `unknown`**: Some assets that normalize.js couldn't classify get `os: "unknown"`.
+  The Go cache either classifies them correctly or excludes them.
+- These are improvements — the filtered results better reflect what webi can
+  actually install.
+
 ### Known Pre-existing Issues
 
 - **`transform-releases.js` self-test**: The `if (require.main === module)`
@@ -45,9 +55,107 @@ naming is used directly — the build-classifier handles its own mappings.
 - **Go illumos/solaris warnings**: Go's illumos and solaris builds trigger
   "wrong os" warnings (expected `sunos`).
 
+## Public API Endpoints
+
+The HTTP routing is NOT in this repo — an external server calls into the Node
+modules. Here's the complete endpoint catalog:
+
+### 1. Bootstrap / Installer Scripts
+
+```
+GET /{package}@{tag}
+GET /{package}@{tag}.sh
+GET /{package}@{tag}.ps1
+```
+
+**Handler**: `serve-installer.js:serveInstaller(baseurl, ua, pkg, tag, ext, formats, libc)`
+
+**Flow**:
+1. Parse User-Agent → `build-classifier/host-targets.js:termsToTarget()` → `{os, arch, libc}`
+2. Resolve alias → `builds-cacher.js:getProjectType()` (symlink or README alias)
+3. Load cache → `builds-cacher.js:getPackages()` → reads `_cache/YYYY-MM/{pkg}.json`
+4. Classify → `builds-cacher.js:transformAndUpdate()` → triplets, versions, formats
+5. Match → `builds-cacher.js:findMatchingPackages()` → filter by OS/arch/libc/version
+6. Select → `builds-cacher.js:selectPackage()` → pick preferred format
+7. Render → `installers.js:renderBash()` or `renderPowerShell()` with template vars
+
+**UA format** (sent by webi bootstrap): `{arch}/unknown {OS}/{version} {libc}`
+- e.g. `aarch64/unknown Darwin/24.2.0 libc`
+- e.g. `x86_64/unknown Linux/5.15.0 musl`
+
+**Template variables injected**: `WEBI_VERSION`, `WEBI_PKG_URL`, `WEBI_PKG_FILE`,
+`WEBI_OS`, `WEBI_ARCH`, `WEBI_EXT`, `WEBI_CHANNEL`, `PKG_NAME`
+
+### 2. Release Metadata API (Legacy)
+
+```
+GET /api/releases/{package}.json
+GET /api/releases/{package}@{version}.json
+GET /api/releases/{package}.tab
+GET /api/releases/{package}@{version}.tab
+```
+
+**Handler**: `transform-releases.js:getReleases({pkg, ver, os, arch, libc, lts, channel, formats, limit})`
+
+**Flow**:
+1. Read cache → `_cache/YYYY-MM/{pkg}.json`
+2. Re-normalize → `normalize.js` (clears pre-classified fields, re-detects from filenames)
+3. Filter → `filterReleases()` by query params
+4. Sort → by version (descending), then format preference
+
+**Query params**: `os`, `arch`, `libc`, `lts`, `channel`, `formats`, `limit`
+
+**Response (JSON)**: `{ oses, arches, libcs, formats, releases: [{name, version, lts, channel, date, os, arch, ext, download, libc}] }`
+
+**Response (TSV)**: `version \t lts \t channel \t date \t os \t arch \t ext \t - \t download`
+
+**Key format details**:
+- OS: `macos` (not `darwin`), `linux`, `windows`
+- Arch: `arm64` (not `aarch64`), `amd64`, `armv6l`, `armv7l`, `x86`
+- Versions: no `v` prefix (`0.26.1` not `v0.26.1`)
+
+### 3. Curl-pipe Bootstrap
+
+```
+GET /{package}@{tag} (with curl/wget User-Agent)
+```
+
+**Handler**: `serve-installer.js:getPosixCurlPipeBootstrap({baseurl, pkg, ver})`
+or `getPwshCurlPipeBootstrap({baseurl, pkg, ver, exename})`
+
+Sets env vars `WEBI_PKG`, `WEBI_HOST`, `WEBI_CHECKSUM` in the bootstrap template.
+
+### 4. Package Metadata
+
+```
+GET /packages/{package}/README.md (or other assets)
+```
+
+**Handler**: `packages.js:get(name)` → reads README.md frontmatter via `frontmarker.js`
+
+**Response**: `{ title, tagline, description, bash, windows }`
+
+### 5. Debug
+
+```
+GET /api/debug
+```
+
+**Handler**: `ua-detect.js:request(req)` — returns detected OS/arch/libc from UA.
+
 ## Testing
 
-All paths verified locally:
+### Automated compatibility test
+
+```sh
+# Refresh golden data from live site
+node _webi/test-api-compat.js --refresh   # (not yet implemented)
+
+# Run comparison
+node _webi/test-api-compat.js
+```
+
+### Manual smoke tests
 
 ```sh
 # builds-cacher: load packages from cache
@@ -63,16 +171,6 @@ node -e "let I = require('./_webi/serve-installer.js'); I.helper({unameAgent:'aa
 node _webi/classify-one.js bat
 ```
 
-## Public API Endpoints (live at webinstall.dev)
-
-- `GET /api/releases/{pkg}.json` — Returns JSON with `oses`, `arches`, `libcs`,
-  `formats`, and `releases` array. Uses `macos` not `darwin`.
-- `GET /api/releases/{pkg}.tab` — Tab-separated release data
-- `GET /{pkg}@{tag}` — Returns installer script (bash or ps1 based on UA)
-
-The HTTP routing is NOT in this repo — an external server calls into the Node
-modules (`Releases.getReleases()` and `InstallerServer.serveInstaller()`).
-
 ## Project Type Detection
 
 `builds-cacher.js:getProjectTypeByEntry()` classifies packages:

_webi/test-api-compat.js

@@ -0,0 +1,208 @@
+'use strict';
+
+let Fs = require('node:fs/promises');
+let Path = require('node:path');
+
+let Releases = require('./transform-releases.js');
+
+let TESTDATA_DIR = Path.join(__dirname, 'testdata');
+
+// These mirror what the live API returns for /api/releases/{pkg}@stable.json?...
+let FILTERED_CASES = [
+  { pkg: 'bat', os: 'macos', arch: 'amd64' },
+  { pkg: 'bat', os: 'macos', arch: 'arm64' },
+  { pkg: 'bat', os: 'linux', arch: 'amd64' },
+  { pkg: 'bat', os: 'windows', arch: 'amd64' },
+  { pkg: 'go', os: 'macos', arch: 'amd64' },
+  { pkg: 'go', os: 'macos', arch: 'arm64' },
+  { pkg: 'go', os: 'linux', arch: 'amd64' },
+  { pkg: 'go', os: 'windows', arch: 'amd64' },
+  { pkg: 'rg', os: 'macos', arch: 'amd64' },
+  { pkg: 'rg', os: 'macos', arch: 'arm64' },
+  { pkg: 'rg', os: 'linux', arch: 'amd64' },
+  { pkg: 'rg', os: 'windows', arch: 'amd64' },
+  { pkg: 'caddy', os: 'macos', arch: 'amd64' },
+  { pkg: 'caddy', os: 'macos', arch: 'arm64' },
+  { pkg: 'caddy', os: 'linux', arch: 'amd64' },
+  { pkg: 'caddy', os: 'windows', arch: 'amd64' },
+];
+
+// Fields to compare between live and local
+let COMPARE_FIELDS = ['version', 'os', 'arch', 'ext', 'libc', 'channel', 'download'];
+
+async function main() {
+  let failures = 0;
+  let passes = 0;
+  let skips = 0;
+
+  // Test 1: Unfiltered release list — compare structure and field values
+  console.log('=== Test 1: Unfiltered /api/releases/{pkg}.json ===');
+  console.log('');
+  for (let pkg of ['bat', 'go', 'node', 'rg', 'jq', 'caddy']) {
+    let liveFile = `${TESTDATA_DIR}/live_${pkg}.json`;
+    let liveExists = await Fs.access(liveFile).then(
+      function () { return true; },
+      function () { return false; },
+    );
+    if (!liveExists) {
+      console.log(`  SKIP ${pkg}: no golden data`);
+      skips++;
+      continue;
+    }
+
+    let liveJson = await Fs.readFile(liveFile, 'utf8');
+    let liveReleases = JSON.parse(liveJson);
+
+    let localResult = await Releases.getReleases({
+      pkg: pkg,
+      ver: '',
+      os: '',
+      arch: '',
+      libc: '',
+      lts: false,
+      channel: '',
+      formats: [],
+      limit: 100,
+    });
+
+    let localReleases = localResult.releases;
+
+    // Compare OS vocabulary
+    let liveOses = [...new Set(liveReleases.map(function (r) { return r.os; }))].sort();
+    let localOses = [...new Set(localReleases.map(function (r) { return r.os; }))].sort();
+    let osMatch = JSON.stringify(liveOses) === JSON.stringify(localOses);
+    if (!osMatch) {
+      console.log(`  FAIL ${pkg} OS values: live=${JSON.stringify(liveOses)} local=${JSON.stringify(localOses)}`);
+      failures++;
+    } else {
+      console.log(`  PASS ${pkg} OS values: ${JSON.stringify(liveOses)}`);
+      passes++;
+    }
+
+    // Compare arch vocabulary
+    let liveArches = [...new Set(liveReleases.map(function (r) { return r.arch; }))].sort();
+    let localArches = [...new Set(localReleases.map(function (r) { return r.arch; }))].sort();
+    let archMatch = JSON.stringify(liveArches) === JSON.stringify(localArches);
+    if (!archMatch) {
+      console.log(`  FAIL ${pkg} arch values: live=${JSON.stringify(liveArches)} local=${JSON.stringify(localArches)}`);
+      failures++;
+    } else {
+      console.log(`  PASS ${pkg} arch values: ${JSON.stringify(liveArches)}`);
+      passes++;
+    }
+
+    // Compare latest version
+    let liveLatest = liveReleases[0]?.version;
+    let localLatest = localReleases[0]?.version;
+    if (liveLatest !== localLatest) {
+      // Version differences may be expected if cache is newer/older
+      console.log(`  WARN ${pkg} latest version: live=${liveLatest} local=${localLatest}`);
+    } else {
+      console.log(`  PASS ${pkg} latest version: ${liveLatest}`);
+      passes++;
+    }
+
+    // Compare ext vocabulary
+    let liveExts = [...new Set(liveReleases.map(function (r) { return r.ext; }))].sort();
+    let localExts = [...new Set(localReleases.map(function (r) { return r.ext; }))].sort();
+    let extMatch = JSON.stringify(liveExts) === JSON.stringify(localExts);
+    if (!extMatch) {
+      console.log(`  FAIL ${pkg} ext values: live=${JSON.stringify(liveExts)} local=${JSON.stringify(localExts)}`);
+      failures++;
+    } else {
+      console.log(`  PASS ${pkg} ext values: ${JSON.stringify(liveExts)}`);
+      passes++;
+    }
+
+    // Compare that version strings don't have 'v' prefix
+    let localHasVPrefix = localReleases.some(function (r) {
+      return r.version.startsWith('v');
+    });
+    if (localHasVPrefix) {
+      console.log(`  FAIL ${pkg} versions have 'v' prefix (should be stripped)`);
+      failures++;
+    } else {
+      console.log(`  PASS ${pkg} no 'v' prefix on versions`);
+      passes++;
+    }
+  }
+
+  // Test 2: Filtered queries — compare selected package for specific OS/arch
+  console.log('');
+  console.log('=== Test 2: Filtered /api/releases/{pkg}@stable.json?os=...&arch=... ===');
+  console.log('');
+  for (let tc of FILTERED_CASES) {
+    let fname = `live_${tc.pkg}_os_${tc.os}_arch_${tc.arch}.json`;
+    let liveFile = `${TESTDATA_DIR}/${fname}`;
+    let liveExists = await Fs.access(liveFile).then(
+      function () { return true; },
+      function () { return false; },
+    );
+    if (!liveExists) {
+      skips++;
+      continue;
+    }
+
+    let liveJson = await Fs.readFile(liveFile, 'utf8');
+    let liveReleases = JSON.parse(liveJson);
+    let liveFirst = liveReleases[0];
+    if (!liveFirst || liveFirst.channel === 'error') {
+      console.log(`  SKIP ${tc.pkg} ${tc.os}/${tc.arch}: live returned error/empty`);
+      skips++;
+      continue;
+    }
+
+    let localResult = await Releases.getReleases({
+      pkg: tc.pkg,
+      ver: '',
+      os: tc.os,
+      arch: tc.arch,
+      libc: '',
+      lts: false,
+      channel: 'stable',
+      formats: ['tar', 'zip', 'exe', 'xz'],
+      limit: 1,
+    });
+    let localFirst = localResult.releases[0];
+
+    if (!localFirst || localFirst.channel === 'error') {
+      console.log(`  FAIL ${tc.pkg} ${tc.os}/${tc.arch}: local returned error/empty, live had ${liveFirst.version}`);
+      failures++;
+      continue;
+    }
+
+    let diffs = [];
+    for (let field of COMPARE_FIELDS) {
+      let liveVal = String(liveFirst[field] || '');
+      let localVal = String(localFirst[field] || '');
+      if (liveVal !== localVal) {
+        // Version differences are OK if cache age differs
+        if (field === 'version' || field === 'download' || field === 'date') {
+          continue;
+        }
+        diffs.push(`${field}: live=${liveVal} local=${localVal}`);
+      }
+    }
+
+    if (diffs.length > 0) {
+      console.log(`  FAIL ${tc.pkg} ${tc.os}/${tc.arch}: ${diffs.join(', ')}`);
+      failures++;
+    } else {
+      let ver = localFirst.version;
+      let ext = localFirst.ext;
+      console.log(`  PASS ${tc.pkg} ${tc.os}/${tc.arch}: v${ver} .${ext}`);
+      passes++;
+    }
+  }
+
+  console.log('');
+  console.log(`=== Results: ${passes} passed, ${failures} failed, ${skips} skipped ===`);
+  if (failures > 0) {
+    process.exit(1);
+  }
+}
+
+main().catch(function (err) {
+  console.error(err.stack);
+  process.exit(1);
+});

_webi/testdata/live_bat_os_linux_arch_amd64.json

@@ -0,0 +1 @@
+[{"name":"bat-musl_0.26.1_musl-linux-amd64.deb","version":"0.26.1","lts":false,"channel":"stable","date":"2025-12-02","os":"linux","arch":"amd64","ext":"deb","download":"https://github.com/sharkdp/bat/releases/download/v0.26.1/bat-musl_0.26.1_musl-linux-amd64.deb","libc":"none"}]

_webi/testdata/live_bat_os_macos_arch_amd64.json

@@ -0,0 +1 @@
+[{"name":"bat-v0.26.1-x86_64-apple-darwin.tar.gz","version":"0.26.1","lts":false,"channel":"stable","date":"2025-12-02","os":"macos","arch":"amd64","ext":"tar.gz","download":"https://github.com/sharkdp/bat/releases/download/v0.26.1/bat-v0.26.1-x86_64-apple-darwin.tar.gz","libc":"none"}]

_webi/testdata/live_bat_os_macos_arch_arm64.json

@@ -0,0 +1 @@
+[{"name":"bat-v0.26.1-aarch64-apple-darwin.tar.gz","version":"0.26.1","lts":false,"channel":"stable","date":"2025-12-02","os":"macos","arch":"arm64","ext":"tar.gz","download":"https://github.com/sharkdp/bat/releases/download/v0.26.1/bat-v0.26.1-aarch64-apple-darwin.tar.gz","libc":"none"}]

_webi/testdata/live_bat_os_windows_arch_amd64.json

@@ -0,0 +1 @@
+[{"name":"bat-v0.26.1-x86_64-pc-windows-msvc.zip","version":"0.26.1","lts":false,"channel":"stable","date":"2025-12-02","os":"windows","arch":"amd64","ext":"zip","download":"https://github.com/sharkdp/bat/releases/download/v0.26.1/bat-v0.26.1-x86_64-pc-windows-msvc.zip","libc":"msvc"}]

_webi/testdata/live_caddy_os_linux_arch_amd64.json

@@ -0,0 +1 @@
+[{"name":"caddy_2.11.2_linux_amd64.deb","version":"2.11.2","lts":false,"channel":"stable","date":"2026-03-06","os":"linux","arch":"amd64","ext":"deb","download":"https://github.com/caddyserver/caddy/releases/download/v2.11.2/caddy_2.11.2_linux_amd64.deb","libc":"none"}]

_webi/testdata/live_caddy_os_macos_arch_amd64.json

@@ -0,0 +1 @@
+[{"name":"caddy_2.11.2_mac_amd64.pem","version":"2.11.2","lts":false,"channel":"stable","date":"2026-03-06","os":"macos","arch":"amd64","ext":"pem","download":"https://github.com/caddyserver/caddy/releases/download/v2.11.2/caddy_2.11.2_mac_amd64.pem","libc":"none"}]

_webi/testdata/live_caddy_os_macos_arch_arm64.json

@@ -0,0 +1 @@
+[{"name":"caddy_2.11.2_mac_arm64.pem","version":"2.11.2","lts":false,"channel":"stable","date":"2026-03-06","os":"macos","arch":"arm64","ext":"pem","download":"https://github.com/caddyserver/caddy/releases/download/v2.11.2/caddy_2.11.2_mac_arm64.pem","libc":"none"}]

_webi/testdata/live_caddy_os_windows_arch_amd64.json

@@ -0,0 +1 @@
+[{"name":"caddy_2.11.2_windows_amd64.pem","version":"2.11.2","lts":false,"channel":"stable","date":"2026-03-06","os":"windows","arch":"amd64","ext":"pem","download":"https://github.com/caddyserver/caddy/releases/download/v2.11.2/caddy_2.11.2_windows_amd64.pem","libc":"none"}]

_webi/testdata/live_go_os_linux_arch_amd64.json

@@ -0,0 +1 @@
+[{"version":"1.26.1","_version":"1.26.1","lts":true,"channel":"stable","date":"1970-01-01","os":"linux","arch":"amd64","ext":"tar.gz","hash":"-","download":"https://dl.google.com/go/go1.26.1.linux-amd64.tar.gz","name":"go1.26.1.linux-amd64.tar.gz","libc":"none"}]

_webi/testdata/live_go_os_macos_arch_amd64.json

@@ -0,0 +1 @@
+[{"version":"1.26.1","_version":"1.26.1","lts":true,"channel":"stable","date":"1970-01-01","os":"macos","arch":"amd64","ext":"tar.gz","hash":"-","download":"https://dl.google.com/go/go1.26.1.darwin-amd64.tar.gz","name":"go1.26.1.darwin-amd64.tar.gz","libc":"none"}]

_webi/testdata/live_go_os_macos_arch_arm64.json

@@ -0,0 +1 @@
+[{"version":"1.26.1","_version":"1.26.1","lts":true,"channel":"stable","date":"1970-01-01","os":"macos","arch":"arm64","ext":"tar.gz","hash":"-","download":"https://dl.google.com/go/go1.26.1.darwin-arm64.tar.gz","name":"go1.26.1.darwin-arm64.tar.gz","libc":"none"}]

_webi/testdata/live_go_os_windows_arch_amd64.json

@@ -0,0 +1 @@
+[{"version":"1.26.1","_version":"1.26.1","lts":true,"channel":"stable","date":"1970-01-01","os":"windows","arch":"amd64","ext":"zip","hash":"-","download":"https://dl.google.com/go/go1.26.1.windows-amd64.zip","name":"go1.26.1.windows-amd64.zip","libc":"none"}]

_webi/testdata/live_rg_os_linux_arch_amd64.json

@@ -0,0 +1 @@
+[{"name":"ripgrep-15.1.0-x86_64-unknown-linux-musl.tar.gz","version":"15.1.0","lts":false,"channel":"stable","date":"2025-10-22","os":"linux","arch":"amd64","ext":"tar.gz","download":"https://github.com/BurntSushi/ripgrep/releases/download/15.1.0/ripgrep-15.1.0-x86_64-unknown-linux-musl.tar.gz","libc":"none"}]

_webi/testdata/live_rg_os_macos_arch_amd64.json

@@ -0,0 +1 @@
+[{"name":"ripgrep-15.1.0-x86_64-apple-darwin.tar.gz","version":"15.1.0","lts":false,"channel":"stable","date":"2025-10-22","os":"macos","arch":"amd64","ext":"tar.gz","download":"https://github.com/BurntSushi/ripgrep/releases/download/15.1.0/ripgrep-15.1.0-x86_64-apple-darwin.tar.gz","libc":"none"}]

_webi/testdata/live_rg_os_macos_arch_arm64.json

@@ -0,0 +1 @@
+[{"name":"ripgrep-15.1.0-aarch64-apple-darwin.tar.gz","version":"15.1.0","lts":false,"channel":"stable","date":"2025-10-22","os":"macos","arch":"arm64","ext":"tar.gz","download":"https://github.com/BurntSushi/ripgrep/releases/download/15.1.0/ripgrep-15.1.0-aarch64-apple-darwin.tar.gz","libc":"none"}]

_webi/testdata/live_rg_os_windows_arch_amd64.json

@@ -0,0 +1 @@
+[{"name":"ripgrep-15.1.0-x86_64-pc-windows-gnu.zip","version":"15.1.0","lts":false,"channel":"stable","date":"2025-10-22","os":"windows","arch":"amd64","ext":"zip","download":"https://github.com/BurntSushi/ripgrep/releases/download/15.1.0/ripgrep-15.1.0-x86_64-pc-windows-gnu.zip","libc":"none"}]