From 73188d50e12a401f1479c3db61abf153e005f19e Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Thu, 7 May 2026 01:02:00 -0600 Subject: [PATCH] test: add cache-only validation and live-comparison suites New test programs that exercise the cache-only Node server path and compare its output against the legacy upstream-fetching path: - test-cache-api-ready.js: pre-flight check that the cache file matches what /api/releases would return for each name in _cache//. - test-cache-compat.js: parameterized cache vs. fresh-fetch diff suite, walks every package and OS/arch combo. - test-api-compat.js, test-installer-resolve.js, test-broad-resolve.js: installer-side parity checks across the resolver. - test-live-compare.js, test-live-cache-diff.js, test-live-installer-diff.js: TSV-output sweeps comparing two remote URLs across the cached package x OS/arch matrix; supports --concurrency and --packages filters. - test-fleet-diff.js: fleet-wide TSV diff with package/OS/arch filtering, used to validate beta and next. Also adds 36 golden snapshots under _webi/testdata/live_*.json covering 6 packages (bat, caddy, go, jq, node, rg) x 4 OS/arch combos plus the unfiltered baseline per package. --- .jshintignore | 3 +- _webi/test-api-compat.js | 208 +++++++++ _webi/test-broad-resolve.js | 77 ++++ _webi/test-cache-api-ready.js | 277 ++++++++++++ _webi/test-cache-compat.js | 687 ++++++++++++++++++++++++++++++ _webi/test-fleet-diff.js | 339 +++++++++++++++ _webi/test-installer-resolve.js | 444 +++++++++++++++++++ _webi/test-live-cache-diff.js | 450 +++++++++++++++++++ _webi/test-live-compare.js | 577 +++++++++++++++++++++++++ _webi/test-live-installer-diff.js | 202 +++++++++ 10 files changed, 3263 insertions(+), 1 deletion(-) create mode 100644 _webi/test-api-compat.js create mode 100644 _webi/test-broad-resolve.js create mode 100644 _webi/test-cache-api-ready.js create mode 100644 _webi/test-cache-compat.js create mode 100644 _webi/test-fleet-diff.js create mode 100644 _webi/test-installer-resolve.js create mode 100644 _webi/test-live-cache-diff.js create mode 100644 _webi/test-live-compare.js create mode 100644 _webi/test-live-installer-diff.js diff --git a/.jshintignore b/.jshintignore index f8a704c..940f0ef 100644 --- a/.jshintignore +++ b/.jshintignore @@ -1 +1,2 @@ -DELETEMEnode_modules/**/* +node_modules/**/* +_webi/test-*.js diff --git a/_webi/test-api-compat.js b/_webi/test-api-compat.js new file mode 100644 index 0000000..321d657 --- /dev/null +++ b/_webi/test-api-compat.js @@ -0,0 +1,208 @@ +'use strict'; + +let Fs = require('node:fs/promises'); +let Path = require('node:path'); + +let Releases = require('./transform-releases.js'); + +let TESTDATA_DIR = Path.join(__dirname, 'testdata'); + +// These mirror what the live API returns for /api/releases/{pkg}@stable.json?... +let FILTERED_CASES = [ + { pkg: 'bat', os: 'macos', arch: 'amd64' }, + { pkg: 'bat', os: 'macos', arch: 'arm64' }, + { pkg: 'bat', os: 'linux', arch: 'amd64' }, + { pkg: 'bat', os: 'windows', arch: 'amd64' }, + { pkg: 'go', os: 'macos', arch: 'amd64' }, + { pkg: 'go', os: 'macos', arch: 'arm64' }, + { pkg: 'go', os: 'linux', arch: 'amd64' }, + { pkg: 'go', os: 'windows', arch: 'amd64' }, + { pkg: 'rg', os: 'macos', arch: 'amd64' }, + { pkg: 'rg', os: 'macos', arch: 'arm64' }, + { pkg: 'rg', os: 'linux', arch: 'amd64' }, + { pkg: 'rg', os: 'windows', arch: 'amd64' }, + { pkg: 'caddy', os: 'macos', arch: 'amd64' }, + { pkg: 'caddy', os: 'macos', arch: 'arm64' }, + { pkg: 'caddy', os: 'linux', arch: 'amd64' }, + { pkg: 'caddy', os: 'windows', arch: 'amd64' }, +]; + +// Fields to compare between live and local +let COMPARE_FIELDS = ['version', 'os', 'arch', 'ext', 'libc', 'channel', 'download']; + +async function main() { + let failures = 0; + let passes = 0; + let skips = 0; + + // Test 1: Unfiltered release list — compare structure and field values + console.log('=== Test 1: Unfiltered /api/releases/{pkg}.json ==='); + console.log(''); + for (let pkg of ['bat', 'go', 'node', 'rg', 'jq', 'caddy']) { + let liveFile = `${TESTDATA_DIR}/live_${pkg}.json`; + let liveExists = await Fs.access(liveFile).then( + function () { return true; }, + function () { return false; }, + ); + if (!liveExists) { + console.log(` SKIP ${pkg}: no golden data`); + skips++; + continue; + } + + let liveJson = await Fs.readFile(liveFile, 'utf8'); + let liveReleases = JSON.parse(liveJson); + + let localResult = await Releases.getReleases({ + pkg: pkg, + ver: '', + os: '', + arch: '', + libc: '', + lts: false, + channel: '', + formats: [], + limit: 100, + }); + + let localReleases = localResult.releases; + + // Compare OS vocabulary + let liveOses = [...new Set(liveReleases.map(function (r) { return r.os; }))].sort(); + let localOses = [...new Set(localReleases.map(function (r) { return r.os; }))].sort(); + let osMatch = JSON.stringify(liveOses) === JSON.stringify(localOses); + if (!osMatch) { + console.log(` FAIL ${pkg} OS values: live=${JSON.stringify(liveOses)} local=${JSON.stringify(localOses)}`); + failures++; + } else { + console.log(` PASS ${pkg} OS values: ${JSON.stringify(liveOses)}`); + passes++; + } + + // Compare arch vocabulary + let liveArches = [...new Set(liveReleases.map(function (r) { return r.arch; }))].sort(); + let localArches = [...new Set(localReleases.map(function (r) { return r.arch; }))].sort(); + let archMatch = JSON.stringify(liveArches) === JSON.stringify(localArches); + if (!archMatch) { + console.log(` FAIL ${pkg} arch values: live=${JSON.stringify(liveArches)} local=${JSON.stringify(localArches)}`); + failures++; + } else { + console.log(` PASS ${pkg} arch values: ${JSON.stringify(liveArches)}`); + passes++; + } + + // Compare latest version + let liveLatest = liveReleases[0]?.version; + let localLatest = localReleases[0]?.version; + if (liveLatest !== localLatest) { + // Version differences may be expected if cache is newer/older + console.log(` WARN ${pkg} latest version: live=${liveLatest} local=${localLatest}`); + } else { + console.log(` PASS ${pkg} latest version: ${liveLatest}`); + passes++; + } + + // Compare ext vocabulary + let liveExts = [...new Set(liveReleases.map(function (r) { return r.ext; }))].sort(); + let localExts = [...new Set(localReleases.map(function (r) { return r.ext; }))].sort(); + let extMatch = JSON.stringify(liveExts) === JSON.stringify(localExts); + if (!extMatch) { + console.log(` FAIL ${pkg} ext values: live=${JSON.stringify(liveExts)} local=${JSON.stringify(localExts)}`); + failures++; + } else { + console.log(` PASS ${pkg} ext values: ${JSON.stringify(liveExts)}`); + passes++; + } + + // Compare that version strings don't have 'v' prefix + let localHasVPrefix = localReleases.some(function (r) { + return r.version.startsWith('v'); + }); + if (localHasVPrefix) { + console.log(` FAIL ${pkg} versions have 'v' prefix (should be stripped)`); + failures++; + } else { + console.log(` PASS ${pkg} no 'v' prefix on versions`); + passes++; + } + } + + // Test 2: Filtered queries — compare selected package for specific OS/arch + console.log(''); + console.log('=== Test 2: Filtered /api/releases/{pkg}@stable.json?os=...&arch=... ==='); + console.log(''); + for (let tc of FILTERED_CASES) { + let fname = `live_${tc.pkg}_os_${tc.os}_arch_${tc.arch}.json`; + let liveFile = `${TESTDATA_DIR}/${fname}`; + let liveExists = await Fs.access(liveFile).then( + function () { return true; }, + function () { return false; }, + ); + if (!liveExists) { + skips++; + continue; + } + + let liveJson = await Fs.readFile(liveFile, 'utf8'); + let liveReleases = JSON.parse(liveJson); + let liveFirst = liveReleases[0]; + if (!liveFirst || liveFirst.channel === 'error') { + console.log(` SKIP ${tc.pkg} ${tc.os}/${tc.arch}: live returned error/empty`); + skips++; + continue; + } + + let localResult = await Releases.getReleases({ + pkg: tc.pkg, + ver: '', + os: tc.os, + arch: tc.arch, + libc: '', + lts: false, + channel: 'stable', + formats: ['tar', 'zip', 'exe', 'xz'], + limit: 1, + }); + let localFirst = localResult.releases[0]; + + if (!localFirst || localFirst.channel === 'error') { + console.log(` FAIL ${tc.pkg} ${tc.os}/${tc.arch}: local returned error/empty, live had ${liveFirst.version}`); + failures++; + continue; + } + + let diffs = []; + for (let field of COMPARE_FIELDS) { + let liveVal = String(liveFirst[field] || ''); + let localVal = String(localFirst[field] || ''); + if (liveVal !== localVal) { + // Version differences are OK if cache age differs + if (field === 'version' || field === 'download' || field === 'date') { + continue; + } + diffs.push(`${field}: live=${liveVal} local=${localVal}`); + } + } + + if (diffs.length > 0) { + console.log(` FAIL ${tc.pkg} ${tc.os}/${tc.arch}: ${diffs.join(', ')}`); + failures++; + } else { + let ver = localFirst.version; + let ext = localFirst.ext; + console.log(` PASS ${tc.pkg} ${tc.os}/${tc.arch}: v${ver} .${ext}`); + passes++; + } + } + + console.log(''); + console.log(`=== Results: ${passes} passed, ${failures} failed, ${skips} skipped ===`); + if (failures > 0) { + process.exit(1); + } +} + +main().catch(function (err) { + console.error(err.stack); + process.exit(1); +}); diff --git a/_webi/test-broad-resolve.js b/_webi/test-broad-resolve.js new file mode 100644 index 0000000..8b31f57 --- /dev/null +++ b/_webi/test-broad-resolve.js @@ -0,0 +1,77 @@ +'use strict'; + +// Broad sweep: test that all cached packages resolve on macOS arm64 +// and Linux amd64. Catches any package that completely fails to resolve. +// +// Usage: node _webi/test-broad-resolve.js + +var Path = require('node:path'); +var InstallerServer = require('./serve-installer.js'); +var Builds = require('./builds.js'); +var BuildsCacher = require('./builds-cacher.js'); + +var UA_CASES = [ + { label: 'macOS arm64', ua: 'aarch64/unknown Darwin/24.2.0 libc' }, + { label: 'Linux amd64', ua: 'x86_64/unknown Linux/5.15.0 libc' }, +]; + +async function main() { + console.log('Initializing build cache...'); + await Builds.init(); + console.log(''); + + var bc = BuildsCacher.create({ + caches: Path.join(__dirname, '../_cache'), + installers: Path.join(__dirname, '..'), + }); + var dirs = await bc.getProjectsByType(); + var pkgs = Object.keys(dirs.valid).sort(); + console.log('Testing ' + pkgs.length + ' packages...'); + console.log(''); + + var pass = 0; + var fail = 0; + var failures = []; + + for (var i = 0; i < pkgs.length; i++) { + var pkg = pkgs[i]; + for (var j = 0; j < UA_CASES.length; j++) { + var tc = UA_CASES[j]; + try { + var r = await InstallerServer.helper({ + unameAgent: tc.ua, + projectName: pkg, + tag: 'stable', + formats: ['tar', 'exe', 'zip', 'xz', 'dmg'], + libc: '', + }); + var p = r[0]; + if (p.channel === 'error' || p.ext === 'err') { + failures.push(pkg + ' ' + tc.label + ': error (v' + p.version + ')'); + fail++; + } else { + pass++; + } + } catch (e) { + failures.push(pkg + ' ' + tc.label + ': ' + e.message.substring(0, 60)); + fail++; + } + } + } + + if (failures.length > 0) { + console.log('Failures:'); + for (var k = 0; k < failures.length; k++) { + console.log(' FAIL ' + failures[k]); + } + console.log(''); + } + + var total = pkgs.length * UA_CASES.length; + console.log('=== ' + pass + '/' + total + ' passed (' + fail + ' failed) ==='); +} + +main().catch(function (err) { + console.error(err.stack); + process.exit(1); +}); diff --git a/_webi/test-cache-api-ready.js b/_webi/test-cache-api-ready.js new file mode 100644 index 0000000..28a05a0 --- /dev/null +++ b/_webi/test-cache-api-ready.js @@ -0,0 +1,277 @@ +'use strict'; + +// Tests that _cache JSON files are ready for direct use by the API endpoint +// (transform-releases.js) WITHOUT needing normalize.js to fix up fields. +// +// These tests drive GOER to add missing features to ExportLegacy so that +// normalize.js can be eliminated from the API path entirely. +// +// Usage: node _webi/test-cache-api-ready.js + +var Fs = require('node:fs'); +var Os = require('node:os'); +var Path = require('node:path'); + +var CACHE_DIR = Path.join(Os.homedir(), '.cache/webi/legacy'); + +// Packages to spot-check (mix of github_releases, custom releases.js, gittag) +var CHECK_PKGS = [ + 'bat', + 'caddy', + 'go', + 'hugo', + 'jq', + 'node', + 'rg', + 'terraform', + 'zig', +]; + +function loadReleases(dir, pkg) { + var file = Path.join(dir, pkg + '.json'); + if (!Fs.existsSync(file)) { + return null; + } + try { + return JSON.parse(Fs.readFileSync(file, 'utf8')); + } catch (e) { + return null; + } +} + +async function main() { + var passes = 0; + var failures = 0; + + if (!Fs.existsSync(CACHE_DIR)) { + console.error('No cache directory at ' + CACHE_DIR); + process.exit(1); + } + console.log('Testing ' + CACHE_DIR); + console.log(''); + + // ================================================================ + // Test 1: Version has no 'v' prefix + // ================================================================ + console.log('=== Test 1: Version v-prefix stripped ==='); + console.log(''); + + for (var vi = 0; vi < CHECK_PKGS.length; vi++) { + var vpkg = CHECK_PKGS[vi]; + var vdata = loadReleases(CACHE_DIR, vpkg); + if (!vdata) { + console.log(' SKIP ' + vpkg + ': no data'); + continue; + } + + var vPrefixed = 0; + for (var vri = 0; vri < vdata.releases.length; vri++) { + var ver = vdata.releases[vri].version || ''; + if (ver.startsWith('v')) { + vPrefixed++; + } + } + + if (vPrefixed === 0) { + console.log(' PASS ' + vpkg + ': no v-prefixed versions'); + passes++; + } else { + console.log(' FAIL ' + vpkg + ': ' + vPrefixed + '/' + vdata.releases.length + ' versions have v prefix'); + failures++; + } + } + + // ================================================================ + // Test 2: libc is never empty string (should be "none", "gnu", "musl", "msvc") + // ================================================================ + console.log(''); + console.log('=== Test 2: libc never empty ==='); + console.log(''); + + for (var li = 0; li < CHECK_PKGS.length; li++) { + var lpkg = CHECK_PKGS[li]; + var ldata = loadReleases(CACHE_DIR, lpkg); + if (!ldata) { + console.log(' SKIP ' + lpkg + ': no data'); + continue; + } + + var emptyLibc = 0; + for (var lri = 0; lri < ldata.releases.length; lri++) { + if (ldata.releases[lri].libc === '') { + emptyLibc++; + } + } + + if (emptyLibc === 0) { + console.log(' PASS ' + lpkg + ': all entries have libc set'); + passes++; + } else { + console.log(' FAIL ' + lpkg + ': ' + emptyLibc + '/' + ldata.releases.length + ' entries have empty libc (should be "none")'); + failures++; + } + } + + // ================================================================ + // Test 3: ext has no leading dot + // ================================================================ + console.log(''); + console.log('=== Test 3: ext has no leading dot ==='); + console.log(''); + + for (var ei = 0; ei < CHECK_PKGS.length; ei++) { + var epkg = CHECK_PKGS[ei]; + var edata = loadReleases(CACHE_DIR, epkg); + if (!edata) { + console.log(' SKIP ' + epkg + ': no data'); + continue; + } + + var dotExt = 0; + for (var eri = 0; eri < edata.releases.length; eri++) { + var ext = edata.releases[eri].ext || ''; + if (ext.startsWith('.')) { + dotExt++; + } + } + + if (dotExt === 0) { + console.log(' PASS ' + epkg + ': no leading dots on ext'); + passes++; + } else { + console.log(' FAIL ' + epkg + ': ' + dotExt + '/' + edata.releases.length + ' entries have leading dot on ext'); + failures++; + } + } + + // ================================================================ + // Test 4: bare binaries have ext "exe" (not empty) + // ================================================================ + console.log(''); + console.log('=== Test 4: bare binaries have ext "exe" ==='); + console.log(''); + + var barePkgs = ['jq', 'bat', 'rg', 'caddy']; + for (var bi = 0; bi < barePkgs.length; bi++) { + var bpkg = barePkgs[bi]; + var bdata = loadReleases(CACHE_DIR, bpkg); + if (!bdata) { + console.log(' SKIP ' + bpkg + ': no data'); + continue; + } + + var emptyExt = 0; + for (var bri = 0; bri < bdata.releases.length; bri++) { + var brel = bdata.releases[bri]; + if (brel.ext === '' && brel.name && !brel.name.includes('.')) { + emptyExt++; + } + } + + if (emptyExt === 0) { + console.log(' PASS ' + bpkg + ': no bare binaries with empty ext'); + passes++; + } else { + console.log(' FAIL ' + bpkg + ': ' + emptyExt + ' bare binaries have empty ext (should be "exe")'); + failures++; + } + } + + // ================================================================ + // Test 5: Top-level summary arrays present + // ================================================================ + console.log(''); + console.log('=== Test 5: Summary arrays (oses, arches, libcs, formats) ==='); + console.log(''); + + for (var si = 0; si < CHECK_PKGS.length; si++) { + var spkg = CHECK_PKGS[si]; + var sdata = loadReleases(CACHE_DIR, spkg); + if (!sdata) { + console.log(' SKIP ' + spkg + ': no data'); + continue; + } + + var missing = []; + if (!Array.isArray(sdata.oses)) { missing.push('oses'); } + if (!Array.isArray(sdata.arches)) { missing.push('arches'); } + if (!Array.isArray(sdata.libcs)) { missing.push('libcs'); } + if (!Array.isArray(sdata.formats)) { missing.push('formats'); } + + if (missing.length === 0) { + // Verify they contain the right values + var hasMacos = sdata.oses.includes('macos') || sdata.oses.includes('darwin'); + var hasLinux = sdata.oses.includes('linux'); + var ok = true; + if (sdata.releases.some(function (r) { return r.os === 'macos' || r.os === 'darwin'; }) && !hasMacos) { + console.log(' FAIL ' + spkg + ': oses array missing macos/darwin'); + failures++; + ok = false; + } + if (sdata.releases.some(function (r) { return r.os === 'linux'; }) && !hasLinux) { + console.log(' FAIL ' + spkg + ': oses array missing linux'); + failures++; + ok = false; + } + if (ok) { + console.log(' PASS ' + spkg + ': has oses, arches, libcs, formats'); + passes++; + } + } else { + console.log(' FAIL ' + spkg + ': missing top-level arrays: ' + missing.join(', ')); + failures++; + } + } + + // ================================================================ + // Test 6: Version sort order — stable before beta, newest first + // ================================================================ + console.log(''); + console.log('=== Test 6: Version sort order ==='); + console.log(''); + + var sortPkgs = ['go', 'node', 'terraform']; + for (var oi = 0; oi < sortPkgs.length; oi++) { + var opkg = sortPkgs[oi]; + var odata = loadReleases(CACHE_DIR, opkg); + if (!odata) { + console.log(' SKIP ' + opkg + ': no data'); + continue; + } + + // Find first stable release + var firstStable = odata.releases.find(function (r) { return r.channel === 'stable'; }); + // Find first beta release + var firstBeta = odata.releases.find(function (r) { return r.channel === 'beta'; }); + + if (!firstStable) { + console.log(' SKIP ' + opkg + ': no stable release'); + continue; + } + + // The first entry overall should be a stable release (newest stable > any beta) + // unless the beta is a newer version number + var firstEntry = odata.releases[0]; + if (firstEntry.channel === 'stable') { + console.log(' PASS ' + opkg + ': first entry is stable (' + firstEntry.version + ')'); + passes++; + } else { + console.log(' FAIL ' + opkg + ': first entry is ' + firstEntry.channel + ' (' + firstEntry.version + '), expected stable (' + firstStable.version + ')'); + failures++; + } + } + + // ================================================================ + // Summary + // ================================================================ + console.log(''); + console.log('=== Results: ' + passes + ' passed, ' + failures + ' failed ==='); + if (failures > 0) { + process.exit(1); + } +} + +main().catch(function (err) { + console.error(err.stack); + process.exit(1); +}); diff --git a/_webi/test-cache-compat.js b/_webi/test-cache-compat.js new file mode 100644 index 0000000..45def04 --- /dev/null +++ b/_webi/test-cache-compat.js @@ -0,0 +1,687 @@ +'use strict'; + +// Cache compatibility tests: verify that Go-generated cache files work +// correctly with the Node build-classifier and installer resolution pipeline. +// +// Tests cover: +// 1. Cache completeness — all packages have releases, required fields present +// 2. Format selection — correct archive format per platform +// 3. Edge-case platforms — FreeBSD, ARM variants, musl/Alpine +// 4. Script generation — installer scripts render without error +// 5. API compat — transform-releases output matches expected vocabulary +// 6. Version format — no 'v' prefix, valid semver-ish +// 7. Channel detection — stable vs beta correctly identified +// +// Usage: node _webi/test-cache-compat.js + +var Fs = require('node:fs'); +var Path = require('node:path'); + +var InstallerServer = require('./serve-installer.js'); +var Builds = require('./builds.js'); +var Releases = require('./transform-releases.js'); + +var CACHE_DIR = Path.join(require('node:os').homedir(), '.cache/webi/legacy'); + +// ==================================================================== +// Test 1: Cache completeness — every package has releases with valid fields +// ==================================================================== + +// Packages that are expected to have binary releases (not gittag-only) +var BINARY_PKGS = [ + 'bat', + 'caddy', + 'cmake', + 'delta', + 'deno', + 'fd', + 'fzf', + 'gh', + 'go', + 'goreleaser', + 'hugo', + 'jq', + 'k9s', + 'node', + 'ollama', + 'rg', + 'shellcheck', + 'shfmt', + 'syncthing', + 'terraform', + 'xz', + 'yq', + 'zig', + 'zoxide', +]; + +// Packages that are gittag/source-only — must have releases but os/arch may be special +var GITTAG_PKGS = [ + 'aliasman', + 'serviceman', + 'vim-airline', + 'vim-go', + 'vim-sensible', +]; + +// ==================================================================== +// Test 2: Format selection — correct ext per platform +// ==================================================================== + +// For these packages, verify the resolved format is correct for each platform. +// Linux/macOS should get .tar.gz or .tar.xz (not .zip except for specific packages). +// Windows should get .zip or .exe (not .tar.gz). +var FORMAT_CASES = [ + // Go projects — should be .tar.gz on Linux/macOS, .zip on Windows + { + label: 'go Linux amd64 format', + pkg: 'go', + ua: 'x86_64/unknown Linux/5.15.0 libc', + expectExt: 'tar.gz', + }, + { + label: 'go Windows amd64 format', + pkg: 'go', + ua: 'x86_64/unknown Windows/10.0.19041 msvc', + expectExt: 'zip', + }, + { + label: 'go macOS arm64 format', + pkg: 'go', + ua: 'aarch64/unknown Darwin/24.2.0 libc', + expectExt: 'tar.gz', + }, + // Rust projects — should be .tar.gz on Linux/macOS, .zip on Windows + { + label: 'bat Linux amd64 format', + pkg: 'bat', + ua: 'x86_64/unknown Linux/5.15.0 libc', + expectExt: 'tar.gz', + }, + { + label: 'bat Windows amd64 format', + pkg: 'bat', + ua: 'x86_64/unknown Windows/10.0.19041 msvc', + expectExt: 'zip', + }, + { + label: 'rg Linux amd64 format', + pkg: 'rg', + ua: 'x86_64/unknown Linux/5.15.0 libc', + expectExt: 'tar.gz', + }, + { + label: 'rg Windows amd64 format', + pkg: 'rg', + ua: 'x86_64/unknown Windows/10.0.19041 msvc', + expectExt: 'zip', + }, + // Node — uses .tar.xz on Linux/macOS + { + label: 'node Linux amd64 format', + pkg: 'node', + ua: 'x86_64/unknown Linux/5.15.0 libc', + expectExt: 'tar.xz', + }, + { + label: 'node macOS arm64 format', + pkg: 'node', + ua: 'aarch64/unknown Darwin/24.2.0 libc', + expectExt: 'tar.xz', + }, + // delta — Rust, .tar.gz on Linux/macOS + { + label: 'delta Linux amd64 format', + pkg: 'delta', + ua: 'x86_64/unknown Linux/5.15.0 libc', + expectExt: 'tar.gz', + }, + // shfmt — Go, bare exe on Linux, .exe on Windows + { + label: 'shfmt Linux amd64 format', + pkg: 'shfmt', + ua: 'x86_64/unknown Linux/5.15.0 libc', + expectExt: 'exe', + }, +]; + +// ==================================================================== +// Test 3: Edge-case platforms +// ==================================================================== + +var EDGE_CASES = [ + // Linux ARM variants + { + label: 'go Linux aarch64', + pkg: 'go', + ua: 'aarch64/unknown Linux/5.15.0 libc', + expectOs: 'linux', + expectNotError: true, + }, + { + label: 'node Linux aarch64', + pkg: 'node', + ua: 'aarch64/unknown Linux/5.15.0 libc', + expectOs: 'linux', + expectNotError: true, + }, + // Alpine/musl — bat should resolve to musl build + { + label: 'bat Linux musl amd64', + pkg: 'bat', + ua: 'x86_64/unknown Linux/5.15.0 musl', + expectOs: 'linux', + expectNotError: true, + }, + { + label: 'rg Linux musl amd64', + pkg: 'rg', + ua: 'x86_64/unknown Linux/5.15.0 musl', + expectOs: 'linux', + expectNotError: true, + }, + // node musl — separate musl build + { + label: 'node Linux musl amd64', + pkg: 'node', + ua: 'x86_64/unknown Linux/5.15.0 musl', + expectOs: 'linux', + expectNotError: true, + }, + // FreeBSD — packages that have freebsd builds + { + label: 'syncthing FreeBSD amd64', + pkg: 'syncthing', + ua: 'x86_64/unknown FreeBSD/14.0 libc', + expectOs: 'freebsd', + expectNotError: true, + }, + { + label: 'caddy FreeBSD amd64', + pkg: 'caddy', + ua: 'x86_64/unknown FreeBSD/14.0 libc', + expectOs: 'freebsd', + expectNotError: true, + }, + // Windows aarch64 — should fallback to amd64 for most packages + { + label: 'go Windows aarch64', + pkg: 'go', + ua: 'aarch64/unknown Windows/10.0.22000 msvc', + expectOs: 'windows', + expectNotError: true, + }, +]; + +// ==================================================================== +// Test 4: Script generation smoke tests +// ==================================================================== + +var SCRIPT_CASES = [ + { pkg: 'bat', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'bat Linux' }, + { pkg: 'go', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'go macOS' }, + { pkg: 'node', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'node Linux' }, + { pkg: 'rg', ua: 'x86_64/unknown Windows/10.0.19041 msvc', label: 'rg Windows' }, + { pkg: 'jq', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'jq macOS' }, + { pkg: 'caddy', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'caddy Linux' }, + { pkg: 'shellcheck', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'shellcheck Linux' }, + { pkg: 'shfmt', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'shfmt macOS' }, + { pkg: 'hugo', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'hugo Linux' }, + { pkg: 'delta', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'delta Linux' }, + { pkg: 'fd', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'fd Linux' }, + { pkg: 'fzf', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'fzf Linux' }, + { pkg: 'zoxide', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'zoxide Linux' }, + { pkg: 'k9s', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'k9s Linux' }, + { pkg: 'yq', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'yq Linux' }, +]; + +// ==================================================================== +// Test 5: API compat — transform-releases vocabulary +// ==================================================================== + +// The /api/releases/ endpoint uses normalize.js which has its own OS/arch names. +// Verify that the Go cache produces correct values after normalization. +var API_VOCAB_CASES = [ + { + label: 'bat has macos+linux+windows', + pkg: 'bat', + expectOses: ['linux', 'macos', 'windows'], + }, + { + label: 'go has macos+linux+windows', + pkg: 'go', + expectOses: ['linux', 'macos', 'windows'], + }, + { + label: 'node has linux+macos+windows', + pkg: 'node', + expectOses: ['linux', 'macos', 'windows'], + }, + { + label: 'syncthing has freebsd+linux+macos+windows', + pkg: 'syncthing', + expectOses: ['freebsd', 'linux', 'macos', 'windows'], + }, + { + label: 'bat has amd64+arm64 arches', + pkg: 'bat', + expectArches: ['amd64', 'arm64'], + }, + { + label: 'go has amd64+arm64 arches', + pkg: 'go', + expectArches: ['amd64', 'arm64'], + }, +]; + +// ==================================================================== +// Test 6: Version format validation +// ==================================================================== + +var VERSION_CHECKS = [ + 'bat', + 'go', + 'node', + 'rg', + 'caddy', + 'hugo', + 'terraform', + 'jq', + 'cmake', + 'zig', +]; + +// ==================================================================== +// Test 7: Channel detection +// ==================================================================== + +// Packages that should have both stable and beta releases +var CHANNEL_CASES = [ + { + label: 'go has stable releases', + pkg: 'go', + channel: 'stable', + expectMinCount: 10, + }, + { + label: 'node has stable releases', + pkg: 'node', + channel: 'stable', + expectMinCount: 10, + }, + { + label: 'cmake has stable releases', + pkg: 'cmake', + channel: 'stable', + expectMinCount: 5, + }, +]; + +// ==================================================================== +// Runner +// ==================================================================== + +async function main() { + var passes = 0; + var failures = 0; + var knowns = 0; + var skips = 0; + + console.log('Initializing build cache...'); + await Builds.init(); + + if (!Fs.existsSync(CACHE_DIR)) { + console.error('No cache directory at ' + CACHE_DIR); + process.exit(1); + } + var cachePath = CACHE_DIR; + console.log('Using cache: ' + cachePath); + console.log(''); + + // ================================================================ + // Test 1: Cache completeness + // ================================================================ + console.log('=== Test 1: Cache Completeness ==='); + console.log(''); + + for (var bi = 0; bi < BINARY_PKGS.length; bi++) { + var bpkg = BINARY_PKGS[bi]; + var bfile = Path.join(cachePath, bpkg + '.json'); + if (!Fs.existsSync(bfile)) { + console.log(' FAIL ' + bpkg + ': cache file missing'); + failures++; + continue; + } + var bdata = JSON.parse(Fs.readFileSync(bfile, 'utf8')); + if (!bdata.releases || bdata.releases.length === 0) { + console.log(' FAIL ' + bpkg + ': 0 releases'); + failures++; + continue; + } + // Check that at least one release has a download URL + var hasDownload = bdata.releases.some(function (r) { + return r.download && r.download.startsWith('http'); + }); + if (!hasDownload) { + console.log(' FAIL ' + bpkg + ': no releases with download URLs'); + failures++; + continue; + } + console.log(' PASS ' + bpkg + ': ' + bdata.releases.length + ' releases'); + passes++; + } + + for (var gi = 0; gi < GITTAG_PKGS.length; gi++) { + var gpkg = GITTAG_PKGS[gi]; + var gfile = Path.join(cachePath, gpkg + '.json'); + if (!Fs.existsSync(gfile)) { + console.log(' FAIL ' + gpkg + ' (gittag): cache file missing'); + failures++; + continue; + } + var gdata = JSON.parse(Fs.readFileSync(gfile, 'utf8')); + if (!gdata.releases || gdata.releases.length === 0) { + console.log(' FAIL ' + gpkg + ' (gittag): 0 releases'); + failures++; + continue; + } + console.log(' PASS ' + gpkg + ' (gittag): ' + gdata.releases.length + ' releases'); + passes++; + } + + // ================================================================ + // Test 2: Format selection + // ================================================================ + console.log(''); + console.log('=== Test 2: Format Selection ==='); + console.log(''); + + for (var fi = 0; fi < FORMAT_CASES.length; fi++) { + var ftc = FORMAT_CASES[fi]; + try { + var fr = await InstallerServer.helper({ + unameAgent: ftc.ua, + projectName: ftc.pkg, + tag: 'stable', + formats: ['tar', 'exe', 'zip', 'xz', 'dmg'], + libc: '', + }); + var fpkg = fr[0]; + if (fpkg.channel === 'error') { + console.log(' FAIL ' + ftc.label + ': resolved to error'); + failures++; + continue; + } + if (fpkg.ext !== ftc.expectExt) { + console.log(' FAIL ' + ftc.label + ': got .' + fpkg.ext + ' want .' + ftc.expectExt); + failures++; + } else { + console.log(' PASS ' + ftc.label + ': .' + fpkg.ext); + passes++; + } + } catch (e) { + console.log(' ERROR ' + ftc.label + ': ' + e.message); + failures++; + } + } + + // ================================================================ + // Test 3: Edge-case platforms + // ================================================================ + console.log(''); + console.log('=== Test 3: Edge-Case Platforms ==='); + console.log(''); + + for (var ei = 0; ei < EDGE_CASES.length; ei++) { + var etc = EDGE_CASES[ei]; + try { + var er = await InstallerServer.helper({ + unameAgent: etc.ua, + projectName: etc.pkg, + tag: 'stable', + formats: ['tar', 'exe', 'zip', 'xz', 'dmg'], + libc: '', + }); + var epkg = er[0]; + if (etc.expectNotError && epkg.channel === 'error') { + console.log(' FAIL ' + etc.label + ': resolved to error (v' + epkg.version + ')'); + failures++; + continue; + } + if (etc.expectOs && epkg.os !== etc.expectOs) { + console.log(' FAIL ' + etc.label + ': os=' + epkg.os + ' want=' + etc.expectOs); + failures++; + continue; + } + console.log(' PASS ' + etc.label + ': v' + epkg.version + ' .' + epkg.ext); + passes++; + } catch (e) { + if (etc.known) { + console.log(' KNOWN ' + etc.label + ': ' + e.message); + knowns++; + } else { + console.log(' FAIL ' + etc.label + ': ' + e.message); + failures++; + } + } + } + + // ================================================================ + // Test 4: Script generation smoke tests + // ================================================================ + console.log(''); + console.log('=== Test 4: Script Generation ==='); + console.log(''); + + for (var si = 0; si < SCRIPT_CASES.length; si++) { + var stc = SCRIPT_CASES[si]; + try { + var script = await InstallerServer.serveInstaller( + 'https://webi.sh', + stc.ua, + stc.pkg, + 'stable', + 'sh', + ['tar', 'exe', 'zip', 'xz', 'dmg'], + '', + ); + + // Script must contain WEBI_PKG_URL and WEBI_VERSION + var hasUrl = /WEBI_PKG_URL='[^']+'/m.test(script); + var hasVersion = /WEBI_VERSION='[^']+'/m.test(script); + var hasExt = /WEBI_EXT='[^']+'/m.test(script); + + if (!hasUrl) { + console.log(' FAIL ' + stc.label + ': missing WEBI_PKG_URL'); + failures++; + } else if (!hasVersion) { + console.log(' FAIL ' + stc.label + ': missing WEBI_VERSION'); + failures++; + } else if (!hasExt) { + console.log(' FAIL ' + stc.label + ': missing WEBI_EXT'); + failures++; + } else { + var vMatch = script.match(/WEBI_VERSION='([^']+)'/); + var extMatch = script.match(/WEBI_EXT='([^']+)'/); + console.log(' PASS ' + stc.label + ': v' + vMatch[1] + ' .' + extMatch[1]); + passes++; + } + } catch (e) { + console.log(' FAIL ' + stc.label + ': ' + e.message.substring(0, 80)); + failures++; + } + } + + // ================================================================ + // Test 5: API compat — transform-releases vocabulary + // ================================================================ + console.log(''); + console.log('=== Test 5: API Vocabulary (transform-releases) ==='); + console.log(''); + + for (var ai = 0; ai < API_VOCAB_CASES.length; ai++) { + var atc = API_VOCAB_CASES[ai]; + try { + var ares = await Releases.getReleases({ + pkg: atc.pkg, + ver: '', + os: '', + arch: '', + libc: '', + lts: false, + channel: '', + formats: [], + limit: 100, + }); + var arels = ares.releases; + + if (atc.expectOses) { + var actualOses = []; + for (var oi = 0; oi < arels.length; oi++) { + if (actualOses.indexOf(arels[oi].os) === -1) { + actualOses.push(arels[oi].os); + } + } + actualOses.sort(); + var missingOses = []; + for (var mi = 0; mi < atc.expectOses.length; mi++) { + if (actualOses.indexOf(atc.expectOses[mi]) === -1) { + missingOses.push(atc.expectOses[mi]); + } + } + if (missingOses.length > 0) { + console.log(' FAIL ' + atc.label + ': missing ' + JSON.stringify(missingOses) + ' (has ' + JSON.stringify(actualOses) + ')'); + failures++; + } else { + console.log(' PASS ' + atc.label + ': ' + JSON.stringify(atc.expectOses)); + passes++; + } + } + + if (atc.expectArches) { + var actualArches = []; + for (var ari = 0; ari < arels.length; ari++) { + if (arels[ari].arch && actualArches.indexOf(arels[ari].arch) === -1) { + actualArches.push(arels[ari].arch); + } + } + actualArches.sort(); + var missingArches = []; + for (var mai = 0; mai < atc.expectArches.length; mai++) { + if (actualArches.indexOf(atc.expectArches[mai]) === -1) { + missingArches.push(atc.expectArches[mai]); + } + } + if (missingArches.length > 0) { + console.log(' FAIL ' + atc.label + ': missing ' + JSON.stringify(missingArches) + ' (has ' + JSON.stringify(actualArches) + ')'); + failures++; + } else { + console.log(' PASS ' + atc.label); + passes++; + } + } + } catch (e) { + console.log(' FAIL ' + atc.label + ': ' + e.message); + failures++; + } + } + + // ================================================================ + // Test 6: Version format + // ================================================================ + console.log(''); + console.log('=== Test 6: Version Format ==='); + console.log(''); + + // Version format check: the Go cache may have 'v' prefixes (that's OK — + // normalize.js and serve-installer.js strip them). What matters is that + // after normalization via transform-releases, versions have no 'v' prefix. + for (var vi = 0; vi < VERSION_CHECKS.length; vi++) { + var vpkg = VERSION_CHECKS[vi]; + try { + var vres = await Releases.getReleases({ + pkg: vpkg, + ver: '', + os: '', + arch: '', + libc: '', + lts: false, + channel: '', + formats: [], + limit: 10, + }); + var vrels = vres.releases; + var badVersions = []; + for (var vri = 0; vri < vrels.length; vri++) { + var ver = vrels[vri].version; + if (!ver) { + badVersions.push('(empty)'); + break; + } + if (ver.startsWith('v')) { + badVersions.push(ver); + break; + } + } + if (badVersions.length > 0) { + console.log(' FAIL ' + vpkg + ': v-prefix not stripped: ' + badVersions[0]); + failures++; + } else { + console.log(' PASS ' + vpkg + ': latest=' + (vrels[0] ? vrels[0].version : '?')); + passes++; + } + } catch (e) { + console.log(' FAIL ' + vpkg + ': ' + e.message); + failures++; + } + } + + // ================================================================ + // Test 7: Channel detection + // ================================================================ + console.log(''); + console.log('=== Test 7: Channel Detection ==='); + console.log(''); + + for (var ci = 0; ci < CHANNEL_CASES.length; ci++) { + var ctc = CHANNEL_CASES[ci]; + try { + var cres = await Releases.getReleases({ + pkg: ctc.pkg, + ver: '', + os: '', + arch: '', + libc: '', + lts: false, + channel: ctc.channel, + formats: [], + limit: 100, + }); + var count = cres.releases.length; + if (count < ctc.expectMinCount) { + console.log(' FAIL ' + ctc.label + ': only ' + count + ' (want >=' + ctc.expectMinCount + ')'); + failures++; + } else { + console.log(' PASS ' + ctc.label + ': ' + count + ' releases'); + passes++; + } + } catch (e) { + console.log(' FAIL ' + ctc.label + ': ' + e.message); + failures++; + } + } + + // ================================================================ + // Summary + // ================================================================ + console.log(''); + console.log('=== Results: ' + passes + ' passed, ' + failures + ' failed, ' + knowns + ' known, ' + skips + ' skipped ==='); + if (failures > 0) { + process.exit(1); + } +} + +main().catch(function (err) { + console.error(err.stack); + process.exit(1); +}); diff --git a/_webi/test-fleet-diff.js b/_webi/test-fleet-diff.js new file mode 100644 index 0000000..7767d67 --- /dev/null +++ b/_webi/test-fleet-diff.js @@ -0,0 +1,339 @@ +'use strict'; + +// Fleet-wide diff: compare a candidate host (e.g. beta.webi.sh) against +// production for every cached package, across multiple OS/arch combos. +// Outputs TSV for grep/sort. +// +// Usage: +// node _webi/test-fleet-diff.js +// --cand-url=https://beta.webi.sh +// --prod-url=https://webinstall.dev # default +// --kind=api # default; or "installer" +// --pkgs=bat,go,... # default: all cached packages +// --concurrency=8 # default +// --out=fleet-api.tsv # default: stdout + +let Fs = require('node:fs/promises'); +let Os = require('node:os'); +let Path = require('node:path'); +let Https = require('node:https'); + +let CACHE_DIR = Path.join(Os.homedir(), '.cache/webi/legacy'); + +function arg(name, dflt) { + for (let a of process.argv) { + if (a.startsWith(`--${name}=`)) { + return a.slice(name.length + 3); + } + } + return dflt; +} + +let CAND_URL = arg('cand-url', 'https://beta.webi.sh').replace(/\/+$/, ''); +let PROD_URL = arg('prod-url', 'https://webinstall.dev').replace(/\/+$/, ''); +let KIND = arg('kind', 'api'); +let PKGS_ARG = arg('pkgs', ''); +let CONCURRENCY = parseInt(arg('concurrency', '8'), 10); +let OUT = arg('out', ''); + +// OS/arch matrix for API mode +let API_MATRIX = [ + { os: 'macos', arch: 'amd64' }, + { os: 'macos', arch: 'arm64' }, + { os: 'linux', arch: 'amd64' }, + { os: 'linux', arch: 'arm64' }, + { os: 'linux', arch: 'armv7l' }, + { os: 'windows', arch: 'amd64' }, + { os: 'freebsd', arch: 'amd64' }, +]; + +// UA strings for installer mode +let INSTALLER_MATRIX = [ + { label: 'macos_arm64', ua: 'aarch64/unknown Darwin/24.2.0 libc' }, + { label: 'macos_amd64', ua: 'x86_64/unknown Darwin/23.0.0 libc' }, + { label: 'linux_amd64', ua: 'x86_64/unknown Linux/5.15.0 libc' }, + { label: 'linux_arm64', ua: 'aarch64/unknown Linux/5.15.0 libc' }, + { label: 'linux_musl', ua: 'x86_64/unknown Linux/5.15.0 musl' }, + { label: 'windows_amd64', ua: 'x86_64/unknown Windows/10.0.19041 msvc' }, +]; + +function httpsGet(url, headers) { + return new Promise(function (resolve, reject) { + let opts = { headers: headers || {}, timeout: 15000 }; + let req = Https.get(url, opts, function (res) { + // Follow one redirect + if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + let redir = res.headers.location; + if (redir.startsWith('/')) { + let m = url.match(/^(https?:\/\/[^/]+)/); + redir = (m ? m[1] : '') + redir; + } + Https.get(redir, opts, function (res2) { + let data = ''; + res2.on('data', function (c) { data += c; }); + res2.on('end', function () { resolve({ status: res2.statusCode, body: data }); }); + }).on('error', reject); + return; + } + let data = ''; + res.on('data', function (c) { data += c; }); + res.on('end', function () { resolve({ status: res.statusCode, body: data }); }); + }); + req.on('error', reject); + req.on('timeout', function () { req.destroy(new Error('timeout')); }); + }); +} + +async function listCachedPkgs() { + let entries = await Fs.readdir(CACHE_DIR); + return entries + .filter(function (n) { return n.endsWith('.json') && !n.endsWith('.updated.txt'); }) + .map(function (n) { return n.slice(0, -5); }) + .sort(); +} + +function safeFirst(json) { + try { + let arr = JSON.parse(json); + if (!Array.isArray(arr) || arr.length === 0) { + return null; + } + return arr[0]; + } catch (e) { + return null; + } +} + +function parseInstallerVars(script) { + let vars = {}; + let re = /^(?:export\s+)?(WEBI_\w+|PKG_NAME)='([^']*)'/gm; + let m; + while ((m = re.exec(script)) !== null) { + vars[m[1]] = m[2]; + } + return vars; +} + +async function diffApi(pkg, os, arch) { + let qs = `?os=${os}&arch=${arch}&limit=1`; + let candUrl = `${CAND_URL}/api/releases/${pkg}@stable.json${qs}`; + let prodUrl = `${PROD_URL}/api/releases/${pkg}@stable.json${qs}`; + + let cand, prod; + try { + [cand, prod] = await Promise.all([httpsGet(candUrl), httpsGet(prodUrl)]); + } catch (e) { + return { pkg, os, arch, status: 'fetch_error', detail: e.message }; + } + + if (cand.status !== 200 || prod.status !== 200) { + return { + pkg, os, arch, status: 'http_error', + detail: `cand=${cand.status} prod=${prod.status}`, + }; + } + + let candFirst = safeFirst(cand.body); + let prodFirst = safeFirst(prod.body); + + let candErr = !candFirst || candFirst.channel === 'error'; + let prodErr = !prodFirst || prodFirst.channel === 'error'; + + if (candErr && prodErr) { + return { pkg, os, arch, status: 'both_error', detail: '' }; + } + if (candErr && !prodErr) { + return { + pkg, os, arch, status: 'cand_only_error', + detail: `prod=${prodFirst.version}/${prodFirst.ext}`, + }; + } + if (!candErr && prodErr) { + return { + pkg, os, arch, status: 'prod_only_error', + detail: `cand=${candFirst.version}/${candFirst.ext}`, + }; + } + + // Both succeeded — diff the key fields + let diffs = []; + for (let f of ['os', 'arch', 'libc', 'ext']) { + if (candFirst[f] !== prodFirst[f]) { + diffs.push(`${f}:cand=${candFirst[f]}|prod=${prodFirst[f]}`); + } + } + + let ver = candFirst.version === prodFirst.version + ? candFirst.version + : `cand=${candFirst.version}|prod=${prodFirst.version}`; + + return { + pkg, os, arch, + status: diffs.length === 0 ? 'match' : 'diff', + detail: diffs.length === 0 ? `v${ver} ${candFirst.ext}` : `v${ver} ${diffs.join(',')}`, + }; +} + +async function diffInstaller(pkg, label, ua) { + let candUrl = `${CAND_URL}/api/installers/${pkg}@stable.sh`; + let prodUrl = `${PROD_URL}/api/installers/${pkg}@stable.sh`; + let headers = { 'User-Agent': ua }; + + let cand, prod; + try { + [cand, prod] = await Promise.all([ + httpsGet(candUrl, headers), + httpsGet(prodUrl, headers), + ]); + } catch (e) { + return { pkg, label, status: 'fetch_error', detail: e.message }; + } + + if (cand.status !== 200 || prod.status !== 200) { + return { + pkg, label, status: 'http_error', + detail: `cand=${cand.status} prod=${prod.status}`, + }; + } + + let candVars = parseInstallerVars(cand.body); + let prodVars = parseInstallerVars(prod.body); + + let candHas = candVars.WEBI_PKG_URL && candVars.WEBI_EXT && candVars.WEBI_EXT !== 'err'; + let prodHas = prodVars.WEBI_PKG_URL && prodVars.WEBI_EXT && prodVars.WEBI_EXT !== 'err'; + + if (!candHas && !prodHas) { + return { pkg, label, status: 'both_error', detail: '' }; + } + if (!candHas && prodHas) { + return { + pkg, label, status: 'cand_only_error', + detail: `prod=${prodVars.WEBI_VERSION}/${prodVars.WEBI_EXT}`, + }; + } + if (candHas && !prodHas) { + return { + pkg, label, status: 'prod_only_error', + detail: `cand=${candVars.WEBI_VERSION}/${candVars.WEBI_EXT}`, + }; + } + + // Diff WEBI_OS, WEBI_ARCH, WEBI_EXT (PKG_NAME may differ for aliases) + let diffs = []; + for (let v of ['WEBI_OS', 'WEBI_ARCH', 'WEBI_EXT']) { + if (candVars[v] !== prodVars[v]) { + diffs.push(`${v}:cand=${candVars[v]}|prod=${prodVars[v]}`); + } + } + + let ver = candVars.WEBI_VERSION === prodVars.WEBI_VERSION + ? candVars.WEBI_VERSION + : `cand=${candVars.WEBI_VERSION}|prod=${prodVars.WEBI_VERSION}`; + + return { + pkg, label, + status: diffs.length === 0 ? 'match' : 'diff', + detail: diffs.length === 0 ? `v${ver} ${candVars.WEBI_EXT}` : `v${ver} ${diffs.join(',')}`, + }; +} + +async function pool(items, fn, concurrency) { + let results = new Array(items.length); + let i = 0; + async function worker() { + while (true) { + let idx = i++; + if (idx >= items.length) { + return; + } + try { + results[idx] = await fn(items[idx], idx); + } catch (e) { + results[idx] = { status: 'exception', detail: e.message, _item: items[idx] }; + } + } + } + let workers = []; + for (let k = 0; k < concurrency; k++) { + workers.push(worker()); + } + await Promise.all(workers); + return results; +} + +async function main() { + let pkgs; + if (PKGS_ARG) { + pkgs = PKGS_ARG.split(',').filter(Boolean); + } else { + pkgs = await listCachedPkgs(); + } + + console.error(`Comparing ${pkgs.length} packages: ${CAND_URL} (cand) vs ${PROD_URL} (prod)`); + console.error(`Mode: ${KIND}, concurrency: ${CONCURRENCY}`); + + let jobs = []; + if (KIND === 'api') { + for (let pkg of pkgs) { + for (let combo of API_MATRIX) { + jobs.push({ pkg, os: combo.os, arch: combo.arch }); + } + } + } else if (KIND === 'installer') { + for (let pkg of pkgs) { + for (let combo of INSTALLER_MATRIX) { + jobs.push({ pkg, label: combo.label, ua: combo.ua }); + } + } + } else { + console.error(`Unknown kind: ${KIND}`); + process.exit(2); + } + + let started = Date.now(); + let results = await pool(jobs, async function (job) { + if (KIND === 'api') { + return diffApi(job.pkg, job.os, job.arch); + } + return diffInstaller(job.pkg, job.label, job.ua); + }, CONCURRENCY); + let elapsed = ((Date.now() - started) / 1000).toFixed(1); + + // TSV output + let lines = []; + if (KIND === 'api') { + lines.push(['pkg', 'os', 'arch', 'status', 'detail'].join('\t')); + for (let r of results) { + lines.push([r.pkg, r.os, r.arch, r.status, r.detail || ''].join('\t')); + } + } else { + lines.push(['pkg', 'target', 'status', 'detail'].join('\t')); + for (let r of results) { + lines.push([r.pkg, r.label, r.status, r.detail || ''].join('\t')); + } + } + let body = lines.join('\n') + '\n'; + + if (OUT) { + await Fs.writeFile(OUT, body, 'utf8'); + console.error(`Wrote ${OUT}`); + } else { + process.stdout.write(body); + } + + // Summary to stderr + let counts = {}; + for (let r of results) { + counts[r.status] = (counts[r.status] || 0) + 1; + } + console.error(''); + console.error(`=== Summary (${elapsed}s, ${results.length} jobs) ===`); + for (let s of Object.keys(counts).sort()) { + console.error(` ${s}: ${counts[s]}`); + } +} + +main().catch(function (err) { + console.error(err.stack); + process.exit(1); +}); diff --git a/_webi/test-installer-resolve.js b/_webi/test-installer-resolve.js new file mode 100644 index 0000000..2d7154b --- /dev/null +++ b/_webi/test-installer-resolve.js @@ -0,0 +1,444 @@ +'use strict'; + +let InstallerServer = require('./serve-installer.js'); +let Builds = require('./builds.js'); + +// Real User-Agent strings sent by webi bootstrap scripts. +// +// Libc taxonomy: +// none = static build, no runtime libc dep (often built with musl, but self-contained) +// musl = requires musl C/C++ runtime at runtime (e.g. node-musl) +// gnu = requires glibc at runtime (crashes on musl-only/Alpine) +// libc = host UA value meaning "I have glibc" (not used in release metadata) +// +// Known issues: +// +// 1. WATERFALL libc vs gnu: The WATERFALL maps `libc` => ['none', 'libc'] +// but never tries 'gnu'. Packages with glibc-linked builds (libc='gnu' in +// Go cache) won't match for hosts reporting 'libc'. Fix: update WATERFALL +// to `libc: ['none', 'gnu', 'libc']` in build-classifier submodule. +// +// 2. Go cache .git regression: The Go cache includes .git source repo URLs +// as releases, creating ANYOS/ANYARCH triplets. These match before +// platform-specific binaries. Fix: exclude .git from Go cache output. + +let UA_CASES = [ + // === macOS (no libc issue — darwin uses libc='none') === + { + label: 'bat macOS arm64', + pkg: 'bat', + ua: 'aarch64/unknown Darwin/24.2.0 libc', + expectOs: 'darwin', + expectArch: 'aarch64', + expectExt: 'tar.gz', + }, + { + label: 'bat macOS amd64', + pkg: 'bat', + ua: 'x86_64/unknown Darwin/23.0.0 libc', + expectOs: 'darwin', + expectArch: 'x86_64', + expectExt: 'tar.gz', + }, + { + label: 'go macOS arm64', + pkg: 'go', + ua: 'aarch64/unknown Darwin/24.2.0 libc', + expectOs: 'darwin', + expectArch: 'aarch64', + expectExt: 'tar.gz', + }, + { + label: 'node macOS arm64', + pkg: 'node', + ua: 'aarch64/unknown Darwin/24.2.0 libc', + expectOs: 'darwin', + expectArch: 'aarch64', + expectExt: 'tar.xz', + }, + { + label: 'rg macOS arm64', + pkg: 'rg', + ua: 'aarch64/unknown Darwin/24.2.0 libc', + expectOs: 'darwin', + expectArch: 'aarch64', + expectExt: 'tar.gz', + }, + + // === macOS universal2 — packages where recent darwin builds are universal-only === + // These currently resolve to ancient versions because universal2 entries are + // dropped by the classifier. The GOER's legacy export needs to emit these + // with arch: "x86_64" so the classifier accepts them. The darwin WATERFALL + // (aarch64 falls back to x86_64) handles aarch64 users. + { + label: 'cmake macOS arm64 (universal2)', + pkg: 'cmake', + ua: 'aarch64/unknown Darwin/24.2.0 libc', + expectOs: 'darwin', + expectExt: 'tar.gz', + expectMinVersion: '4.0.0', + known: true, + }, + { + label: 'cmake macOS amd64 (universal2)', + pkg: 'cmake', + ua: 'x86_64/unknown Darwin/23.0.0 libc', + expectOs: 'darwin', + expectExt: 'tar.gz', + expectMinVersion: '4.0.0', + known: true, + }, + { + label: 'hugo macOS arm64 (universal2)', + pkg: 'hugo', + ua: 'aarch64/unknown Darwin/24.2.0 libc', + expectOs: 'darwin', + expectExt: 'tar.gz', + expectMinVersion: '0.140.0', + known: true, + }, + { + label: 'hugo macOS amd64 (universal2)', + pkg: 'hugo', + ua: 'x86_64/unknown Darwin/23.0.0 libc', + expectOs: 'darwin', + expectExt: 'tar.gz', + expectMinVersion: '0.140.0', + known: true, + }, + + // === Windows === + { + label: 'bat Windows amd64', + pkg: 'bat', + ua: 'x86_64/unknown Windows/10.0.19041 msvc', + expectOs: 'windows', + expectArch: 'x86_64', + expectExt: 'zip', + }, + { + label: 'go Windows amd64', + pkg: 'go', + ua: 'x86_64/unknown Windows/10.0.19041 msvc', + expectOs: 'windows', + expectArch: 'x86_64', + expectExt: 'zip', + }, + + // === Linux musl (Alpine/Docker) === + { + label: 'bat Linux musl', + pkg: 'bat', + ua: 'x86_64/unknown Linux/5.15.0 musl', + expectOs: 'linux', + expectArch: 'x86_64', + expectExt: 'tar.gz', + }, + + // === Linux glibc — packages with libc='none' in cache === + { + label: 'go Linux amd64', + pkg: 'go', + ua: 'x86_64/unknown Linux/5.15.0 libc', + expectOs: 'linux', + expectArch: 'x86_64', + expectExt: 'tar.gz', + }, + // === Linux glibc — packages with libc='gnu' in cache === + // These previously failed (WATERFALL libc→gnu gap). Fixed by adding + // 'gnu' to the libc candidates for glibc hosts in _enumerateTriplets. + { + label: 'bat Linux amd64', + pkg: 'bat', + ua: 'x86_64/unknown Linux/5.15.0 libc', + expectOs: 'linux', + expectArch: 'x86_64', + expectExt: 'tar.gz', + }, + { + label: 'rg Linux amd64', + pkg: 'rg', + ua: 'x86_64/unknown Linux/5.15.0 libc', + expectOs: 'linux', + expectArch: 'x86_64', + expectExt: 'tar.gz', + }, + { + label: 'node Linux amd64', + pkg: 'node', + ua: 'x86_64/unknown Linux/5.15.0 libc', + expectOs: 'linux', + expectArch: 'x86_64', + expectExt: 'tar.xz', + }, + + // === Packages with .git source URLs in old releases === + // These previously failed (ANYOS .git matched before platform binary). + // Fixed by putting specific OS before ANYOS in triplet enumeration. + { + label: 'jq macOS arm64', + pkg: 'jq', + ua: 'aarch64/unknown Darwin/24.2.0 libc', + expectOs: 'darwin', + expectArch: 'aarch64', + expectExt: 'exe', + }, + { + label: 'caddy macOS arm64', + pkg: 'caddy', + ua: 'aarch64/unknown Darwin/24.2.0 libc', + expectOs: 'darwin', + expectArch: 'aarch64', + expectExt: 'tar.gz', + }, + { + label: 'caddy Linux amd64', + pkg: 'caddy', + ua: 'x86_64/unknown Linux/5.15.0 libc', + expectOs: 'linux', + expectArch: 'x86_64', + expectExt: 'tar.gz', + }, +]; + +async function main() { + let failures = 0; + let passes = 0; + let knowns = 0; + let errors = 0; + + console.log('Initializing build cache...'); + await Builds.init(); + console.log(''); + + console.log('=== Installer Resolution Tests ==='); + console.log(''); + + for (let tc of UA_CASES) { + try { + let [pkg, params] = await InstallerServer.helper({ + unameAgent: tc.ua, + projectName: tc.pkg, + tag: 'stable', + formats: ['tar', 'exe', 'zip', 'xz', 'dmg'], + libc: '', + }); + + // Known issue — just verify it fails as expected + if (tc.known) { + let isError = pkg.channel === 'error' || !pkg.download || pkg.download.includes('doesntexist') || pkg.ext === 'git'; + let isStale = false; + if (tc.expectMinVersion && pkg.version) { + let got = pkg.version.replace(/^v/, '').split('.').map(Number); + let want = tc.expectMinVersion.split('.').map(Number); + for (let i = 0; i < want.length; i++) { + if ((got[i] || 0) < want[i]) { isStale = true; break; } + if ((got[i] || 0) > want[i]) { break; } + } + } + if (isError || isStale) { + let detail = isStale ? `stale v${pkg.version} < v${tc.expectMinVersion}` : ''; + console.log(` KNOWN ${tc.label}${detail ? ': ' + detail : ''}`); + knowns++; + } else { + console.log(` PASS ${tc.label} (known issue resolved!): v${pkg.version} .${pkg.ext} ${(pkg.download || '').split('/').pop()}`); + passes++; + } + continue; + } + + if (pkg.channel === 'error') { + console.log(` FAIL ${tc.label}: resolved to error package`); + failures++; + continue; + } + + let diffs = []; + + if (tc.expectOs && pkg.os !== tc.expectOs) { + diffs.push(`os: got=${pkg.os} want=${tc.expectOs}`); + } + if (tc.expectArch && pkg.arch !== tc.expectArch) { + diffs.push(`arch: got=${pkg.arch} want=${tc.expectArch}`); + } + if (tc.expectExt && pkg.ext !== tc.expectExt) { + diffs.push(`ext: got=${pkg.ext} want=${tc.expectExt}`); + } + + if (!pkg.version || pkg.version === '0.0.0') { + diffs.push('version: missing or zero'); + } + + if (!pkg.download || pkg.download.includes('doesntexist')) { + diffs.push('download: missing or error'); + } + + if (diffs.length > 0) { + console.log(` FAIL ${tc.label}: ${diffs.join(', ')}`); + failures++; + } else { + console.log(` PASS ${tc.label}: v${pkg.version} .${pkg.ext} ${pkg.download.split('/').pop()}`); + passes++; + } + } catch (err) { + if (tc.known) { + console.log(` KNOWN ${tc.label} (error: ${err.message})`); + knowns++; + continue; + } + console.log(` ERROR ${tc.label}: ${err.message}`); + errors++; + } + } + + console.log(''); + console.log(`=== Results: ${passes} passed, ${failures} failed, ${knowns} known, ${errors} errors ===`); + if (failures > 0 || errors > 0) { + process.exit(1); + } + + // Cache value validation: the classifier re-parses filenames and rejects + // entries where the cache os/arch doesn't match. These checks prevent + // regressions where someone "normalizes" cache values in a way that + // breaks the classifier. + console.log(''); + console.log('=== Cache Value Validation ==='); + console.log(''); + let cacheFailures = await validateCacheValues(); + if (cacheFailures > 0) { + process.exit(1); + } +} + +// Verify that cache os/arch values match what the Node classifier expects +// to extract from the download filename. The classifier is a submodule and +// is NOT being modified — the cache must emit values it already recognizes. +// +// Known bug (LIVE_cache): the Go legacy export previously translated +// solaris/illumos → sunos in the cache, but the filenames still say +// solaris/illumos. The classifier detects the filename value and rejects +// the entry when it doesn't match. Same issue with universal2 arch. +// +// Rule: cache os/arch must match the filename, not some "canonical" form. + +// Cache os/arch values must match what the Node classifier extracts from the +// download filename. The classifier already recognizes solaris, illumos, sunos, +// armhf, armel, etc. — these are not new values. The only value the classifier +// does NOT recognize is "universal2" — use "x86_64" instead. +// +// matchField: which field to check in the release entry ('name' or 'download') +let CACHE_CHECKS = [ + // The classifier knows "solaris" as an OS. Filenames/URLs say "solaris". + // Do NOT translate to "sunos" — that creates a mismatch and drops the entry. + { + label: 'terraform solaris entries have os=solaris (not sunos)', + pkg: 'terraform', + matchField: 'download', + filenameMatch: /solaris/, + field: 'os', + expect: 'solaris', + }, + { + label: 'syncthing solaris entries have os=solaris (not sunos)', + pkg: 'syncthing', + matchField: 'download', + filenameMatch: /solaris/, + field: 'os', + expect: 'solaris', + }, + // The classifier knows "illumos" as an OS. Don't translate to sunos. + { + label: 'syncthing illumos entries have os=illumos (not sunos)', + pkg: 'syncthing', + matchField: 'download', + filenameMatch: /illumos/, + field: 'os', + expect: 'illumos', + }, + // node.js uses "sunos" in filenames — cache must say "sunos" (already correct) + { + label: 'node sunos entries have os=sunos', + pkg: 'node', + matchField: 'name', + filenameMatch: /sunos/, + field: 'os', + expect: 'sunos', + }, + // The classifier maps "universal" in filenames → x86_64. The classifier does + // NOT recognize "universal2". Cache must say arch="x86_64" for these entries. + // aarch64 users get them via the darwin WATERFALL (aarch64 → x86_64 fallback). + { + label: 'cmake universal entries have arch=x86_64 (not universal2)', + pkg: 'cmake', + matchField: 'download', + filenameMatch: /universal/, + field: 'arch', + expect: 'x86_64', + }, + { + label: 'hugo universal entries have arch=x86_64 (not universal2)', + pkg: 'hugo', + matchField: 'download', + filenameMatch: /universal/, + field: 'arch', + expect: 'x86_64', + }, +]; + +async function validateCacheValues() { + let Os = require('node:os'); + let Path = require('path'); + let Fs = require('fs'); + + let cachePath = Path.join(Os.homedir(), '.cache/webi/legacy'); + if (!Fs.existsSync(cachePath)) { + console.log(' SKIP: no cache directory at ' + cachePath); + return 0; + } + + let failures = 0; + + for (let check of CACHE_CHECKS) { + let filePath = Path.join(cachePath, `${check.pkg}.json`); + if (!Fs.existsSync(filePath)) { + console.log(` SKIP ${check.label}: no cache file`); + continue; + } + + let data = JSON.parse(Fs.readFileSync(filePath, 'utf8')); + let matchField = check.matchField || 'name'; + let matched = data.releases.filter(function (r) { + return check.filenameMatch.test(r[matchField]); + }); + + if (matched.length === 0) { + console.log(` SKIP ${check.label}: no matching filenames`); + continue; + } + + let wrong = matched.filter(function (r) { + return r[check.field] !== check.expect; + }); + + if (wrong.length > 0) { + let sample = wrong[0]; + console.log( + ` FAIL ${check.label}: ${wrong.length}/${matched.length} entries have` + + ` ${check.field}="${sample[check.field]}" (want "${check.expect}")` + + ` e.g. ${sample.name}`, + ); + failures++; + } else { + console.log(` PASS ${check.label}: ${matched.length} entries OK`); + } + } + + console.log(''); + console.log(`=== Cache Validation: ${CACHE_CHECKS.length - failures} passed, ${failures} failed ===`); + return failures; +} + +main().catch(function (err) { + console.error(err.stack); + process.exit(1); +}); diff --git a/_webi/test-live-cache-diff.js b/_webi/test-live-cache-diff.js new file mode 100644 index 0000000..814f8cb --- /dev/null +++ b/_webi/test-live-cache-diff.js @@ -0,0 +1,450 @@ +'use strict'; + +// Compare _cache vs LIVE_cache for correctness and compatibility. +// +// Rules (from NODER_PURPOSE.md): +// - _cache should be more complete and more correct than LIVE_cache +// - Must NOT introduce new tags (OS, arch, libc) that don't exist in LIVE_cache +// - Must NOT break compatibility with existing data +// +// Usage: node _webi/test-live-cache-diff.js + +var Fs = require('node:fs'); +var Os = require('node:os'); +var Path = require('node:path'); + +// CACHE_DIR is the live cache produced by webicached (flat layout). +// LIVE_DIR is the historical snapshot taken pre-cutover (month-bucketed). +var CACHE_DIR = Path.join(Os.homedir(), '.cache/webi/legacy'); +var LIVE_DIR = Path.join(__dirname, '..', 'LIVE_cache'); + +// resolveLayout: figures out whether dir uses the flat layout +// (~/.cache/webi/legacy/.json) or the legacy month-bucketed layout +// (LIVE_cache//.json) and returns the directory to read from. +function resolveLayout(dir) { + if (!Fs.existsSync(dir)) { + return null; + } + var entries = Fs.readdirSync(dir); + var months = entries + .filter(function (d) { return /^\d{4}-\d{2}$/.test(d); }) + .sort() + .reverse(); + if (months[0]) { + return Path.join(dir, months[0]); + } + // Flat layout (cache files directly under dir). + return dir; +} + +function loadReleases(layoutPath, pkg) { + var file = Path.join(layoutPath, pkg + '.json'); + if (!Fs.existsSync(file)) { + return null; + } + try { + return JSON.parse(Fs.readFileSync(file, 'utf8')); + } catch (e) { + return null; + } +} + +function uniqueValues(releases, field) { + var seen = {}; + for (var i = 0; i < releases.length; i++) { + var val = releases[i][field]; + if (val !== null && val !== undefined && val !== '') { + seen[val] = true; + } + } + return Object.keys(seen).sort(); +} + +async function main() { + var passes = 0; + var failures = 0; + var warns = 0; + + var cachePath = resolveLayout(CACHE_DIR); + var livePath = resolveLayout(LIVE_DIR); + + if (!cachePath) { + console.error('No _cache directory found'); + process.exit(1); + } + if (!livePath) { + console.error('No LIVE_cache directory found'); + process.exit(1); + } + + console.log('Using cache: ' + cachePath + ' vs ' + livePath); + console.log(''); + + // Get all packages that exist in both caches + var cacheFiles = Fs.readdirSync(cachePath) + .filter(function (f) { return f.endsWith('.json'); }) + .map(function (f) { return f.replace('.json', ''); }); + var liveFiles = Fs.readdirSync(livePath) + .filter(function (f) { return f.endsWith('.json'); }) + .map(function (f) { return f.replace('.json', ''); }); + + var cacheSet = {}; + var liveSet = {}; + cacheFiles.forEach(function (f) { cacheSet[f] = true; }); + liveFiles.forEach(function (f) { liveSet[f] = true; }); + + var common = cacheFiles.filter(function (f) { return liveSet[f]; }); + + // ================================================================ + // Test 1: Global vocabulary — no new OS/arch/libc tags + // ================================================================ + console.log('=== Test 1: No New Tags (OS/Arch/Libc) ==='); + console.log(''); + + var allLiveOs = {}; + var allLiveArch = {}; + var allLiveLibc = {}; + var allCacheOs = {}; + var allCacheArch = {}; + var allCacheLibc = {}; + + for (var ci = 0; ci < common.length; ci++) { + var pkg = common[ci]; + var liveData = loadReleases(livePath, pkg); + var cacheData = loadReleases(cachePath, pkg); + if (!liveData || !cacheData) { continue; } + + uniqueValues(liveData.releases, 'os').forEach(function (v) { allLiveOs[v] = true; }); + uniqueValues(liveData.releases, 'arch').forEach(function (v) { allLiveArch[v] = true; }); + uniqueValues(liveData.releases, 'libc').forEach(function (v) { allLiveLibc[v] = true; }); + uniqueValues(cacheData.releases, 'os').forEach(function (v) { allCacheOs[v] = true; }); + uniqueValues(cacheData.releases, 'arch').forEach(function (v) { allCacheArch[v] = true; }); + uniqueValues(cacheData.releases, 'libc').forEach(function (v) { allCacheLibc[v] = true; }); + } + + var newOs = Object.keys(allCacheOs).filter(function (v) { return !allLiveOs[v]; }).sort(); + var newArch = Object.keys(allCacheArch).filter(function (v) { return !allLiveArch[v]; }).sort(); + var newLibc = Object.keys(allCacheLibc).filter(function (v) { return !allLiveLibc[v]; }).sort(); + + if (newOs.length > 0) { + console.log(' FAIL new OS values in _cache not in LIVE_cache: ' + JSON.stringify(newOs)); + failures++; + } else { + console.log(' PASS no new OS values'); + passes++; + } + + if (newArch.length > 0) { + console.log(' FAIL new arch values in _cache not in LIVE_cache: ' + JSON.stringify(newArch)); + failures++; + } else { + console.log(' PASS no new arch values'); + passes++; + } + + if (newLibc.length > 0) { + console.log(' FAIL new libc values in _cache not in LIVE_cache: ' + JSON.stringify(newLibc)); + failures++; + } else { + console.log(' PASS no new libc values'); + passes++; + } + + // Show what LIVE has that _cache doesn't (informational) + var missingOs = Object.keys(allLiveOs).filter(function (v) { return !allCacheOs[v]; }).sort(); + var missingArch = Object.keys(allLiveArch).filter(function (v) { return !allCacheArch[v]; }).sort(); + if (missingOs.length > 0) { + console.log(' INFO OS in LIVE but not _cache: ' + JSON.stringify(missingOs)); + } + if (missingArch.length > 0) { + console.log(' INFO arch in LIVE but not _cache: ' + JSON.stringify(missingArch)); + } + + // ================================================================ + // Test 2: Per-package release count — _cache should have >= LIVE + // ================================================================ + console.log(''); + console.log('=== Test 2: Release Count (per package) ==='); + console.log(''); + + // LIVE_cache includes junk entries (.pem, .sig, .sha256, .deb, .rpm, etc.) + // that Go correctly filters out. Only count installable entries. + var junkExts = /\.(pem|sig|asc|sha256|sha512|sha256sum|sha512sum|deb|rpm|apk|sbom|json|txt|sum|md5|cosign-bundle|intoto-jsonl)$/i; + + function countInstallable(releases) { + var n = 0; + for (var i = 0; i < releases.length; i++) { + if (!junkExts.test(releases[i].name || '')) { + n++; + } + } + return n; + } + + var countIssues = []; + for (var pi = 0; pi < common.length; pi++) { + var ppkg = common[pi]; + var pLive = loadReleases(livePath, ppkg); + var pCache = loadReleases(cachePath, ppkg); + if (!pLive || !pCache) { continue; } + + var liveCount = countInstallable(pLive.releases); + var cacheCount = pCache.releases.length; + + // _cache should have at least as many installable releases as LIVE + var ratio = liveCount > 0 ? cacheCount / liveCount : 1; + if (ratio < 0.5 && liveCount > 10) { + countIssues.push({ pkg: ppkg, live: liveCount, cache: cacheCount, ratio: ratio }); + } + } + + if (countIssues.length === 0) { + console.log(' PASS all packages have adequate release counts'); + passes++; + } else { + for (var ii = 0; ii < countIssues.length; ii++) { + var issue = countIssues[ii]; + console.log(' FAIL ' + issue.pkg + ': _cache=' + issue.cache + ' LIVE=' + issue.live + ' (ratio=' + issue.ratio.toFixed(2) + ')'); + failures++; + } + } + + // ================================================================ + // Test 3: Per-package OS coverage — _cache should have same core OSes + // ================================================================ + console.log(''); + console.log('=== Test 3: OS Coverage (per package) ==='); + console.log(''); + + var coreOses = ['darwin', 'linux', 'windows']; + var osIssues = []; + for (var oi = 0; oi < common.length; oi++) { + var opkg = common[oi]; + var oLive = loadReleases(livePath, opkg); + var oCache = loadReleases(cachePath, opkg); + if (!oLive || !oCache) { continue; } + + var liveOses = uniqueValues(oLive.releases, 'os'); + var cacheOses = uniqueValues(oCache.releases, 'os'); + + // For each core OS in LIVE, it should also be in _cache + for (var coi = 0; coi < coreOses.length; coi++) { + var os = coreOses[coi]; + if (liveOses.indexOf(os) >= 0 && cacheOses.indexOf(os) < 0) { + osIssues.push({ pkg: opkg, os: os }); + } + } + } + + if (osIssues.length === 0) { + console.log(' PASS all packages have matching core OS coverage'); + passes++; + } else { + for (var oii = 0; oii < osIssues.length; oii++) { + var oisue = osIssues[oii]; + console.log(' FAIL ' + oisue.pkg + ': missing os=' + oisue.os + ' (present in LIVE)'); + failures++; + } + } + + // ================================================================ + // Test 4: Per-package new tags — flag packages introducing new values + // ================================================================ + console.log(''); + console.log('=== Test 4: Per-Package New Tags ==='); + console.log(''); + + var tagIssues = []; + var tagSkipped = 0; + for (var ti = 0; ti < common.length; ti++) { + var tpkg = common[ti]; + var tLive = loadReleases(livePath, tpkg); + var tCache = loadReleases(cachePath, tpkg); + if (!tLive || !tCache) { continue; } + + var tLiveOs = {}; + var tLiveArch = {}; + uniqueValues(tLive.releases, 'os').forEach(function (v) { tLiveOs[v] = true; }); + uniqueValues(tLive.releases, 'arch').forEach(function (v) { tLiveArch[v] = true; }); + + // Skip packages where LIVE has no classified entries (all empty os/arch). + // These are github_releases packages where classification happens at query + // time. Our _cache filling in values is an improvement, not a regression. + var liveOsKeys = Object.keys(tLiveOs); + var liveArchKeys = Object.keys(tLiveArch); + if (liveOsKeys.length === 0 && liveArchKeys.length === 0) { + tagSkipped++; + continue; + } + + var tCacheOs = uniqueValues(tCache.releases, 'os'); + var tCacheArch = uniqueValues(tCache.releases, 'arch'); + + var tNewOs = tCacheOs.filter(function (v) { return !tLiveOs[v]; }); + var tNewArch = tCacheArch.filter(function (v) { return !tLiveArch[v]; }); + + if (tNewOs.length > 0 || tNewArch.length > 0) { + tagIssues.push({ + pkg: tpkg, + newOs: tNewOs, + newArch: tNewArch, + }); + } + } + + if (tagSkipped > 0) { + console.log(' INFO skipped ' + tagSkipped + ' packages with no LIVE classification (unclassified github_releases)'); + } + if (tagIssues.length === 0) { + console.log(' PASS no pre-classified packages introduce new tags'); + passes++; + } else { + for (var tii = 0; tii < tagIssues.length; tii++) { + var tissue = tagIssues[tii]; + var parts = []; + if (tissue.newOs.length > 0) { + parts.push('os: ' + JSON.stringify(tissue.newOs)); + } + if (tissue.newArch.length > 0) { + parts.push('arch: ' + JSON.stringify(tissue.newArch)); + } + console.log(' WARN ' + tissue.pkg + ': new tags: ' + parts.join(', ')); + warns++; + } + } + + // ================================================================ + // Test 5: Latest stable version — _cache should be >= LIVE + // ================================================================ + console.log(''); + console.log('=== Test 5: Latest Stable Version ==='); + console.log(''); + + var stableCheckPkgs = ['bat', 'go', 'node', 'rg', 'caddy', 'jq', 'hugo', 'terraform']; + for (var si = 0; si < stableCheckPkgs.length; si++) { + var spkg = stableCheckPkgs[si]; + var sLive = loadReleases(livePath, spkg); + var sCache = loadReleases(cachePath, spkg); + if (!sLive || !sCache) { + console.log(' SKIP ' + spkg + ': missing data'); + continue; + } + + // Find first stable release in each + var liveStable = sLive.releases.find(function (r) { return r.channel === 'stable'; }); + var cacheStable = sCache.releases.find(function (r) { return r.channel === 'stable'; }); + + if (!liveStable || !cacheStable) { + console.log(' SKIP ' + spkg + ': no stable release found'); + continue; + } + + var lv = (liveStable.version || '').replace(/^v/, ''); + var cv = (cacheStable.version || '').replace(/^v/, ''); + + if (lv === cv) { + console.log(' PASS ' + spkg + ': ' + cv); + passes++; + } else { + // Just warn — versions may differ due to cache age + console.log(' WARN ' + spkg + ': LIVE=' + lv + ' _cache=' + cv); + warns++; + } + } + + // ================================================================ + // Test 6: Download URLs — all entries should have valid URLs + // ================================================================ + console.log(''); + console.log('=== Test 6: Download URL Validity ==='); + console.log(''); + + var urlIssues = []; + for (var ui = 0; ui < common.length; ui++) { + var upkg = common[ui]; + var uCache = loadReleases(cachePath, upkg); + if (!uCache) { continue; } + + var emptyUrls = 0; + var badUrls = 0; + for (var uri = 0; uri < uCache.releases.length; uri++) { + var rel = uCache.releases[uri]; + var url = rel.download || ''; + if (url === '') { + emptyUrls++; + } else if (!/^https?:\/\//.test(url)) { + badUrls++; + } + } + if (emptyUrls > 0 || badUrls > 0) { + urlIssues.push({ pkg: upkg, empty: emptyUrls, bad: badUrls }); + } + } + + if (urlIssues.length === 0) { + console.log(' PASS all packages have valid download URLs'); + passes++; + } else { + for (var uii = 0; uii < urlIssues.length; uii++) { + var uissue = urlIssues[uii]; + var uparts = []; + if (uissue.empty > 0) { uparts.push(uissue.empty + ' empty'); } + if (uissue.bad > 0) { uparts.push(uissue.bad + ' malformed'); } + console.log(' FAIL ' + uissue.pkg + ': ' + uparts.join(', ')); + failures++; + } + } + + // ================================================================ + // Test 7: Required fields — all entries should have version + name + // ================================================================ + console.log(''); + console.log('=== Test 7: Required Fields ==='); + console.log(''); + + var fieldIssues = []; + for (var fi = 0; fi < common.length; fi++) { + var fpkg = common[fi]; + var fCache = loadReleases(cachePath, fpkg); + if (!fCache) { continue; } + + var noVersion = 0; + var noName = 0; + for (var fri = 0; fri < fCache.releases.length; fri++) { + var frel = fCache.releases[fri]; + if (!frel.version) { noVersion++; } + if (!frel.name && !frel.download) { noName++; } + } + if (noVersion > 0 || noName > 0) { + fieldIssues.push({ pkg: fpkg, noVersion: noVersion, noName: noName }); + } + } + + if (fieldIssues.length === 0) { + console.log(' PASS all packages have version and name/download'); + passes++; + } else { + for (var fii = 0; fii < fieldIssues.length; fii++) { + var fissue = fieldIssues[fii]; + var fparts = []; + if (fissue.noVersion > 0) { fparts.push(fissue.noVersion + ' missing version'); } + if (fissue.noName > 0) { fparts.push(fissue.noName + ' missing name+download'); } + console.log(' FAIL ' + fissue.pkg + ': ' + fparts.join(', ')); + failures++; + } + } + + // ================================================================ + // Summary + // ================================================================ + console.log(''); + console.log('=== Results: ' + passes + ' passed, ' + failures + ' failed, ' + warns + ' warnings ==='); + if (failures > 0) { + process.exit(1); + } +} + +main().catch(function (err) { + console.error(err.stack); + process.exit(1); +}); diff --git a/_webi/test-live-compare.js b/_webi/test-live-compare.js new file mode 100644 index 0000000..25c1f98 --- /dev/null +++ b/_webi/test-live-compare.js @@ -0,0 +1,577 @@ +'use strict'; + +// Comprehensive live-vs-local comparison test. +// Fetches from a remote API (default: webinstall.dev) and compares against +// local cache-only output to catch regressions. +// +// Usage: +// node _webi/test-live-compare.js # compare against prod +// node _webi/test-live-compare.js --refresh # refresh golden data +// node _webi/test-live-compare.js --base-url=https://beta.webi.sh +// node _webi/test-live-compare.js --all # all cached pkgs +// node _webi/test-live-compare.js --all --tsv # TSV output +// node _webi/test-live-compare.js --base-url=https://webi.sh \ +// --cand-url=https://beta.webi.sh # remote-vs-remote +// # (Test 4 only; +// # no local cache +// # needed) + +let Fs = require('node:fs/promises'); +let Os = require('node:os'); +let Path = require('node:path'); +let Https = require('node:https'); + +let Releases = require('./transform-releases.js'); +let InstallerServer = require('./serve-installer.js'); +let Builds = require('./builds.js'); + +let TESTDATA_DIR = Path.join(__dirname, 'testdata'); +let CACHE_DIR = Path.join(Os.homedir(), '.cache/webi/legacy'); + +let REFRESH = process.argv.includes('--refresh'); +let ALL_PKGS = process.argv.includes('--all'); +let TSV = process.argv.includes('--tsv'); + +let BASE_URL = 'https://webinstall.dev'; +let CAND_URL = ''; +for (let arg of process.argv) { + if (arg.startsWith('--base-url=')) { + BASE_URL = arg.slice('--base-url='.length).replace(/\/+$/, ''); + } else if (arg.startsWith('--cand-url=')) { + CAND_URL = arg.slice('--cand-url='.length).replace(/\/+$/, ''); + } +} + +// Packages to test — mix of Go-built, Rust-built, and C/C++ projects +let TEST_PKGS = ['bat', 'go', 'node', 'rg', 'jq', 'caddy']; + +async function listCachedPkgs() { + let entries; + try { + entries = await Fs.readdir(CACHE_DIR); + } catch (e) { + console.error(`No cache directory: ${CACHE_DIR}`); + return []; + } + let pkgs = entries + .filter(function (n) { return n.endsWith('.json'); }) + .map(function (n) { return n.slice(0, -5); }) + .sort(); + return pkgs; +} + +// OS/arch combos for filtered release API tests +let RELEASE_API_CASES = [ + { os: 'macos', arch: 'amd64' }, + { os: 'macos', arch: 'arm64' }, + { os: 'linux', arch: 'amd64' }, + { os: 'windows', arch: 'amd64' }, +]; + +// Older / channel-specific version specs that map to webi @ +// invocations. Each case exercises a code path that the unfiltered +// "@stable" sweep above would never hit: +// - LTS filter (lts=true) +// - Channel filter (channel=beta) +// - Major-series prefix (ver=20 → /^20\b/) +// - Older minor (ver=0.18 → /^0.18\b/) +// - Older major (ver=1.21 → /^1.21\b/) for projects with deep history +let VERSION_SPEC_CASES = [ + { pkg: 'node', spec: 'lts', ver: '', channel: '', lts: true, os: 'linux', arch: 'amd64' }, + { pkg: 'node', spec: '20', ver: '20', channel: '', lts: false, os: 'linux', arch: 'amd64' }, + { pkg: 'node', spec: 'beta', ver: '', channel: 'beta', lts: false, os: 'linux', arch: 'amd64' }, + { pkg: 'go', spec: '1.22', ver: '1.22', channel: '', lts: false, os: 'linux', arch: 'amd64' }, + { pkg: 'go', spec: '1.21', ver: '1.21', channel: '', lts: false, os: 'macos', arch: 'arm64' }, + { pkg: 'bat', spec: '0.20', ver: '0.20', channel: '', lts: false, os: 'linux', arch: 'amd64' }, + { pkg: 'bat', spec: '0.18', ver: '0.18', channel: '', lts: false, os: 'linux', arch: 'amd64' }, + { pkg: 'rg', spec: '13', ver: '13', channel: '', lts: false, os: 'linux', arch: 'amd64' }, + { pkg: 'caddy', spec: '2.7', ver: '2.7', channel: '', lts: false, os: 'linux', arch: 'amd64' }, +]; + +// UA strings for installer resolution tests +let INSTALLER_CASES = [ + { label: 'macOS arm64', ua: 'aarch64/unknown Darwin/24.2.0 libc' }, + { label: 'macOS amd64', ua: 'x86_64/unknown Darwin/23.0.0 libc' }, + { label: 'Linux musl', ua: 'x86_64/unknown Linux/5.15.0 musl' }, + { label: 'Windows amd64', ua: 'x86_64/unknown Windows/10.0.19041 msvc' }, +]; + +// Known differences between Go cache and production (not regressions) +// Extensions that the Go cache correctly excludes (non-installable formats) +// OR that the Go cache includes but shouldn't (man pages, etc.) +let KNOWN_EXT_DIFFS = new Set([ + 'deb', 'rpm', 'sha256', 'sig', 'pem', 'sbom', 'txt', + '1', '2', '3', '4', '5', '6', '7', '8', // man page extensions +]); + +function httpsGet(url) { + return new Promise(function (resolve, reject) { + Https.get(url, function (res) { + let data = ''; + res.on('data', function (chunk) { + data += chunk; + }); + res.on('end', function () { + if (res.statusCode !== 200) { + reject(new Error(`HTTP ${res.statusCode}: ${url}`)); + return; + } + resolve(data); + }); + }).on('error', reject); + }); +} + +async function fetchLiveReleases(pkg, os, arch, limit) { + let url = `${BASE_URL}/api/releases/${pkg}@stable.json?limit=${limit || 100}`; + if (os) { + url += `&os=${os}`; + } + if (arch) { + url += `&arch=${arch}`; + } + let json = await httpsGet(url); + return JSON.parse(json); +} + +async function fetchAtSpec(baseUrl, pkg, spec, os, arch) { + let url = `${baseUrl}/api/releases/${pkg}@${spec}.json?limit=1`; + if (os) { + url += `&os=${os}`; + } + if (arch) { + url += `&arch=${arch}`; + } + let json = await httpsGet(url); + return JSON.parse(json); +} + +async function fetchLiveInstaller(pkg, ua) { + return new Promise(function (resolve, reject) { + let url = `${BASE_URL}/${pkg}@stable.sh`; + let opts = { + headers: { 'User-Agent': ua }, + }; + Https.get(url, opts, function (res) { + // Follow redirects + if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + let redir = res.headers.location; + if (redir.startsWith('/')) { + redir = BASE_URL + redir; + } + Https.get(redir, opts, function (res2) { + let data = ''; + res2.on('data', function (chunk) { + data += chunk; + }); + res2.on('end', function () { + resolve(data); + }); + }).on('error', reject); + return; + } + let data = ''; + res.on('data', function (chunk) { + data += chunk; + }); + res.on('end', function () { + resolve(data); + }); + }).on('error', reject); + }); +} + +function parseInstallerVars(scriptText) { + let vars = {}; + // Match both `export WEBI_FOO='...'` and `WEBI_FOO='...'` + let names = ['WEBI_PKG_URL', 'WEBI_VERSION', 'WEBI_EXT', 'WEBI_OS', 'WEBI_ARCH', 'PKG_NAME']; + for (let name of names) { + let re = new RegExp("^(?:export\\s+)?" + name + "='([^']*)'", 'm'); + let m = scriptText.match(re); + if (m) { + vars[name] = m[1]; + } + } + return vars; +} + +async function saveGolden(name, data) { + await Fs.mkdir(TESTDATA_DIR, { recursive: true }); + let file = Path.join(TESTDATA_DIR, name); + await Fs.writeFile(file, JSON.stringify(data), 'utf8'); +} + +async function loadGolden(name) { + let file = Path.join(TESTDATA_DIR, name); + try { + let json = await Fs.readFile(file, 'utf8'); + return JSON.parse(json); + } catch (e) { + if (e.code === 'ENOENT') { + return null; + } + throw e; + } +} + +async function main() { + let passes = 0; + let failures = 0; + let skips = 0; + let knowns = 0; + + console.log('Initializing build cache...'); + await Builds.init(); + console.log(''); + + // ================================================================ + // Test 1: Release API — unfiltered + // ================================================================ + console.log('=== Test 1: Unfiltered /api/releases/{pkg}.json ==='); + console.log(''); + + for (let pkg of TEST_PKGS) { + let goldenName = `live_${pkg}.json`; + let liveReleases; + + if (REFRESH) { + try { + liveReleases = await fetchLiveReleases(pkg); + await saveGolden(goldenName, liveReleases); + console.log(` refreshed ${goldenName}`); + } catch (e) { + console.log(` SKIP ${pkg}: fetch error: ${e.message}`); + skips++; + continue; + } + } else { + liveReleases = await loadGolden(goldenName); + if (!liveReleases) { + console.log(` SKIP ${pkg}: no golden data (run with --refresh)`); + skips++; + continue; + } + } + + let localResult = await Releases.getReleases({ + pkg: pkg, + ver: '', + os: '', + arch: '', + libc: '', + lts: false, + channel: '', + formats: [], + limit: 100, + }); + let localReleases = localResult.releases; + + // Compare OS vocabulary — check that local has the core OSes that live has + let liveOses = [...new Set(liveReleases.map(function (r) { return r.os; }))].sort(); + let localOses = [...new Set(localReleases.map(function (r) { return r.os; }))].sort(); + let coreOses = ['linux', 'macos', 'windows']; + let liveCore = liveOses.filter(function (o) { return coreOses.includes(o); }).sort(); + let localCore = localOses.filter(function (o) { return coreOses.includes(o); }).sort(); + // Local should have at least the core OSes that live has + let missingCore = liveCore.filter(function (o) { return !localCore.includes(o); }); + if (missingCore.length > 0) { + console.log(` FAIL ${pkg} OS: missing core OSes: ${JSON.stringify(missingCore)}`); + failures++; + } else { + console.log(` PASS ${pkg} OS: ${JSON.stringify(localCore)}`); + passes++; + } + + // Compare ext vocabulary (excluding known non-installable formats) + let liveExts = [...new Set(liveReleases.map(function (r) { return r.ext; }))].sort(); + let localExts = [...new Set(localReleases.map(function (r) { return r.ext; }))].sort(); + let liveExtsFiltered = liveExts.filter(function (e) { return !KNOWN_EXT_DIFFS.has(e); }); + let localExtsFiltered = localExts.filter(function (e) { return !KNOWN_EXT_DIFFS.has(e); }); + let extMatch = true; + for (let ext of localExtsFiltered) { + if (!liveExtsFiltered.includes(ext)) { + // Local has a real ext that live doesn't — may be due to limit or sampling + // Only fail if it's clearly wrong (not a standard installable format) + let installable = ['tar.gz', 'tar.xz', 'tar.bz2', 'zip', 'exe', 'dmg', 'pkg', 'msi', '7z', 'xz']; + if (!installable.includes(ext)) { + console.log(` FAIL ${pkg} ext: local has unexpected '${ext}'`); + failures++; + extMatch = false; + break; + } + } + } + if (extMatch) { + console.log(` PASS ${pkg} ext: ${JSON.stringify(localExtsFiltered)}`); + passes++; + } + + // Version format — no 'v' prefix + let hasVPrefix = localReleases.some(function (r) { + return r.version && r.version.startsWith('v'); + }); + if (hasVPrefix) { + console.log(` FAIL ${pkg}: versions have 'v' prefix`); + failures++; + } else { + console.log(` PASS ${pkg}: no 'v' prefix`); + passes++; + } + } + + // ================================================================ + // Test 2: Release API — filtered by OS/arch + // ================================================================ + console.log(''); + console.log('=== Test 2: Filtered /api/releases/{pkg}@stable.json?os=...&arch=... ==='); + console.log(''); + + for (let pkg of TEST_PKGS) { + for (let tc of RELEASE_API_CASES) { + let goldenName = `live_${pkg}_os_${tc.os}_arch_${tc.arch}.json`; + let liveReleases; + + if (REFRESH) { + try { + liveReleases = await fetchLiveReleases(pkg, tc.os, tc.arch, 1); + await saveGolden(goldenName, liveReleases); + } catch (e) { + skips++; + continue; + } + } else { + liveReleases = await loadGolden(goldenName); + if (!liveReleases) { + skips++; + continue; + } + } + + let liveFirst = liveReleases[0]; + if (!liveFirst || liveFirst.channel === 'error') { + skips++; + continue; + } + + let localResult = await Releases.getReleases({ + pkg: pkg, + ver: '', + os: tc.os, + arch: tc.arch, + libc: '', + lts: false, + channel: 'stable', + formats: [], + limit: 1, + }); + let localFirst = localResult.releases[0]; + + if (!localFirst || localFirst.channel === 'error') { + console.log(` FAIL ${pkg} ${tc.os}/${tc.arch}: local returned error/empty`); + failures++; + continue; + } + + let diffs = []; + // Compare os, arch, ext (skip version/download since cache age may differ) + if (liveFirst.os !== localFirst.os) { + diffs.push(`os: live=${liveFirst.os} local=${localFirst.os}`); + } + if (liveFirst.arch !== localFirst.arch) { + diffs.push(`arch: live=${liveFirst.arch} local=${localFirst.arch}`); + } + if (liveFirst.ext !== localFirst.ext) { + if (KNOWN_EXT_DIFFS.has(liveFirst.ext)) { + // Live returns a non-installable format (deb, pem, etc.) — known + console.log(` KNOWN ${pkg} ${tc.os}/${tc.arch}: live ext '${liveFirst.ext}' excluded by Go cache, local='${localFirst.ext}'`); + knowns++; + continue; + } + diffs.push(`ext: live=${liveFirst.ext} local=${localFirst.ext}`); + } + + if (diffs.length > 0) { + console.log(` FAIL ${pkg} ${tc.os}/${tc.arch}: ${diffs.join(', ')}`); + failures++; + } else { + console.log(` PASS ${pkg} ${tc.os}/${tc.arch}: v${localFirst.version} .${localFirst.ext}`); + passes++; + } + } + } + + // ================================================================ + // Test 3: Installer resolution — compare rendered script vars + // ================================================================ + console.log(''); + console.log('=== Test 3: Installer script variables (local serveInstaller vs live) ==='); + console.log(''); + + for (let pkg of ['bat', 'go', 'rg']) { + for (let tc of INSTALLER_CASES) { + // Get local result + let localVars; + try { + let script = await InstallerServer.serveInstaller( + 'https://webi.sh', + tc.ua, + pkg, + 'stable', + 'sh', + ['tar', 'exe', 'zip', 'xz', 'dmg'], + '', + ); + localVars = parseInstallerVars(script); + } catch (e) { + console.log(` ERROR ${pkg} ${tc.label}: ${e.message}`); + failures++; + continue; + } + + if (!localVars.WEBI_PKG_URL || localVars.WEBI_PKG_URL === '') { + // Check if this is a known issue + if (localVars.WEBI_EXT === 'err') { + console.log(` KNOWN ${pkg} ${tc.label}: no match (WATERFALL gap)`); + knowns++; + continue; + } + console.log(` FAIL ${pkg} ${tc.label}: empty WEBI_PKG_URL`); + failures++; + continue; + } + + // Verify the URL looks like a real download + let url = localVars.WEBI_PKG_URL; + let hasRealDomain = url.includes('github.com') || + url.includes('dl.google.com') || + url.includes('nodejs.org') || + url.includes('jqlang'); + if (!hasRealDomain) { + console.log(` FAIL ${pkg} ${tc.label}: suspicious URL: ${url}`); + failures++; + continue; + } + + // Verify version is set and has no 'v' prefix + if (!localVars.WEBI_VERSION || localVars.WEBI_VERSION === '0.0.0') { + console.log(` FAIL ${pkg} ${tc.label}: bad version: ${localVars.WEBI_VERSION}`); + failures++; + continue; + } + + // Verify ext is a real installable format + let ext = localVars.WEBI_EXT; + let goodExts = ['tar.gz', 'tar.xz', 'zip', 'exe', 'dmg', 'pkg', 'msi', '7z']; + if (!goodExts.includes(ext)) { + console.log(` FAIL ${pkg} ${tc.label}: bad ext: ${ext}`); + failures++; + continue; + } + + console.log(` PASS ${pkg} ${tc.label}: v${localVars.WEBI_VERSION} .${ext} ${url.split('/').pop()}`); + passes++; + } + } + + // ================================================================ + // Test 4: @version path-form parity (webi @) + // Exercises lts/channel/version filters that the unfiltered sweep + // doesn't touch. Two modes: + // - --cand-url= set: remote (BASE_URL) vs remote (CAND_URL). + // No local cache needed. + // - --cand-url unset: remote (BASE_URL) vs in-process Releases. + // ================================================================ + console.log(''); + if (CAND_URL) { + console.log(`=== Test 4: @version filter parity (${BASE_URL} vs ${CAND_URL}) ===`); + } else { + console.log('=== Test 4: @version filter parity (remote vs local resolver) ==='); + } + console.log(''); + + for (let tc of VERSION_SPEC_CASES) { + let label = `${tc.pkg}@${tc.spec} ${tc.os}/${tc.arch}`; + + let baseFirst; + try { + let baseRel = await fetchAtSpec(BASE_URL, tc.pkg, tc.spec, tc.os, tc.arch); + baseFirst = baseRel[0]; + } catch (e) { + console.log(` SKIP ${label}: base fetch error: ${e.message}`); + skips++; + continue; + } + if (!baseFirst || baseFirst.channel === 'error') { + console.log(` SKIP ${label}: base returned error/empty`); + skips++; + continue; + } + + let candFirst; + if (CAND_URL) { + try { + let candRel = await fetchAtSpec(CAND_URL, tc.pkg, tc.spec, tc.os, tc.arch); + candFirst = candRel[0]; + } catch (e) { + console.log(` FAIL ${label}: cand fetch error: ${e.message}`); + failures++; + continue; + } + } else { + let localResult = await Releases.getReleases({ + pkg: tc.pkg, + ver: tc.ver, + os: tc.os, + arch: tc.arch, + libc: '', + lts: tc.lts, + channel: tc.channel, + formats: [], + limit: 1, + }); + candFirst = localResult.releases && localResult.releases[0]; + } + if (!candFirst || candFirst.channel === 'error') { + console.log(` FAIL ${label}: cand returned error/empty (base=${baseFirst.version})`); + failures++; + continue; + } + + // Both should satisfy the requested spec. A version diff is only a + // real failure if cand picked something the spec excludes (e.g. + // requested '20' but got '26'). Same prefix on both sides is just + // cache-age skew (e.g. 1.21.13 vs 1.21.14). + if (baseFirst.version !== candFirst.version) { + let prefix = tc.ver; + let candMatches = !prefix || new RegExp('^' + prefix + '\\b').test(candFirst.version); + let baseMatches = !prefix || new RegExp('^' + prefix + '\\b').test(baseFirst.version); + if (!candMatches && baseMatches) { + console.log(` FAIL ${label}: cand=${candFirst.version} doesn't match spec; base=${baseFirst.version}`); + failures++; + } else if (candMatches && !baseMatches) { + console.log(` PASS ${label}: cand=${candFirst.version} (base=${baseFirst.version} is wrong)`); + passes++; + } else { + console.log(` PASS ${label}: cand=${candFirst.version} base=${baseFirst.version} (both match '${prefix}'; cache-age skew)`); + passes++; + } + } else { + console.log(` PASS ${label}: v${candFirst.version}`); + passes++; + } + } + + // ================================================================ + // Summary + // ================================================================ + console.log(''); + console.log(`=== Results: ${passes} passed, ${failures} failed, ${knowns} known, ${skips} skipped ===`); + if (failures > 0) { + process.exit(1); + } +} + +main().catch(function (err) { + console.error(err.stack); + process.exit(1); +}); diff --git a/_webi/test-live-installer-diff.js b/_webi/test-live-installer-diff.js new file mode 100644 index 0000000..482e057 --- /dev/null +++ b/_webi/test-live-installer-diff.js @@ -0,0 +1,202 @@ +'use strict'; + +// Direct side-by-side comparison: fetch the live installer script from +// a remote host with the same UA, and compare WEBI_* variables against +// what our local serveInstaller() produces. +// +// This is the most direct behavioral equivalence test — if the same UA +// produces the same WEBI_PKG_URL, WEBI_VERSION, WEBI_EXT, and WEBI_OS, +// then the user gets the same binary. +// +// Usage: +// node _webi/test-live-installer-diff.js +// node _webi/test-live-installer-diff.js --base-url=https://beta.webi.sh + +let Https = require('node:https'); +let InstallerServer = require('./serve-installer.js'); +let Builds = require('./builds.js'); + +let BASE_URL = 'https://webinstall.dev'; +for (let arg of process.argv) { + if (arg.startsWith('--base-url=')) { + BASE_URL = arg.slice('--base-url='.length).replace(/\/+$/, ''); + } +} + +let CASES = [ + // bat — Rust project, gnu-linked Linux builds + { pkg: 'bat', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'bat macOS arm64' }, + { pkg: 'bat', ua: 'x86_64/unknown Darwin/23.0.0 libc', label: 'bat macOS amd64' }, + { pkg: 'bat', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'bat Linux amd64' }, + { pkg: 'bat', ua: 'x86_64/unknown Linux/5.15.0 musl', label: 'bat Linux musl' }, + { pkg: 'bat', ua: 'x86_64/unknown Windows/10.0.19041 msvc', label: 'bat Windows amd64' }, + // go — Go project, static builds (libc='none') + { pkg: 'go', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'go macOS arm64' }, + { pkg: 'go', ua: 'x86_64/unknown Darwin/23.0.0 libc', label: 'go macOS amd64' }, + { pkg: 'go', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'go Linux amd64' }, + { pkg: 'go', ua: 'x86_64/unknown Windows/10.0.19041 msvc', label: 'go Windows amd64' }, + // node — C++ project, gnu-linked Linux builds, separate musl build + { pkg: 'node', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'node macOS arm64' }, + { pkg: 'node', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'node Linux amd64', + known: 'live fails (WATERFALL gap), local correctly resolves gnu build' }, + { pkg: 'node', ua: 'x86_64/unknown Linux/5.15.0 musl', label: 'node Linux musl' }, + // rg — Rust project, gnu-linked Linux builds + { pkg: 'rg', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'rg macOS arm64' }, + { pkg: 'rg', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'rg Linux amd64' }, + { pkg: 'rg', ua: 'x86_64/unknown Linux/5.15.0 musl', label: 'rg Linux musl' }, + { pkg: 'rg', ua: 'x86_64/unknown Windows/10.0.19041 msvc', label: 'rg Windows amd64' }, + // jq — C project, had .git source URLs in old releases + { pkg: 'jq', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'jq macOS arm64' }, + { pkg: 'jq', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'jq Linux amd64' }, + { pkg: 'jq', ua: 'x86_64/unknown Windows/10.0.19041 msvc', label: 'jq Windows amd64' }, + // caddy — Go project, had .git source URLs in old releases + { pkg: 'caddy', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'caddy macOS arm64' }, + { pkg: 'caddy', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'caddy Linux amd64' }, + { pkg: 'caddy', ua: 'x86_64/unknown Windows/10.0.19041 msvc', label: 'caddy Windows amd64' }, + // Additional packages for broader coverage + { pkg: 'shellcheck', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'shellcheck macOS arm64' }, + { pkg: 'shellcheck', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'shellcheck Linux amd64' }, + { pkg: 'shfmt', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'shfmt macOS arm64' }, + { pkg: 'shfmt', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'shfmt Linux amd64' }, + { pkg: 'fd', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'fd macOS arm64' }, + { pkg: 'fd', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'fd Linux amd64' }, + { pkg: 'hugo', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'hugo macOS arm64', + known: 'classifier rejects darwin-universal as x86_64!=universal2' }, + { pkg: 'hugo', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'hugo Linux amd64' }, + // Alias tests — these should resolve to the real package + { pkg: 'golang', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'golang alias macOS arm64' }, + { pkg: 'ripgrep', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'ripgrep alias macOS arm64' }, +]; + +// Variables that must match between live and local for the install to work +let CRITICAL_VARS = ['PKG_NAME', 'WEBI_OS', 'WEBI_ARCH', 'WEBI_EXT']; +// Variables where version differences are OK (cache age) +let VERSION_VARS = ['WEBI_VERSION', 'WEBI_PKG_URL', 'WEBI_PKG_FILE']; + +function fetchLiveInstaller(pkg, ua) { + return new Promise(function (resolve, reject) { + let url = `${BASE_URL}/api/installers/${pkg}@stable.sh`; + let opts = { headers: { 'User-Agent': ua } }; + Https.get(url, opts, function (res) { + let data = ''; + res.on('data', function (chunk) { data += chunk; }); + res.on('end', function () { resolve(data); }); + }).on('error', reject); + }); +} + +function parseVars(script) { + let vars = {}; + // Match: PKG_NAME='bat' or WEBI_VERSION='1.2.3' (with or without export) + let re = /^(?:export\s+)?(WEBI_\w+|PKG_NAME)='([^']*)'/gm; + let m; + while ((m = re.exec(script)) !== null) { + vars[m[1]] = m[2]; + } + return vars; +} + +async function main() { + let passes = 0; + let failures = 0; + let knowns = 0; + let errors = 0; + + console.log('Initializing build cache...'); + await Builds.init(); + console.log(''); + + console.log('=== Live vs Local Installer Diff ==='); + console.log(''); + + for (let tc of CASES) { + // Fetch live + let liveScript; + try { + liveScript = await fetchLiveInstaller(tc.pkg, tc.ua); + } catch (e) { + console.log(` SKIP ${tc.label}: fetch error: ${e.message}`); + continue; + } + let liveVars = parseVars(liveScript); + + if (!liveVars.WEBI_PKG_URL) { + console.log(` SKIP ${tc.label}: live returned no WEBI_PKG_URL`); + continue; + } + + // Render local + let localScript; + try { + localScript = await InstallerServer.serveInstaller( + BASE_URL, + tc.ua, + tc.pkg, + 'stable', + 'sh', + ['tar', 'exe', 'zip', 'xz', 'dmg'], + '', + ); + } catch (e) { + console.log(` ERROR ${tc.label}: local error: ${e.message}`); + errors++; + continue; + } + let localVars = parseVars(localScript); + + if (tc.known) { + let localExt = localVars.WEBI_EXT || 'err'; + let liveExt = liveVars.WEBI_EXT || '?'; + if (localExt !== liveExt) { + console.log(` KNOWN ${tc.label}: ${tc.known} (live=${liveExt} local=${localExt})`); + knowns++; + } else { + console.log(` PASS ${tc.label}: known issue resolved! v${localVars.WEBI_VERSION} .${localExt}`); + passes++; + } + continue; + } + + if (!localVars.WEBI_PKG_URL || localVars.WEBI_EXT === 'err') { + console.log(` KNOWN ${tc.label}: local failed to resolve (live=${liveVars.WEBI_EXT})`); + knowns++; + continue; + } + + // Compare critical vars + let diffs = []; + for (let v of CRITICAL_VARS) { + let liveVal = liveVars[v] || ''; + let localVal = localVars[v] || ''; + if (liveVal !== localVal) { + diffs.push(`${v}: live='${liveVal}' local='${localVal}'`); + } + } + + // Log version info (informational, not failure) + let versionNote = ''; + if (liveVars.WEBI_VERSION !== localVars.WEBI_VERSION) { + versionNote = ` (version: live=${liveVars.WEBI_VERSION} local=${localVars.WEBI_VERSION})`; + } + + if (diffs.length > 0) { + console.log(` FAIL ${tc.label}: ${diffs.join(', ')}${versionNote}`); + failures++; + } else { + let file = (localVars.WEBI_PKG_URL || '').split('/').pop(); + console.log(` PASS ${tc.label}: v${localVars.WEBI_VERSION} .${localVars.WEBI_EXT} ${file}${versionNote}`); + passes++; + } + } + + console.log(''); + console.log(`=== Results: ${passes} passed, ${failures} failed, ${knowns} known, ${errors} errors ===`); + if (failures > 0 || errors > 0) { + process.exit(1); + } +} + +main().catch(function (err) { + console.error(err.stack); + process.exit(1); +});