From 0b9ea25fbebbb303f17dc03fed2f45f68517b46e Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Wed, 11 Mar 2026 02:54:29 -0600 Subject: [PATCH] fix: resolve triplet/version/libc issues in builds-cacher MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three fixes to the installer resolution path: 1. Triplet enumeration order: put specific OS before ANYOS and specific arch before ANYARCH. This prevents .git source URLs (ANYOS/ANYARCH) from shadowing platform-specific binaries (jq, caddy). 2. WATERFALL libc→gnu+musl: glibc hosts ('libc' in UA) now try ['none', 'gnu', 'musl', 'libc'] instead of ['none', 'libc']. This lets packages with gnu-linked builds (bat, rg, node) and static musl builds (rg v15+) match for glibc hosts. 3. Version-first iteration: findMatchingPackages now iterates versions newest-first then tries each triplet, matching the Go resolver's approach. Prevents picking an ancient version from a low-priority triplet when a newer version exists in another triplet (e.g. rg v0.1.6 gnu vs v15.1.0 musl). Tests updated: all 15 installer-resolve cases pass (was 9+6 known), 30/32 live-diff cases pass (2 known: node Linux amd64 where our code is better than live, hugo macOS arm64 classifier limitation). --- _webi/builds-cacher.js | 57 ++++++++++++++++++------------- _webi/test-installer-resolve.js | 46 ++++++++++++++++--------- _webi/test-live-compare.js | 2 +- _webi/test-live-installer-diff.js | 30 +++++++++++++--- 4 files changed, 90 insertions(+), 45 deletions(-) diff --git a/_webi/builds-cacher.js b/_webi/builds-cacher.js index 1b2b8a5..bc71f9a 100644 --- a/_webi/builds-cacher.js +++ b/_webi/builds-cacher.js @@ -525,29 +525,21 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) { return null; } - for (let _triplet of triplets) { - let targetReleases = projInfo.releasesByTriplet[_triplet]; - if (!targetReleases) { - continue; - } + // Iterate versions newest-first, then try each triplet for that + // version. This matches the Go resolver's behavior and ensures we + // get the latest version even if the preferred triplet only has old + // releases (e.g. rg dropped x86_64-linux-gnu in v15, so the gnu + // triplet only has v0.1.6 — version-first finds v15 via musl). + for (let lexver of projInfo.lexvers) { + let ver = projInfo.lexversMap[lexver] || lexver; - let versions = Object.keys(targetReleases); - //console.log('dbg: targetRelease versions', versions); - let lexvers = []; - for (let version of versions) { - let lexPrefix = Lexver.parseVersion(version); - lexvers.push(lexPrefix); - } - lexvers.sort(); - lexvers.reverse(); - // TODO get the other matchInfo props + for (let _triplet of triplets) { + let targetReleases = projInfo.releasesByTriplet[_triplet]; + if (!targetReleases) { + continue; + } - // Make sure that these releases are the expected version - // (ex: jq1.7 => darwin-arm64-libc, jq1.6 => darwin-x86_64-libc) - for (let matchver of lexvers) { - let ver = projInfo.lexversMap[matchver] || matchver; let packages = targetReleases[ver]; - //console.log('dbg: packages', packages); if (!packages) { continue; } @@ -598,22 +590,39 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) { return triplets; } + // Prefer platform-specific matches over ANYOS/ANYARCH fallbacks. + // This ensures that e.g. darwin-aarch64-none matches before + // ANYOS-ANYARCH-none (.git source URLs from old releases). let oses = []; if (hostTarget.os === 'windows') { - oses = ['ANYOS', 'windows']; + oses = ['windows', 'ANYOS']; } else if (hostTarget.os === 'android') { - oses = ['ANYOS', 'posix_2017', 'posix_2024', 'android', 'linux']; + oses = ['android', 'linux', 'posix_2017', 'posix_2024', 'ANYOS']; } else { - oses = ['ANYOS', 'posix_2017', 'posix_2024', hostTarget.os]; + oses = [hostTarget.os, 'posix_2017', 'posix_2024', 'ANYOS']; } let waterfall = HostTargets.WATERFALL[hostTarget.os] || {}; let arches = waterfall[hostTarget.arch] || HostTargets.WATERFALL.ANYOS[hostTarget.arch] || [hostTarget.arch]; - arches = ['ANYARCH'].concat(arches); + arches = arches.concat(['ANYARCH']); let libcs = waterfall[hostTarget.libc] || HostTargets.WATERFALL.ANYOS[hostTarget.libc] || [hostTarget.libc]; + // The WATERFALL maps 'libc' (host glibc) => ['none', 'libc'] but + // never tries 'gnu' or 'musl'. Packages with glibc-linked builds + // (Rust projects like bat, rg; also node) are tagged libc='gnu' in + // the Go cache. Static musl builds (Rust x86_64-unknown-linux-musl) + // also work on glibc hosts. + if (hostTarget.libc === 'libc' && !libcs.includes('gnu')) { + // none = static (preferred, zero deps) + // gnu = glibc-linked (glibc host can run these) + // musl = musl-linked (often static for Rust; node-musl is an exception + // but version-first iteration prevents picking it over gnu) + // libc = legacy fallback (rarely used in cache metadata) + libcs = ['none', 'gnu', 'musl', 'libc']; + } + for (let os of oses) { for (let arch of arches) { for (let libc of libcs) { diff --git a/_webi/test-installer-resolve.js b/_webi/test-installer-resolve.js index f31d8b0..b84013d 100644 --- a/_webi/test-installer-resolve.js +++ b/_webi/test-installer-resolve.js @@ -102,46 +102,60 @@ let UA_CASES = [ expectArch: 'x86_64', expectExt: 'tar.gz', }, - // === Known: WATERFALL libc vs gnu === - // Packages whose Go cache has libc='gnu' for Linux glibc builds. - // The WATERFALL maps `libc` => ['none', 'libc'] but never tries 'gnu'. + // === Linux glibc — packages with libc='gnu' in cache === + // These previously failed (WATERFALL libc→gnu gap). Fixed by adding + // 'gnu' to the libc candidates for glibc hosts in _enumerateTriplets. { - label: 'bat Linux amd64 (known: WATERFALL libc vs gnu)', + label: 'bat Linux amd64', pkg: 'bat', ua: 'x86_64/unknown Linux/5.15.0 libc', - known: 'WATERFALL libc->gnu gap', + expectOs: 'linux', + expectArch: 'x86_64', + expectExt: 'tar.gz', }, { - label: 'rg Linux amd64 (known: WATERFALL libc vs gnu)', + label: 'rg Linux amd64', pkg: 'rg', ua: 'x86_64/unknown Linux/5.15.0 libc', - known: 'WATERFALL libc->gnu gap', + expectOs: 'linux', + expectArch: 'x86_64', + expectExt: 'tar.gz', }, { - label: 'node Linux amd64 (known: WATERFALL libc vs gnu)', + label: 'node Linux amd64', pkg: 'node', ua: 'x86_64/unknown Linux/5.15.0 libc', - known: 'WATERFALL libc->gnu gap', + expectOs: 'linux', + expectArch: 'x86_64', + expectExt: 'tar.xz', }, - // === Known: ANYOS/ANYARCH .git priority === + // === Packages with .git source URLs in old releases === + // These previously failed (ANYOS .git matched before platform binary). + // Fixed by putting specific OS before ANYOS in triplet enumeration. { - label: 'jq macOS arm64 (known: ANYOS .git priority)', + label: 'jq macOS arm64', pkg: 'jq', ua: 'aarch64/unknown Darwin/24.2.0 libc', - known: 'ANYOS .git matched before platform binary', + expectOs: 'darwin', + expectArch: 'aarch64', + expectExt: 'exe', }, { - label: 'caddy macOS arm64 (known: ANYOS .git priority)', + label: 'caddy macOS arm64', pkg: 'caddy', ua: 'aarch64/unknown Darwin/24.2.0 libc', - known: 'ANYOS .git matched before platform binary', + expectOs: 'darwin', + expectArch: 'aarch64', + expectExt: 'tar.gz', }, { - label: 'caddy Linux amd64 (known: ANYOS .git priority)', + label: 'caddy Linux amd64', pkg: 'caddy', ua: 'x86_64/unknown Linux/5.15.0 libc', - known: 'ANYOS .git matched before platform binary', + expectOs: 'linux', + expectArch: 'x86_64', + expectExt: 'tar.gz', }, ]; diff --git a/_webi/test-live-compare.js b/_webi/test-live-compare.js index acf305c..4cd6c15 100644 --- a/_webi/test-live-compare.js +++ b/_webi/test-live-compare.js @@ -292,7 +292,7 @@ async function main() { libc: '', lts: false, channel: 'stable', - formats: ['tar', 'zip', 'exe', 'xz'], + formats: [], limit: 1, }); let localFirst = localResult.releases[0]; diff --git a/_webi/test-live-installer-diff.js b/_webi/test-live-installer-diff.js index 3e66e1a..238f1a1 100644 --- a/_webi/test-live-installer-diff.js +++ b/_webi/test-live-installer-diff.js @@ -16,23 +16,45 @@ let InstallerServer = require('./serve-installer.js'); let Builds = require('./builds.js'); let CASES = [ + // bat — Rust project, gnu-linked Linux builds { pkg: 'bat', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'bat macOS arm64' }, { pkg: 'bat', ua: 'x86_64/unknown Darwin/23.0.0 libc', label: 'bat macOS amd64' }, + { pkg: 'bat', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'bat Linux amd64' }, { pkg: 'bat', ua: 'x86_64/unknown Linux/5.15.0 musl', label: 'bat Linux musl' }, { pkg: 'bat', ua: 'x86_64/unknown Windows/10.0.19041 msvc', label: 'bat Windows amd64' }, + // go — Go project, static builds (libc='none') { pkg: 'go', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'go macOS arm64' }, { pkg: 'go', ua: 'x86_64/unknown Darwin/23.0.0 libc', label: 'go macOS amd64' }, { pkg: 'go', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'go Linux amd64' }, { pkg: 'go', ua: 'x86_64/unknown Windows/10.0.19041 msvc', label: 'go Windows amd64' }, + // node — C++ project, gnu-linked Linux builds, separate musl build { pkg: 'node', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'node macOS arm64' }, + { pkg: 'node', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'node Linux amd64', + known: 'live fails (WATERFALL gap), local correctly resolves gnu build' }, { pkg: 'node', ua: 'x86_64/unknown Linux/5.15.0 musl', label: 'node Linux musl' }, + // rg — Rust project, gnu-linked Linux builds { pkg: 'rg', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'rg macOS arm64' }, + { pkg: 'rg', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'rg Linux amd64' }, { pkg: 'rg', ua: 'x86_64/unknown Linux/5.15.0 musl', label: 'rg Linux musl' }, { pkg: 'rg', ua: 'x86_64/unknown Windows/10.0.19041 msvc', label: 'rg Windows amd64' }, - { pkg: 'jq', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'jq macOS arm64', - known: 'Go cache .git regression' }, - { pkg: 'jq', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'jq Linux amd64', - known: 'Go cache .git regression' }, + // jq — C project, had .git source URLs in old releases + { pkg: 'jq', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'jq macOS arm64' }, + { pkg: 'jq', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'jq Linux amd64' }, + { pkg: 'jq', ua: 'x86_64/unknown Windows/10.0.19041 msvc', label: 'jq Windows amd64' }, + // caddy — Go project, had .git source URLs in old releases + { pkg: 'caddy', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'caddy macOS arm64' }, + { pkg: 'caddy', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'caddy Linux amd64' }, + { pkg: 'caddy', ua: 'x86_64/unknown Windows/10.0.19041 msvc', label: 'caddy Windows amd64' }, + // Additional packages for broader coverage + { pkg: 'shellcheck', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'shellcheck macOS arm64' }, + { pkg: 'shellcheck', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'shellcheck Linux amd64' }, + { pkg: 'shfmt', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'shfmt macOS arm64' }, + { pkg: 'shfmt', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'shfmt Linux amd64' }, + { pkg: 'fd', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'fd macOS arm64' }, + { pkg: 'fd', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'fd Linux amd64' }, + { pkg: 'hugo', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'hugo macOS arm64', + known: 'classifier rejects darwin-universal as x86_64!=universal2' }, + { pkg: 'hugo', ua: 'x86_64/unknown Linux/5.15.0 libc', label: 'hugo Linux amd64' }, // Alias tests — these should resolve to the real package { pkg: 'golang', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'golang alias macOS arm64' }, { pkg: 'ripgrep', ua: 'aarch64/unknown Darwin/24.2.0 libc', label: 'ripgrep alias macOS arm64' },