github/vim-ale
1
0
Fork 0
mirror of https://github.com/webinstall/webi-installers.git synced 2026-04-06 18:36:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(legacy): translate universal2→x86_64, keep solaris/illumos, add minor arch fixes

Browse Source 
Correct the previous commit's two mistakes (3655ef3):

1. universal2/universal1: translate arch to x86_64 instead of dropping.
   The Node classifier maps 'universal' in filename to x86_64, so the cache
   must say x86_64 to match. The darwin WATERFALL (aarch64→[aarch64,x86_64])
   ensures arm64 users still receive these builds as a fallback.
   Previously dropping these caused cmake/hugo macOS to resolve to old versions.

2. solaris/illumos: keep as-is instead of dropping.
   The build-classifier (triplet.js) recognizes solaris, illumos, and sunos as
   distinct values and matches them correctly. ed5239a was right all along.

Also add minor arch translations:
- x86_64_v2/v3/v4 → x86_64: classifier doesn't recognize micro-arch suffixes
- mips64r6/mips64r6el → mips64: MIPS Release 6 variants
This commit is contained in:
AJ ONeal
2026-03-11 16:50:19 -06:00
parent cbda77a44d
commit 3756bd8e05
2 changed files with 154 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
internal/storage/legacy.go
View File

@@ -32,11 +32,9 @@ type LegacyCache struct {
// LegacyDropStats reports how many assets were excluded during ExportLegacy.
type LegacyDropStats struct {
Variants int // dropped: has build variant tags (e.g. rocm, installer, fxdependent)
Formats int // dropped: format not recognized by the Node.js server
Universal int // dropped: universal2/universal1 arch — classifier maps "universal" in filename to x86_64 and rejects the mismatch
SunOS int // dropped: solaris/illumos OS — Node never served these; classifier mismatches are unfixable
Android int // dropped: android OS — classifier maps android filenames to linux
Variants int // dropped: has build variant tags (e.g. rocm, installer, fxdependent)
Formats int // dropped: format not recognized by the Node.js server
Android int // dropped: android OS — classifier maps android filenames to linux
}
// ToAsset converts a LegacyAsset to the internal Asset type.
@@ -76,13 +74,41 @@ func (a Asset) toLegacy() LegacyAsset {
// values the legacy Node.js resolver expects. This is called at export time
// only — the canonical values are preserved in Go-native storage (pgstore).
//
// Global rules (all packages):
// - ARM arch: translated from Go canonical to the value the Node build-classifier
// extracts from the filename (see legacyARMArchFromFilename).
// The Node build-classifier re-parses each asset's download filename and drops
// any entry where the cache field doesn't match what it extracts from the name.
// These translations ensure the cache matches the classifier's extraction.
//
// Global arch translations (all packages):
// - universal2/universal1 → x86_64: classifier maps "universal" in filename
// to x86_64. The darwin WATERFALL falls back aarch64→x86_64, so arm64
// users still receive these builds.
// - x86_64_v2 → x86_64: classifier doesn't recognize micro-arch level suffixes.
// - mips64r6/mips64r6el → mips64: MIPS Release 6 variants map to the base arch.
// - ARM (filename-based): gnueabihf/armhf→armhf, armel→armel, armv5→armel,
// armv7a→armv7a. Go normalizes these; Node classifier preserves the
// original Debian/Rust naming. See legacyARMArchFromFilename.
//
// Note: solaris/illumos/sunos are kept as-is. The build-classifier (triplet.js)
// recognizes all three as distinct values, and the live cache uses them directly.
//
// Package-specific rules replicate per-package overrides in production's releases.js:
// - ffmpeg: Windows .gz → .exe (prod releases.js: rel.ext = 'exe')
func legacyFieldBackport(pkg string, a Asset) Asset {
// Universal fat binaries: classifier maps "universal" in filename to x86_64.
if a.Arch == "universal2" || a.Arch == "universal1" {
a.Arch = "x86_64"
}
// x86_64 micro-arch levels: classifier doesn't know these suffixes.
if a.Arch == "x86_64_v2" || a.Arch == "x86_64_v3" || a.Arch == "x86_64_v4" {
a.Arch = "x86_64"
}
// MIPS Release 6 variants: map to the base mips64 arch.
if a.Arch == "mips64r6" || a.Arch == "mips64r6el" {
a.Arch = "mips64"
}
// ARM arch: the Node classifier re-parses filenames and expects the cache
// arch to match what it extracts. Go normalizes (gnueabihf→armv6, armhf→armv7)
// but the Node classifier preserves the original Debian/Rust naming.
@@ -166,8 +192,6 @@ var legacyFormats = map[string]bool{
// The pkg name is used to apply per-package field translations (see legacyFieldBackport).
// Assets are excluded when:
// - Variants is non-empty (Node.js has no variant logic)
// - Arch is universal2 or universal1 (classifier maps "universal" in filename to x86_64 and rejects the mismatch)
// - OS is solaris or illumos (Node never served these; classifier mismatches are unfixable)
// - OS is android (classifier maps android filenames to linux)
// - Format is non-empty and not in the Node.js recognized set
//
@@ -182,26 +206,13 @@ func ExportLegacy(pkg string, pd PackageData) (LegacyCache, LegacyDropStats) {
stats.Variants++
continue
}
// Skip universal fat binaries — classifier maps "universal" in filename
// to x86_64 and rejects any cache entry that doesn't say x86_64.
if a.Arch == "universal2" || a.Arch == "universal1" {
stats.Universal++
continue
}
// Skip solaris/illumos — Node never served these platforms;
// the classifier causes mismatches that can't be fixed without
// changing the filename.
if a.OS == "solaris" || a.OS == "illumos" {
stats.SunOS++
continue
}
// Skip android — classifier maps android filenames to linux OS,
// which mismatches cache entries tagged android.
if a.OS == "android" {
stats.Android++
continue
}
// Apply per-package legacy field translations before format check.
// Apply per-package and global legacy field translations.
a = legacyFieldBackport(pkg, a)
// Skip formats Node.js doesn't recognize.
if a.Format != "" && !legacyFormats[a.Format] {

191
internal/storage/legacy_test.go
View File

@@ -110,61 +110,6 @@ func TestExportLegacyDrops(t *testing.T) {
}
})
t.Run("universal2_dropped", func(t *testing.T) {
// universal2 fat binaries are dropped. The Node classifier maps "universal"
// in the filename to x86_64, then rejects the cache entry because it says
// "universal2". The mismatch can't be fixed without changing the filename.
pd := storage.PackageData{
Assets: []storage.Asset{
{Filename: "hugo_0.145.0_darwin-universal.tar.gz", OS: "darwin", Arch: "universal2", Format: ".tar.gz"},
{Filename: "hugo_0.145.0_darwin-arm64.tar.gz", OS: "darwin", Arch: "aarch64", Format: ".tar.gz"},
},
}
lc, stats := storage.ExportLegacy("hugo", pd)
if stats.Universal != 1 {
t.Errorf("Universal dropped = %d, want 1", stats.Universal)
}
if len(lc.Releases) != 1 {
t.Errorf("releases = %d, want 1 (arm64 only)", len(lc.Releases))
}
if len(lc.Releases) > 0 && lc.Releases[0].Arch != "aarch64" {
t.Errorf("kept arch = %q, want aarch64", lc.Releases[0].Arch)
}
})
t.Run("solaris_dropped", func(t *testing.T) {
// Solaris entries are dropped: Node never served these platforms and the
// classifier produces unfixable mismatches for them.
pd := storage.PackageData{
Assets: []storage.Asset{
{Filename: "go1.20.1.solaris-amd64.tar.gz", OS: "solaris", Arch: "x86_64", Format: ".tar.gz"},
{Filename: "go1.20.1.linux-amd64.tar.gz", OS: "linux", Arch: "x86_64", Format: ".tar.gz"},
},
}
lc, stats := storage.ExportLegacy("go", pd)
if stats.SunOS != 1 {
t.Errorf("SunOS dropped = %d, want 1", stats.SunOS)
}
if len(lc.Releases) != 1 {
t.Errorf("releases = %d, want 1 (linux only)", len(lc.Releases))
}
})
t.Run("illumos_dropped", func(t *testing.T) {
pd := storage.PackageData{
Assets: []storage.Asset{
{Filename: "go1.20.1.illumos-amd64.tar.gz", OS: "illumos", Arch: "x86_64", Format: ".tar.gz"},
},
}
lc, stats := storage.ExportLegacy("go", pd)
if stats.SunOS != 1 {
t.Errorf("SunOS dropped = %d, want 1", stats.SunOS)
}
if len(lc.Releases) != 0 {
t.Errorf("releases = %d, want 0", len(lc.Releases))
}
})
t.Run("android_dropped", func(t *testing.T) {
// Android entries are dropped: the classifier maps android filenames to
// linux OS and then rejects the cache entry that says android.
@@ -222,6 +167,104 @@ func TestExportLegacyDrops(t *testing.T) {
// TestExportLegacyTranslations verifies that legacyFieldBackport applies the
// correct field translations for Node.js compatibility.
func TestExportLegacyTranslations(t *testing.T) {
t.Run("universal2_translated_to_x86_64", func(t *testing.T) {
// universal2 fat binaries: the Node classifier sees "universal" in the
// filename and maps it to x86_64. Cache must say x86_64 to match.
// The darwin WATERFALL (aarch64 → [aarch64, x86_64]) means arm64 users
// also receive these builds as a fallback.
pd := storage.PackageData{
Assets: []storage.Asset{
{Filename: "hugo_0.145.0_darwin-universal.tar.gz", OS: "darwin", Arch: "universal2", Format: ".tar.gz"},
{Filename: "hugo_0.145.0_darwin-arm64.tar.gz", OS: "darwin", Arch: "aarch64", Format: ".tar.gz"},
},
}
lc, stats := storage.ExportLegacy("hugo", pd)
if stats.Variants != 0 || stats.Formats != 0 || stats.Android != 0 {
t.Errorf("unexpected drops: %+v", stats)
}
if len(lc.Releases) != 2 {
t.Fatalf("releases = %d, want 2", len(lc.Releases))
}
var universal2Arch string
for _, r := range lc.Releases {
if r.Name == "hugo_0.145.0_darwin-universal.tar.gz" {
universal2Arch = r.Arch
}
}
if universal2Arch != "x86_64" {
t.Errorf("universal2 arch in legacy = %q, want x86_64", universal2Arch)
}
})
t.Run("solaris_kept_as_is", func(t *testing.T) {
// Solaris/illumos/sunos are kept as-is. The build-classifier (triplet.js)
// recognizes all three as distinct values and matches them correctly.
pd := storage.PackageData{
Assets: []storage.Asset{
{Filename: "go1.20.1.solaris-amd64.tar.gz", OS: "solaris", Arch: "x86_64", Format: ".tar.gz"},
},
}
lc, stats := storage.ExportLegacy("go", pd)
if stats.Android != 0 || stats.Variants != 0 || stats.Formats != 0 {
t.Errorf("unexpected drops: %+v", stats)
}
if len(lc.Releases) != 1 {
t.Fatalf("releases = %d, want 1", len(lc.Releases))
}
if lc.Releases[0].OS != "solaris" {
t.Errorf("OS = %q, want solaris", lc.Releases[0].OS)
}
})
t.Run("illumos_kept_as_is", func(t *testing.T) {
pd := storage.PackageData{
Assets: []storage.Asset{
{Filename: "go1.20.1.illumos-amd64.tar.gz", OS: "illumos", Arch: "x86_64", Format: ".tar.gz"},
},
}
lc, _ := storage.ExportLegacy("go", pd)
if len(lc.Releases) != 1 {
t.Fatalf("releases = %d, want 1", len(lc.Releases))
}
if lc.Releases[0].OS != "illumos" {
t.Errorf("OS = %q, want illumos", lc.Releases[0].OS)
}
})
t.Run("x86_64_v2_translated_to_x86_64", func(t *testing.T) {
// Micro-arch level suffixes (v2/v3/v4) are not recognized by the classifier.
pd := storage.PackageData{
Assets: []storage.Asset{
{Filename: "tool-linux-x86_64_v2.tar.gz", OS: "linux", Arch: "x86_64_v2", Format: ".tar.gz"},
},
}
lc, _ := storage.ExportLegacy("tool", pd)
if len(lc.Releases) != 1 {
t.Fatalf("releases = %d, want 1", len(lc.Releases))
}
if lc.Releases[0].Arch != "x86_64" {
t.Errorf("arch = %q, want x86_64", lc.Releases[0].Arch)
}
})
t.Run("mips64r6_translated_to_mips64", func(t *testing.T) {
pd := storage.PackageData{
Assets: []storage.Asset{
{Filename: "tool-linux-mips64r6.tar.gz", OS: "linux", Arch: "mips64r6", Format: ".tar.gz"},
{Filename: "tool-linux-mips64r6el.tar.gz", OS: "linux", Arch: "mips64r6el", Format: ".tar.gz"},
},
}
lc, _ := storage.ExportLegacy("tool", pd)
if len(lc.Releases) != 2 {
t.Fatalf("releases = %d, want 2", len(lc.Releases))
}
for _, r := range lc.Releases {
if r.Arch != "mips64" {
t.Errorf("arch = %q, want mips64 (mips64r6* → mips64)", r.Arch)
}
}
})
t.Run("ffmpeg_windows_gz_to_exe", func(t *testing.T) {
// ffmpeg Windows releases are .gz archives containing a bare .exe.
// Production releases.js overrides ext to 'exe' for install compatibility.
@@ -242,12 +285,11 @@ func TestExportLegacyTranslations(t *testing.T) {
}
}
if windowsExt != ".exe" {
t.Errorf("ffmpeg windows ext = %q, want %q", windowsExt, ".exe")
t.Errorf("ffmpeg windows ext = %q, want .exe", windowsExt)
}
})
t.Run("ffmpeg_translation_not_applied_to_other_packages", func(t *testing.T) {
// The ffmpeg .gz→.exe translation is package-specific.
pd := storage.PackageData{
Assets: []storage.Asset{
{Filename: "othertool-windows-amd64.gz", OS: "windows", Arch: "x86_64", Format: ".gz"},
@@ -258,7 +300,7 @@ func TestExportLegacyTranslations(t *testing.T) {
t.Fatalf("releases = %d, want 1", len(lc.Releases))
}
if lc.Releases[0].Ext != ".gz" {
t.Errorf("ext = %q, want %q (no translation outside ffmpeg)", lc.Releases[0].Ext, ".gz")
t.Errorf("ext = %q, want .gz (no translation outside ffmpeg)", lc.Releases[0].Ext)
}
})
@@ -353,7 +395,7 @@ func TestExportLegacyTranslations(t *testing.T) {
})
t.Run("arm_armv7l_unchanged", func(t *testing.T) {
// armv7l in filename: Go=armv7, Node classifier also extracts armv7. No translation.
// armv7l: Go=armv7, Node classifier also extracts armv7. No translation.
pd := storage.PackageData{
Assets: []storage.Asset{
{Filename: "tool-armv7l-linux.tar.gz", OS: "linux", Arch: "armv7", Format: ".tar.gz"},
@@ -369,7 +411,7 @@ func TestExportLegacyTranslations(t *testing.T) {
})
t.Run("arm_armv6l_unchanged", func(t *testing.T) {
// armv6l in filename: Go=armv6, Node classifier also extracts armv6. No translation.
// armv6l: Go=armv6, Node classifier also extracts armv6. No translation.
pd := storage.PackageData{
Assets: []storage.Asset{
{Filename: "tool-armv6l-linux.tar.gz", OS: "linux", Arch: "armv6", Format: ".tar.gz"},
@@ -394,14 +436,14 @@ func TestExportLegacyMixed(t *testing.T) {
{Filename: "tool-linux-amd64.tar.gz", OS: "linux", Arch: "x86_64", Format: ".tar.gz"},
// dropped: variant build
{Filename: "tool-linux-amd64-rocm.tar.gz", OS: "linux", Arch: "x86_64", Format: ".tar.gz", Variants: []string{"rocm"}},
// dropped: universal2
{Filename: "tool-darwin-universal.tar.gz", OS: "darwin", Arch: "universal2", Format: ".tar.gz"},
// dropped: illumos
{Filename: "tool-illumos-amd64.tar.gz", OS: "illumos", Arch: "x86_64", Format: ".tar.gz"},
// dropped: android
{Filename: "tool-android-arm64.tar.gz", OS: "android", Arch: "aarch64", Format: ".tar.gz"},
// dropped: .AppImage format
{Filename: "tool.AppImage", OS: "linux", Format: ".AppImage"},
// kept (translated): universal2 → x86_64
{Filename: "tool-darwin-universal.tar.gz", OS: "darwin", Arch: "universal2", Format: ".tar.gz"},
// kept: solaris as-is
{Filename: "tool-solaris-amd64.tar.gz", OS: "solaris", Arch: "x86_64", Format: ".tar.gz"},
},
}
lc, stats := storage.ExportLegacy("tool", pd)
@@ -409,19 +451,24 @@ func TestExportLegacyMixed(t *testing.T) {
if stats.Variants != 1 {
t.Errorf("Variants = %d, want 1", stats.Variants)
}
if stats.Universal != 1 {
t.Errorf("Universal = %d, want 1", stats.Universal)
}
if stats.SunOS != 1 {
t.Errorf("SunOS = %d, want 1", stats.SunOS)
}
if stats.Android != 1 {
t.Errorf("Android = %d, want 1", stats.Android)
}
if stats.Formats != 1 {
t.Errorf("Formats = %d, want 1", stats.Formats)
}
if len(lc.Releases) != 1 {
t.Errorf("releases = %d, want 1 (linux only)", len(lc.Releases))
if len(lc.Releases) != 3 {
t.Errorf("releases = %d, want 3 (linux + darwin/x86_64 + solaris)", len(lc.Releases))
}
// Verify universal2 was translated to x86_64.
var darwinArch string
for _, r := range lc.Releases {
if r.OS == "darwin" {
darwinArch = r.Arch
}
}
if darwinArch != "x86_64" {
t.Errorf("darwin arch = %q, want x86_64 (translated from universal2)", darwinArch)
}
}