Compare commits

..

5 Commits

Author SHA1 Message Date
AJ ONeal
f66822295b chore: go mod tidy 2026-05-16 21:30:59 -06:00
AJ ONeal
c538942392 chore(scripts): shellcheck + shfmt clean deploy-webicached.sh 2026-05-16 21:22:38 -06:00
AJ ONeal
af28ddb686 docs: add deploy scripts, skills, and pattern guides
Deploy scripts for webicached and webid (build, upload, restart).
AGENTS.md with releases.conf reference and variant tagging docs.
Installer archive pattern guide and version oddities reference.
2026-05-16 21:22:38 -06:00
AJ ONeal
631147901a feat: add Go release cache daemon (webicached)
Rewrites the Node.js release classification pipeline in Go. webicached
fetches upstream releases (GitHub, Gitea, GitLab, HashiCorp, custom
sources), classifies assets by OS/arch/variant, and writes legacy-format
JSON caches compatible with the existing webinstall.dev API.

Git-clone packages emit git_tag and git_commit_hash from real repo
clones — no fabricated refs.
2026-05-16 21:22:38 -06:00
AJ ONeal
b3375d0e24 fix: serve Windows packages to CYGWIN and MINGW user-agents
CYGWIN_NT-* and MINGW64_NT-* UAs (Git Bash / Cygwin on Windows) were
classified as linux, so Windows users got linux binaries or no match.

Three fixes:
- build-classifier v1.0.4: CYGWIN/MINGW → windows in termsToTarget
- ua-detect.js: same fix for the Node server's UA detection path
- builds-cacher.js: default hostTarget.libc to 'libc' when unset —
  termsToTarget omits libc for plain UAs, causing triplets like
  'linux-x86_64-undefined' that never matched cache entries
2026-05-14 17:06:06 -06:00
30 changed files with 20 additions and 8584 deletions

View File

@@ -621,13 +621,15 @@ BuildsCacher.create = function ({ ALL_TERMS, installers }) {
let arches = waterfall[hostTarget.arch] ||
HostTargets.WATERFALL.ANYOS[hostTarget.arch] || [hostTarget.arch];
arches = arches.concat(['ANYARCH']);
let libcs = waterfall[hostTarget.libc] ||
HostTargets.WATERFALL.ANYOS[hostTarget.libc] || [hostTarget.libc];
// termsToTarget omits libc for plain UAs; 'libc' → waterfall ['none','libc',...]
let libc = hostTarget.libc || 'libc';
let libcs = waterfall[libc] ||
HostTargets.WATERFALL.ANYOS[libc] || [libc];
// Extend the glibc-host waterfall: the table only lists [none, libc]
// but Rust projects (bat, rg) and node ship libc='gnu' builds, and
// static musl builds also run on glibc hosts.
if (hostTarget.libc === 'libc' && !libcs.includes('gnu')) {
if (libc === 'libc' && !libcs.includes('gnu')) {
libcs = ['none', 'gnu', 'musl', 'libc'];
}

View File

@@ -38,9 +38,8 @@ function getOs(ua) {
// It's the year of the Linux Desktop!
// See also http://www.mslinux.org/
// 'linux' must be tested before 'Microsoft' because WSL
// (TODO: does this affect cygwin / msysgit?)
return 'linux';
} else if (/^ms$|Microsoft|Windows|win32|win|PowerShell/i.test(ua)) {
} else if (/^ms$|Microsoft|Windows|win32|win|PowerShell|CYGWIN|MINGW/i.test(ua)) {
// 'win' must be tested after 'darwin'
return 'windows';
} else if (/Linux|curl|wget/i.test(ua)) {

File diff suppressed because it is too large Load Diff

View File

@@ -1,914 +0,0 @@
// Command comparecache compares Go-generated cache output against the
// Node.js LIVE_cache. It identifies categorical differences in asset
// selection — which filenames appear in one cache but not the other.
//
// The comparison is done at the filename level (not OS/arch/ext fields)
// because the Node.js cache leaves those empty (normalize.js fills them
// at serve time), while the Go pipeline classifies at write time.
//
// Usage:
//
// go run ./cmd/comparecache -live ./LIVE_cache -go ./_cache
// go run ./cmd/comparecache -live ./LIVE_cache -go ./_cache bat jq
// go run ./cmd/comparecache -live ./LIVE_cache -go ./_cache -summary
package main
import (
"encoding/json"
"flag"
"fmt"
"log"
"math/rand/v2"
"os"
"path/filepath"
"slices"
"sort"
"strings"
"time"
"github.com/webinstall/webi-installers/internal/classify"
"github.com/webinstall/webi-installers/internal/lexver"
)
type cacheEntry struct {
Releases []cacheRelease `json:"releases"`
}
type cacheRelease struct {
Name string `json:"name"`
Filename string `json:"_filename"` // Node.js uses _filename for some sources
Version string `json:"version"`
Download string `json:"download"`
Channel string `json:"channel"`
OS string `json:"os"`
Arch string `json:"arch"`
Libc string `json:"libc"`
Ext string `json:"ext"`
}
// fieldDiff records a field-level difference for an asset that exists
// in both caches (same filename) but has different classification.
type fieldDiff struct {
Filename string
Field string // "os", "arch", "libc", "ext", "channel"
Live string
Go string
BothSet bool // true when both live and go have non-empty values
}
type packageDiff struct {
Name string
LiveCount int
GoCount int
OnlyInLive []string // filenames only in Node.js cache
OnlyInGo []string // filenames only in Go cache
FieldDiffs []fieldDiff // classification differences on shared assets
VersionsLive []string // unique versions in live
VersionsGo []string // unique versions in go
GoMissing bool // true if Go didn't produce output for this package
LiveMissing bool // true if no live cache for this package
Categories []string // categorical difference labels
}
func main() {
liveDir := flag.String("live", "./LIVE_cache", "path to Node.js LIVE_cache directory")
goDir := flag.String("go", "./_cache", "path to Go cache directory")
summary := flag.Bool("summary", false, "only print summary, not per-package details")
diffsOnly := flag.Bool("diffs", false, "only show packages with asset differences (skip matches)")
latest := flag.Bool("latest", false, "only compare latest version in each cache")
windowed := flag.Bool("windowed", false, "limit Go versions to the Node.js version range (2nd to 2nd-to-last)")
sample := flag.Int("sample", 0, "for each package diff, show N randomly sampled assets (implies -windowed -diffs)")
flag.Parse()
filterPkgs := flag.Args()
// -sample implies -windowed and -diffs so we focus on real classification
// differences, not version-depth noise.
if *sample > 0 {
*windowed = true
*diffsOnly = true
}
totalStart := time.Now()
// Find the most recent month directory in each cache.
liveMonth := findLatestMonth(*liveDir)
goMonth := findLatestMonth(*goDir)
if liveMonth == "" {
log.Fatalf("no month directories found in %s", *liveDir)
}
livePath := filepath.Join(*liveDir, liveMonth)
goPath := ""
if goMonth != "" {
goPath = filepath.Join(*goDir, goMonth)
}
// Discover all packages across both caches.
discoverStart := time.Now()
allPkgs := discoverPackages(livePath, goPath)
if len(filterPkgs) > 0 {
nameSet := make(map[string]bool, len(filterPkgs))
for _, n := range filterPkgs {
nameSet[n] = true
}
var filtered []string
for _, p := range allPkgs {
if nameSet[p] {
filtered = append(filtered, p)
}
}
allPkgs = filtered
}
log.Printf("discovered %d packages in %s", len(allPkgs), time.Since(discoverStart))
compareStart := time.Now()
var diffs []packageDiff
for _, pkg := range allPkgs {
d := compare(livePath, goPath, pkg, *latest, *windowed)
categorize(&d)
diffs = append(diffs, d)
}
log.Printf("compared %d packages in %s", len(diffs), time.Since(compareStart))
if *summary {
printSummary(diffs)
} else {
printDetails(diffs, *diffsOnly, *sample)
}
log.Printf("total: %s", time.Since(totalStart))
}
func findLatestMonth(dir string) string {
entries, err := os.ReadDir(dir)
if err != nil {
return ""
}
var months []string
for _, e := range entries {
if e.IsDir() && len(e.Name()) == 7 && e.Name()[4] == '-' {
months = append(months, e.Name())
}
}
if len(months) == 0 {
return ""
}
sort.Strings(months)
return months[len(months)-1]
}
func discoverPackages(livePath, goPath string) []string {
seen := make(map[string]bool)
for _, dir := range []string{livePath, goPath} {
if dir == "" {
continue
}
entries, err := os.ReadDir(dir)
if err != nil {
continue
}
for _, e := range entries {
name := e.Name()
if strings.HasSuffix(name, ".json") && !strings.HasSuffix(name, ".updated.txt") {
pkg := strings.TrimSuffix(name, ".json")
seen[pkg] = true
}
}
}
var pkgs []string
for p := range seen {
pkgs = append(pkgs, p)
}
sort.Strings(pkgs)
return pkgs
}
func loadCache(dir, pkg string) *cacheEntry {
if dir == "" {
return nil
}
data, err := os.ReadFile(filepath.Join(dir, pkg+".json"))
if err != nil {
return nil
}
var entry cacheEntry
if err := json.Unmarshal(data, &entry); err != nil {
return nil
}
return &entry
}
// effectiveName returns the best available filename for a release entry.
// Node.js sometimes uses _filename (a path) instead of name.
func effectiveName(name, filename, download string) string {
if name != "" {
return name
}
if filename != "" {
// _filename may be a path like "stable/macos/flutter_macos_3.41.4.zip"
if i := strings.LastIndex(filename, "/"); i >= 0 {
return filename[i+1:]
}
return filename
}
// Last resort: basename of download URL.
if download != "" {
if i := strings.LastIndex(download, "/"); i >= 0 {
return download[i+1:]
}
}
return ""
}
// versionWindow returns the 2nd and 2nd-to-last versions from a sorted
// version list. This trims the edges where Node.js may have a newer fetch
// or Go may have deeper history, focusing on the overlapping middle.
func versionWindow(versions []string) (low, high string) {
if len(versions) <= 2 {
// Too few versions to window — use all.
if len(versions) > 0 {
return versions[0], versions[len(versions)-1]
}
return "", ""
}
// 2nd version (skip oldest) and 2nd-to-last (skip newest).
return versions[1], versions[len(versions)-2]
}
// filterVersionRange returns only the versions in sorted order that fall
// within [low, high] inclusive (by lexver comparison).
func filterVersionRange(vf map[string]map[string]bool, versions []string, low, high string) (map[string]bool, []string) {
lowV := lexver.Parse(low)
highV := lexver.Parse(high)
files := make(map[string]bool)
var kept []string
for _, v := range versions {
pv := lexver.Parse(v)
if lexver.Compare(pv, lowV) >= 0 && lexver.Compare(pv, highV) <= 0 {
kept = append(kept, v)
for f := range vf[v] {
files[f] = true
}
}
}
return files, kept
}
func compare(livePath, goPath, pkg string, latestOnly, windowed bool) packageDiff {
live := loadCache(livePath, pkg)
goCache := loadCache(goPath, pkg)
d := packageDiff{Name: pkg}
if live == nil {
d.LiveMissing = true
}
if goCache == nil {
d.GoMissing = true
}
if d.LiveMissing && d.GoMissing {
return d
}
normVersion := normalizeVersionFunc(pkg)
// Collect filenames by version. If filter is non-nil, skip filenames it rejects.
extractVersionFiles := func(ce *cacheEntry, filter func(string) bool) (map[string]map[string]bool, []string) {
vf := make(map[string]map[string]bool)
for _, r := range ce.Releases {
name := effectiveName(r.Name, r.Filename, r.Download)
if filter != nil && !filter(name) {
continue
}
ver := normVersion(r.Version)
if vf[ver] == nil {
vf[ver] = make(map[string]bool)
}
vf[ver][name] = true
}
var versions []string
for v := range vf {
versions = append(versions, v)
}
slices.SortFunc(versions, func(a, b string) int {
return lexver.Compare(lexver.Parse(a), lexver.Parse(b))
})
return vf, versions
}
notNoise := func(name string) bool { return !isLiveNoise(name) }
var liveFiles, goFiles map[string]bool
// Parse live cache.
var liveVF map[string]map[string]bool
var liveVersions []string
if live != nil {
liveVF, liveVersions = extractVersionFiles(live, notNoise)
d.VersionsLive = liveVersions
d.LiveCount = len(live.Releases)
}
// Parse Go cache.
var goVF map[string]map[string]bool
var goVersions []string
if goCache != nil {
goVF, goVersions = extractVersionFiles(goCache, notNoise)
d.VersionsGo = goVersions
d.GoCount = len(goCache.Releases)
}
// Determine which files to compare based on mode.
if latestOnly {
// Compare only the latest version from each cache.
if live != nil && len(liveVersions) > 0 {
liveFiles = liveVF[liveVersions[len(liveVersions)-1]]
}
if goCache != nil && len(goVersions) > 0 {
goFiles = goVF[goVersions[len(goVersions)-1]]
}
} else if windowed && live != nil && len(liveVersions) > 0 {
// Use the Node.js version range (2nd to 2nd-to-last) to establish
// the window. Include ALL Node.js versions in the window (so missing
// Go versions are visible), but exclude Go-only versions (those are
// just deeper history, not real gaps).
low, high := versionWindow(liveVersions)
lowV := lexver.Parse(low)
highV := lexver.Parse(high)
// Collect all live files in the window.
liveFiles = make(map[string]bool)
liveInWindow := make(map[string]bool)
for _, v := range liveVersions {
pv := lexver.Parse(v)
if lexver.Compare(pv, lowV) >= 0 && lexver.Compare(pv, highV) <= 0 {
liveInWindow[v] = true
for f := range liveVF[v] {
liveFiles[f] = true
}
}
}
// For Go, only include versions that Node.js also has in the window.
// Go-only versions are hidden (deeper history, not gaps).
goFiles = make(map[string]bool)
for _, v := range goVersions {
if !liveInWindow[v] {
continue
}
for f := range goVF[v] {
goFiles[f] = true
}
}
} else {
// Compare all versions — use pre-filtered version maps.
if live != nil {
liveFiles = make(map[string]bool)
for _, files := range liveVF {
for f := range files {
liveFiles[f] = true
}
}
}
if goCache != nil {
goFiles = make(map[string]bool)
for _, files := range goVF {
for f := range files {
goFiles[f] = true
}
}
}
}
if liveFiles == nil {
liveFiles = make(map[string]bool)
}
if goFiles == nil {
goFiles = make(map[string]bool)
}
for f := range liveFiles {
if !goFiles[f] {
d.OnlyInLive = append(d.OnlyInLive, f)
}
}
for f := range goFiles {
if !liveFiles[f] {
d.OnlyInGo = append(d.OnlyInGo, f)
}
}
sort.Strings(d.OnlyInLive)
sort.Strings(d.OnlyInGo)
// Field-level comparison on assets that exist in both caches.
// Build version+filename → fields maps from each cache.
if live != nil && goCache != nil {
type assetKey struct {
version string
filename string
}
liveByKey := make(map[assetKey]cacheRelease)
for _, r := range live.Releases {
name := effectiveName(r.Name, r.Filename, r.Download)
ver := normVersion(r.Version)
liveByKey[assetKey{ver, name}] = r
}
for _, r := range goCache.Releases {
name := effectiveName(r.Name, r.Filename, r.Download)
ver := normVersion(r.Version)
lr, ok := liveByKey[assetKey{ver, name}]
if !ok {
continue
}
// Compare classification fields.
// Use equivalence checks for os/arch/ext so naming
// convention differences don't mask real classification bugs.
for _, cmp := range []struct {
field string
live string
go_ string
equiv bool
}{
{"os", lr.OS, r.OS, equivOS(lr.OS, r.OS)},
{"arch", lr.Arch, r.Arch, equivArch(lr.Arch, r.Arch)},
{"libc", lr.Libc, r.Libc, lr.Libc == r.Libc},
{"ext", lr.Ext, r.Ext, equivExt(lr.Ext, r.Ext)},
{"channel", lr.Channel, r.Channel, lr.Channel == r.Channel},
} {
if cmp.equiv {
continue
}
d.FieldDiffs = append(d.FieldDiffs, fieldDiff{
Filename: name,
Field: cmp.field,
Live: cmp.live,
Go: cmp.go_,
BothSet: cmp.live != "" && cmp.go_ != "",
})
}
}
sort.Slice(d.FieldDiffs, func(i, j int) bool {
if d.FieldDiffs[i].Field != d.FieldDiffs[j].Field {
return d.FieldDiffs[i].Field < d.FieldDiffs[j].Field
}
return d.FieldDiffs[i].Filename < d.FieldDiffs[j].Filename
})
}
return d
}
// equivOS returns true if two OS values are equivalent across naming conventions.
func equivOS(a, b string) bool {
return a == b || canonicalOS(a) == canonicalOS(b)
}
func canonicalOS(s string) string {
switch strings.ToLower(s) {
case "darwin", "macos", "mac", "osx":
return "darwin"
case "win", "windows":
return "windows"
default:
return strings.ToLower(s)
}
}
// equivArch returns true if two arch values are equivalent.
func equivArch(a, b string) bool {
return a == b || canonicalArch(a) == canonicalArch(b)
}
func canonicalArch(s string) string {
switch strings.ToLower(s) {
case "x86_64", "amd64", "x64":
return "x86_64"
case "aarch64", "arm64":
return "aarch64"
case "armv7", "armv7l":
return "armv7"
case "armv6", "armv6l":
return "armv6"
case "x86", "i386", "i686", "386":
return "x86"
default:
return strings.ToLower(s)
}
}
// equivExt returns true if two extension values are equivalent.
func equivExt(a, b string) bool {
// Normalize: strip leading dot, handle common aliases.
return a == b || canonicalExt(a) == canonicalExt(b)
}
func canonicalExt(s string) string {
s = strings.TrimPrefix(s, ".")
switch s {
case "tgz":
return "tar.gz"
default:
return s
}
}
func categorize(d *packageDiff) {
if d.GoMissing {
d.Categories = append(d.Categories, "go-missing")
return
}
if d.LiveMissing {
d.Categories = append(d.Categories, "live-missing")
return
}
if len(d.OnlyInLive) == 0 && len(d.OnlyInGo) == 0 && len(d.FieldDiffs) == 0 {
d.Categories = append(d.Categories, "match")
return
}
if len(d.OnlyInLive) == 0 && len(d.OnlyInGo) == 0 && len(d.FieldDiffs) > 0 {
d.Categories = append(d.Categories, "fields-only")
}
// Check if differences are only version depth (Go has more history).
liveVersionSet := make(map[string]bool, len(d.VersionsLive))
for _, v := range d.VersionsLive {
liveVersionSet[v] = true
}
goVersionSet := make(map[string]bool, len(d.VersionsGo))
for _, v := range d.VersionsGo {
goVersionSet[v] = true
}
goExtraVersions := 0
for _, v := range d.VersionsGo {
if !liveVersionSet[v] {
goExtraVersions++
}
}
liveExtraVersions := 0
for _, v := range d.VersionsLive {
if !goVersionSet[v] {
liveExtraVersions++
}
}
if goExtraVersions > 0 {
d.Categories = append(d.Categories, fmt.Sprintf("go-extra-versions(%d)", goExtraVersions))
}
if liveExtraVersions > 0 {
d.Categories = append(d.Categories, fmt.Sprintf("live-extra-versions(%d)", liveExtraVersions))
}
// Check for meta-asset filtering differences.
metaOnlyInLive := 0
nonMetaOnlyInLive := 0
for _, f := range d.OnlyInLive {
if classify.IsMetaAsset(f) {
metaOnlyInLive++
} else {
nonMetaOnlyInLive++
}
}
metaOnlyInGo := 0
nonMetaOnlyInGo := 0
for _, f := range d.OnlyInGo {
if classify.IsMetaAsset(f) {
metaOnlyInGo++
} else {
nonMetaOnlyInGo++
}
}
if metaOnlyInLive > 0 {
d.Categories = append(d.Categories, fmt.Sprintf("live-has-meta(%d)", metaOnlyInLive))
}
if metaOnlyInGo > 0 {
d.Categories = append(d.Categories, fmt.Sprintf("go-has-meta(%d)", metaOnlyInGo))
}
// Check for source tarball differences.
srcOnlyInGo := 0
for _, f := range d.OnlyInGo {
if strings.HasSuffix(f, ".tar.gz") || strings.HasSuffix(f, ".zip") {
if strings.HasPrefix(f, "v") || strings.HasPrefix(f, "refs/") {
srcOnlyInGo++
}
}
}
if srcOnlyInGo > 0 {
d.Categories = append(d.Categories, fmt.Sprintf("go-has-source-tarballs(%d)", srcOnlyInGo))
}
if nonMetaOnlyInLive > 0 {
d.Categories = append(d.Categories, fmt.Sprintf("live-extra-assets(%d)", nonMetaOnlyInLive))
}
if nonMetaOnlyInGo > 0 {
d.Categories = append(d.Categories, fmt.Sprintf("go-extra-assets(%d)", nonMetaOnlyInGo))
}
// Count field diffs by field name, separating real disagreements
// from expected "live empty, Go classified" differences.
type fieldCount struct {
bothSet int // both caches have a value but they disagree
oneEmpty int // one side is empty (typically live — normalize.js fills at serve time)
}
fieldCounts := make(map[string]fieldCount)
for _, fd := range d.FieldDiffs {
fc := fieldCounts[fd.Field]
if fd.BothSet {
fc.bothSet++
} else {
fc.oneEmpty++
}
fieldCounts[fd.Field] = fc
}
for _, field := range []string{"os", "arch", "libc", "ext", "channel"} {
fc := fieldCounts[field]
if fc.bothSet > 0 {
d.Categories = append(d.Categories, fmt.Sprintf("diff-%s(%d)", field, fc.bothSet))
}
if fc.oneEmpty > 0 {
d.Categories = append(d.Categories, fmt.Sprintf("fill-%s(%d)", field, fc.oneEmpty))
}
}
}
// isLiveNoise returns true for filenames that the Node.js cache keeps
// but Go intentionally filters out. Pre-filtering these from the live
// side prevents them from appearing as live-extra-assets noise.
//
// This includes everything classify.IsMetaAsset catches plus formats
// that Go's legacy export strips (.deb, .rpm, etc.).
func isLiveNoise(name string) bool {
if classify.IsMetaAsset(name) {
return true
}
lower := strings.ToLower(name)
// Formats Go filters from legacy export but Node.js keeps.
for _, suffix := range []string{
".deb", ".rpm", ".gpg",
} {
if strings.HasSuffix(lower, suffix) {
return true
}
}
// Source tarballs (e.g. gitea-src-1.25.4.tar.gz, caddy_2.10.0_src.tar.gz, go1.26.1.src.tar.gz).
if strings.Contains(lower, "-src-") || strings.Contains(lower, "_src.") || strings.Contains(lower, ".src.") || strings.HasPrefix(lower, "src-") {
return true
}
// Docs tarballs (e.g. gitea-docs-1.22.3.tar.gz).
if strings.Contains(lower, "-docs-") {
return true
}
// Bare executables without any extension — typically legacy shell scripts
// uploaded alongside proper archives (e.g. kubectx, kubens).
if !strings.Contains(name, ".") {
return true
}
// GPU accelerator / hardware variants that Go tags as variants
// but Node.js keeps with special arch names.
for _, v := range []string{"-rocm", "-jetpack"} {
if strings.Contains(lower, v) {
return true
}
}
// Linux binaries for packages where Node.js only kept macOS .app.zip.
// Go correctly includes these as installable on Linux.
if strings.HasPrefix(lower, "fish-") && strings.Contains(lower, "-linux-") {
return true
}
return false
}
// normalizeVersionFunc returns a version normalizer for a given package.
// Most packages return the identity function. Some (like git) need
// version string normalization to match across Go and Node.js caches.
func normalizeVersionFunc(pkg string) func(string) string {
switch pkg {
case "git":
return func(v string) string {
// Git for Windows: v2.53.0.windows.1 → v2.53.0
// v2.53.0.windows.2 → v2.53.0.2
idx := strings.Index(v, ".windows.")
if idx < 0 {
return v
}
suffix := v[idx+len(".windows."):]
base := v[:idx]
if suffix == "1" {
return base
}
return base + "." + suffix
}
case "lf":
return func(v string) string {
// lf: r21 → 0.21.0
if strings.HasPrefix(v, "r") {
return "0." + v[1:] + ".0"
}
return v
}
case "bun":
return func(v string) string {
// bun: bun-v1.3.9 → v1.3.9
return strings.TrimPrefix(v, "bun-")
}
case "watchexec":
return func(v string) string {
// watchexec monorepo: cli-v1.20.5 → v1.20.5
return strings.TrimPrefix(v, "cli-")
}
case "go":
return func(v string) string {
// Go: go1.10 → 1.10.0 (pad to 3 parts)
v = strings.TrimPrefix(v, "go")
parts := strings.SplitN(v, ".", 3)
for len(parts) < 3 {
parts = append(parts, "0")
}
return strings.Join(parts, ".")
}
default:
return func(v string) string { return v }
}
}
func printSummary(diffs []packageDiff) {
// Count by category.
categoryCounts := make(map[string]int)
for _, d := range diffs {
for _, c := range d.Categories {
// Strip the count suffix for grouping.
base := c
if idx := strings.Index(c, "("); idx != -1 {
base = c[:idx]
}
categoryCounts[base]++
}
}
fmt.Println("=== COMPARISON SUMMARY ===")
fmt.Printf("Total packages: %d\n\n", len(diffs))
var cats []string
for c := range categoryCounts {
cats = append(cats, c)
}
sort.Strings(cats)
for _, c := range cats {
fmt.Printf(" %-30s %d\n", c, categoryCounts[c])
}
fmt.Println("\n=== PER-PACKAGE CATEGORIES ===")
for _, d := range diffs {
fmt.Printf("%-25s %s\n", d.Name, strings.Join(d.Categories, ", "))
}
}
func printDetails(diffs []packageDiff, diffsOnly bool, sampleN int) {
for _, d := range diffs {
if diffsOnly && len(d.OnlyInLive) == 0 && len(d.OnlyInGo) == 0 && len(d.FieldDiffs) == 0 {
continue
}
fmt.Printf("=== %s ===\n", d.Name)
fmt.Printf(" Categories: %s\n", strings.Join(d.Categories, ", "))
fmt.Printf(" Live: %d assets, %d versions | Go: %d assets, %d versions\n",
d.LiveCount, len(d.VersionsLive), d.GoCount, len(d.VersionsGo))
printAssetList("Only in LIVE", d.OnlyInLive, sampleN)
printAssetList("Only in Go", d.OnlyInGo, sampleN)
printFieldDiffs(d.FieldDiffs, sampleN)
fmt.Println()
}
}
// printFieldDiffs shows classification differences on shared assets.
// Shows "real" diffs (both sides non-empty) first, then "fill" diffs
// (one side empty) as a summary count only.
func printFieldDiffs(diffs []fieldDiff, sampleN int) {
if len(diffs) == 0 {
return
}
// Separate real disagreements from fill diffs.
var real, fill []fieldDiff
for _, fd := range diffs {
if fd.BothSet {
real = append(real, fd)
} else {
fill = append(fill, fd)
}
}
// Show real disagreements in detail.
if len(real) > 0 {
byField := make(map[string][]fieldDiff)
for _, fd := range real {
byField[fd.Field] = append(byField[fd.Field], fd)
}
for _, field := range []string{"os", "arch", "libc", "ext", "channel"} {
fds := byField[field]
if len(fds) == 0 {
continue
}
fmt.Printf(" DISAGREE %s (%d):\n", field, len(fds))
printFieldDiffItems(fds, sampleN)
}
}
// Summarize fill diffs (live empty, Go classified) as counts.
if len(fill) > 0 {
byField := make(map[string]int)
for _, fd := range fill {
byField[fd.Field]++
}
var parts []string
for _, field := range []string{"os", "arch", "libc", "ext", "channel"} {
if n := byField[field]; n > 0 {
parts = append(parts, fmt.Sprintf("%s(%d)", field, n))
}
}
if len(parts) > 0 {
fmt.Printf(" Go fills empty: %s\n", strings.Join(parts, ", "))
}
}
}
func printFieldDiffItems(fds []fieldDiff, sampleN int) {
items := fds
if sampleN > 0 && len(items) > sampleN {
sampled := make([]fieldDiff, len(items))
copy(sampled, items)
rand.Shuffle(len(sampled), func(i, j int) {
sampled[i], sampled[j] = sampled[j], sampled[i]
})
items = sampled[:sampleN]
sort.Slice(items, func(i, j int) bool {
return items[i].Filename < items[j].Filename
})
}
limit := 20
for i, fd := range items {
if sampleN == 0 && i >= limit {
fmt.Printf(" ... and %d more\n", len(fds)-limit)
break
}
fmt.Printf(" - %s: live=%q go=%q\n", fd.Filename, fd.Live, fd.Go)
}
if sampleN > 0 && len(fds) > sampleN {
fmt.Printf(" ... sampled %d of %d\n", sampleN, len(fds))
}
}
// printAssetList prints a list of asset filenames, optionally sampling N at
// random. When sampleN > 0 and the list is longer, it picks N random items
// so you can spot classification bugs across the full range instead of only
// seeing the first alphabetical entries.
func printAssetList(label string, items []string, sampleN int) {
if len(items) == 0 {
return
}
fmt.Printf(" %s (%d):\n", label, len(items))
if sampleN > 0 && len(items) > sampleN {
// Shuffle a copy, take first N, then sort for readable output.
sampled := make([]string, len(items))
copy(sampled, items)
rand.Shuffle(len(sampled), func(i, j int) {
sampled[i], sampled[j] = sampled[j], sampled[i]
})
picked := sampled[:sampleN]
sort.Strings(picked)
for _, f := range picked {
fmt.Printf(" - %s\n", f)
}
fmt.Printf(" ... sampled %d of %d (run again for different sample)\n", sampleN, len(items))
return
}
limit := 20
for i, f := range items {
if i >= limit {
fmt.Printf(" ... and %d more\n", len(items)-limit)
break
}
fmt.Printf(" - %s\n", f)
}
}

View File

@@ -1,846 +0,0 @@
// Command e2etest runs the full release pipeline for selected packages
// and compares results against the live webi.sh API.
//
// It fetches from upstream, classifies assets, resolves the best match
// for a set of test queries, then fetches the same queries from the live
// API and reports any differences.
//
// Usage:
//
// go run ./cmd/e2etest
// go run ./cmd/e2etest -packages goreleaser,ollama,node
// go run ./cmd/e2etest -cache ./_cache/raw # reuse existing cache
package main
import (
"bufio"
"context"
"encoding/json"
"flag"
"fmt"
"io"
"log"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/webinstall/webi-installers/internal/buildmeta"
"github.com/webinstall/webi-installers/internal/installerconf"
"github.com/webinstall/webi-installers/internal/lexver"
"github.com/webinstall/webi-installers/internal/rawcache"
"github.com/webinstall/webi-installers/internal/releases/github"
"github.com/webinstall/webi-installers/internal/releases/githubish"
"github.com/webinstall/webi-installers/internal/releases/nodedist"
"github.com/webinstall/webi-installers/internal/resolve"
)
// testCase is one query to resolve and compare against the live API.
type testCase struct {
Name string
Package string
OS buildmeta.OS
Arch buildmeta.Arch
Libc buildmeta.Libc
Formats []string
UA string // User-Agent for live API query
}
// liveResult holds parsed fields from the live webi API response.
type liveResult struct {
Version string
OS string
Arch string
Libc string
Ext string
PkgURL string
PkgFile string
Channel string
Stable string
Latest string
Oses string
Arches string
Libcs string
Formats string
}
// UA format from webi.sh bootstrap: "curl {uname -s}/{uname -r} {uname -m}/unknown {libc}"
// libc is "gnu", "musl", or "libc" (for darwin/other)
var cases = []testCase{
{
Name: "goreleaser/linux/x86_64", Package: "goreleaser",
OS: buildmeta.OSLinux, Arch: buildmeta.ArchAMD64, Libc: buildmeta.LibcGNU,
Formats: []string{".tar.gz", ".tar.xz", ".zip"},
UA: "curl Linux/6.6.123 x86_64/unknown gnu",
},
{
Name: "goreleaser/darwin/arm64", Package: "goreleaser",
OS: buildmeta.OSDarwin, Arch: buildmeta.ArchARM64, Libc: "",
Formats: []string{".tar.gz", ".tar.xz", ".zip"},
UA: "curl Darwin/25.2.0 arm64/unknown libc",
},
{
Name: "goreleaser/windows/x86_64", Package: "goreleaser",
OS: buildmeta.OSWindows, Arch: buildmeta.ArchAMD64, Libc: "",
Formats: []string{".zip", ".exe"},
UA: "PowerShell/7.0 Windows/10.0 x86_64/unknown msvc",
},
{
Name: "ollama/linux/x86_64", Package: "ollama",
OS: buildmeta.OSLinux, Arch: buildmeta.ArchAMD64, Libc: buildmeta.LibcGNU,
Formats: []string{".tar.gz", ".tar.xz", ".tar.zst", ".zip"},
UA: "curl Linux/6.6.123 x86_64/unknown gnu",
},
{
Name: "ollama/darwin/arm64", Package: "ollama",
OS: buildmeta.OSDarwin, Arch: buildmeta.ArchARM64, Libc: "",
Formats: []string{".tar.gz", ".tar.xz", ".tar.zst", ".zip", ".dmg"},
UA: "curl Darwin/25.2.0 arm64/unknown libc",
},
{
Name: "ollama/linux/arm64", Package: "ollama",
OS: buildmeta.OSLinux, Arch: buildmeta.ArchARM64, Libc: buildmeta.LibcGNU,
Formats: []string{".tar.gz", ".tar.xz", ".tar.zst", ".zip"},
UA: "curl Linux/6.6.123 aarch64/unknown gnu",
},
{
Name: "node/linux/x86_64", Package: "node",
OS: buildmeta.OSLinux, Arch: buildmeta.ArchAMD64, Libc: buildmeta.LibcGNU,
Formats: []string{".tar.xz", ".tar.gz", ".zip"},
UA: "curl Linux/6.6.123 x86_64/unknown gnu",
},
{
Name: "node/darwin/arm64", Package: "node",
OS: buildmeta.OSDarwin, Arch: buildmeta.ArchARM64, Libc: "",
Formats: []string{".tar.xz", ".tar.gz", ".zip"},
UA: "curl Darwin/25.2.0 arm64/unknown libc",
},
{
Name: "node/linux/arm64", Package: "node",
OS: buildmeta.OSLinux, Arch: buildmeta.ArchARM64, Libc: buildmeta.LibcGNU,
Formats: []string{".tar.xz", ".tar.gz", ".zip"},
UA: "curl Linux/6.6.123 aarch64/unknown gnu",
},
}
func main() {
cacheDir := flag.String("cache", "_cache/raw", "root directory for raw cache")
confDir := flag.String("conf", ".", "root directory containing {pkg}/releases.conf files")
token := flag.String("token", os.Getenv("GITHUB_TOKEN"), "GitHub API token")
skipFetch := flag.Bool("skip-fetch", false, "skip fetching, use existing cache")
skipLive := flag.Bool("skip-live", false, "skip live API comparison")
packages := flag.String("packages", "goreleaser,ollama,node", "comma-separated packages to test")
flag.Parse()
pkgList := strings.Split(*packages, ",")
pkgSet := make(map[string]bool, len(pkgList))
for _, p := range pkgList {
pkgSet[strings.TrimSpace(p)] = true
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
client := &http.Client{Timeout: 30 * time.Second}
var auth *githubish.Auth
if *token != "" {
auth = &githubish.Auth{Token: *token}
}
// Step 1: Fetch raw releases.
if !*skipFetch {
log.Println("=== Step 1: Fetching releases ===")
for _, pkg := range pkgList {
if err := fetchPackage(ctx, client, *cacheDir, *confDir, pkg, auth); err != nil {
log.Fatalf("fetch %s: %v", pkg, err)
}
}
} else {
log.Println("=== Step 1: Skipping fetch (using cache) ===")
}
// Step 2: Classify releases.
log.Println("=== Step 2: Classifying releases ===")
allDists := make(map[string][]resolve.Dist)
for _, pkg := range pkgList {
conf, err := installerconf.Read(filepath.Join(*confDir, pkg, "releases.conf"))
if err != nil {
log.Fatalf("read conf %s: %v", pkg, err)
}
d, err := rawcache.Open(filepath.Join(*cacheDir, pkg))
if err != nil {
log.Fatalf("open cache %s: %v", pkg, err)
}
dists, err := classifyFromCache(pkg, conf, d)
if err != nil {
log.Fatalf("classify %s: %v", pkg, err)
}
allDists[pkg] = dists
log.Printf(" %s: %d distributables", pkg, len(dists))
// Show catalog.
cat := resolve.Survey(dists)
log.Printf(" oses=%v arches=%v libcs=%v formats=%v", cat.OSes, cat.Arches, cat.Libcs, cat.Formats)
log.Printf(" latest=%s stable=%s", cat.Latest, cat.Stable)
}
// Step 3: Resolve best match for each test case.
log.Println("=== Step 3: Resolving best matches ===")
type result struct {
tc testCase
match *resolve.Match
live *liveResult
}
var results []result
for _, tc := range cases {
if !pkgSet[tc.Package] {
continue
}
dists := allDists[tc.Package]
q := resolve.Query{
OS: tc.OS,
Arch: tc.Arch,
Libc: tc.Libc,
Formats: tc.Formats,
Channel: "stable",
}
m := resolve.Best(dists, q)
results = append(results, result{tc: tc, match: m})
}
// Step 4: Compare with live API.
if !*skipLive {
log.Println("=== Step 4: Comparing with live API ===")
for i := range results {
tc := results[i].tc
live, err := queryLiveAPI(client, tc)
if err != nil {
log.Printf(" %s: live API error: %v", tc.Name, err)
continue
}
results[i].live = live
}
}
// Step 5: Report.
log.Println("")
log.Println("=== Results ===")
log.Println("")
pass, fail, warn := 0, 0, 0
for _, r := range results {
tc := r.tc
m := r.match
live := r.live
if m == nil {
log.Printf("FAIL %s: no match found", tc.Name)
fail++
continue
}
log.Printf("--- %s ---", tc.Name)
log.Printf(" Go: version=%s file=%s ext=%s url=%s", m.Version, m.Filename, m.Format, m.Download)
if live != nil {
log.Printf(" Live: version=%s file=%s ext=%s url=%s", live.Version, live.PkgFile, live.Ext, live.PkgURL)
if live.Version == "0.0.0" {
log.Printf(" WARN: live API returned error (no match)")
warn++
} else if m.Version == live.Version && m.Filename == live.PkgFile {
log.Printf(" PASS: exact match")
pass++
} else if m.Version == live.Version && m.Download == live.PkgURL {
log.Printf(" PASS: same URL (filename display differs: go=%s live=%s)", m.Filename, live.PkgFile)
pass++
} else if m.Version == live.Version {
log.Printf(" WARN: same version, different file (go=%s live=%s)", m.Filename, live.PkgFile)
warn++
} else {
log.Printf(" DIFF: version mismatch (go=%s live=%s)", m.Version, live.Version)
fail++
}
} else {
log.Printf(" (no live comparison)")
pass++
}
}
log.Println("")
log.Printf("Summary: %d pass, %d fail, %d warn (live API errors)", pass, fail, warn)
if fail > 0 {
os.Exit(1)
}
}
// fetchPackage fetches raw releases for one package.
func fetchPackage(ctx context.Context, client *http.Client, cacheRoot, confDir, pkg string, auth *githubish.Auth) error {
conf, err := installerconf.Read(filepath.Join(confDir, pkg, "releases.conf"))
if err != nil {
return fmt.Errorf("read conf: %w", err)
}
source := conf.Source
log.Printf(" %s: source=%s", pkg, source)
switch source {
case "github":
return fetchGitHub(ctx, client, cacheRoot, pkg, conf, auth)
case "nodedist":
return fetchNodeDist(ctx, client, cacheRoot, pkg, conf)
default:
return fmt.Errorf("unsupported source %q (only github and nodedist for e2e test)", source)
}
}
func fetchGitHub(ctx context.Context, client *http.Client, cacheRoot, pkg string, conf *installerconf.Conf, auth *githubish.Auth) error {
owner := conf.Owner
repo := conf.Repo
tagPrefix := conf.TagPrefix
d, err := rawcache.Open(filepath.Join(cacheRoot, pkg))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range github.Fetch(ctx, client, owner, repo, auth) {
if err != nil {
return fmt.Errorf("github %s/%s: %w", owner, repo, err)
}
for _, rel := range batch {
if rel.Draft {
continue
}
tag := rel.TagName
if tagPrefix != "" {
if !strings.HasPrefix(tag, tagPrefix) {
continue
}
tag = strings.TrimPrefix(tag, tagPrefix)
}
data, err := json.Marshal(rel)
if err != nil {
return fmt.Errorf("marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if latest == "" && !rel.Prerelease {
latest = tag
}
}
}
if latest != "" {
current := d.Latest()
if current == "" || lexver.Compare(lexver.Parse(latest), lexver.Parse(current)) > 0 {
d.SetLatest(latest)
}
}
log.Printf(" +%d ~%d =%d latest=%s", added, changed, skipped, d.Latest())
return nil
}
func fetchNodeDist(ctx context.Context, client *http.Client, cacheRoot, pkg string, conf *installerconf.Conf) error {
baseURL := conf.BaseURL
d, err := rawcache.Open(filepath.Join(cacheRoot, pkg))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range nodedist.Fetch(ctx, client, baseURL) {
if err != nil {
return fmt.Errorf("nodedist: %w", err)
}
for _, entry := range batch {
tag := entry.Version
data, err := json.Marshal(entry)
if err != nil {
return fmt.Errorf("marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if latest == "" {
latest = tag
}
}
}
if latest != "" {
current := d.Latest()
if current == "" || lexver.Compare(lexver.Parse(latest), lexver.Parse(current)) > 0 {
d.SetLatest(latest)
}
}
log.Printf(" +%d ~%d =%d latest=%s", added, changed, skipped, d.Latest())
return nil
}
// classifyFromCache reads the raw cache and produces classified dists.
func classifyFromCache(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]resolve.Dist, error) {
source := conf.Source
switch source {
case "github":
return classifyGitHub(pkg, conf, d)
case "nodedist":
return classifyNodeDist(pkg, conf, d)
default:
return nil, fmt.Errorf("unsupported source %q", source)
}
}
func classifyGitHub(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]resolve.Dist, error) {
tagPrefix := conf.TagPrefix
releases, err := readAllReleases(d)
if err != nil {
return nil, err
}
var dists []resolve.Dist
for _, data := range releases {
var rel struct {
TagName string `json:"tag_name"`
Prerelease bool `json:"prerelease"`
Draft bool `json:"draft"`
PublishedAt string `json:"published_at"`
Assets []struct {
Name string `json:"name"`
BrowserDownloadURL string `json:"browser_download_url"`
Size int64 `json:"size"`
} `json:"assets"`
}
if err := json.Unmarshal(data, &rel); err != nil {
continue
}
if rel.Draft {
continue
}
version := rel.TagName
if tagPrefix != "" {
version = strings.TrimPrefix(version, tagPrefix)
}
// Strip leading "v" for version normalization.
version = strings.TrimPrefix(version, "v")
channel := "stable"
if rel.Prerelease {
channel = "beta"
}
date := ""
if len(rel.PublishedAt) >= 10 {
date = rel.PublishedAt[:10]
}
for _, asset := range rel.Assets {
if isMetaAsset(asset.Name) {
continue
}
r := classifyFilename(asset.Name)
extra := detectExtra(asset.Name)
dists = append(dists, resolve.Dist{
Package: pkg,
Version: version,
Channel: channel,
OS: r.os,
Arch: r.arch,
Libc: r.libc,
Format: r.format,
Download: asset.BrowserDownloadURL,
Filename: asset.Name,
Size: asset.Size,
Date: date,
Extra: extra,
})
}
}
return dists, nil
}
func classifyNodeDist(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]resolve.Dist, error) {
baseURL := conf.BaseURL
releases, err := readAllReleases(d)
if err != nil {
return nil, err
}
var dists []resolve.Dist
for _, data := range releases {
var entry struct {
Version string `json:"version"`
Date string `json:"date"`
Files []string `json:"files"`
LTS json.RawMessage `json:"lts"`
Security bool `json:"security"`
}
if err := json.Unmarshal(data, &entry); err != nil {
continue
}
lts := string(entry.LTS) != "false" && string(entry.LTS) != ""
version := strings.TrimPrefix(entry.Version, "v")
// Webi treats even major versions as "stable" (LTS-eligible).
channel := "stable"
parts := strings.SplitN(version, ".", 2)
if len(parts) > 0 {
var major int
fmt.Sscanf(parts[0], "%d", &major)
if major%2 != 0 {
channel = "beta"
}
}
for _, file := range entry.Files {
if file == "src" || file == "headers" {
continue
}
fileDists := expandNodeFile(pkg, entry.Version, version, channel, entry.Date, lts, baseURL, file)
dists = append(dists, fileDists...)
}
}
return dists, nil
}
func expandNodeFile(pkg, rawVersion, version, channel, date string, lts bool, baseURL, file string) []resolve.Dist {
parts := strings.Split(file, "-")
if len(parts) < 2 {
return nil
}
osMap := map[string]string{
"osx": "darwin", "linux": "linux", "win": "windows",
"sunos": "sunos", "aix": "aix",
}
archMap := map[string]string{
"x64": "x86_64", "x86": "x86", "arm64": "aarch64",
"armv7l": "armv7", "armv6l": "armv6",
"ppc64": "ppc64", "ppc64le": "ppc64le", "s390x": "s390x",
"loong64": "loong64", "riscv64": "riscv64",
}
os_ := osMap[parts[0]]
arch := archMap[parts[1]]
if os_ == "" || arch == "" {
return nil
}
libc := ""
pkgType := ""
if len(parts) > 2 {
pkgType = parts[2]
}
var formats []string
switch pkgType {
case "musl":
libc = "musl"
formats = []string{".tar.gz", ".tar.xz"}
case "tar":
formats = []string{".tar.gz", ".tar.xz"}
case "zip":
formats = []string{".zip"}
case "7z":
formats = []string{".7z"}
case "pkg":
formats = []string{".pkg"}
case "msi":
formats = []string{".msi"}
case "exe":
formats = []string{".exe"}
case "":
formats = []string{".tar.gz", ".tar.xz"}
default:
return nil
}
if libc == "" && os_ == "linux" {
libc = "gnu"
}
osPart := parts[0]
if osPart == "osx" {
osPart = "darwin"
}
archPart := parts[1]
muslExtra := ""
if libc == "musl" {
muslExtra = "-musl"
}
var dists []resolve.Dist
for _, format := range formats {
var filename string
if format == ".msi" {
filename = fmt.Sprintf("node-%s-%s%s%s", rawVersion, archPart, muslExtra, format)
} else {
filename = fmt.Sprintf("node-%s-%s-%s%s%s", rawVersion, osPart, archPart, muslExtra, format)
}
dists = append(dists, resolve.Dist{
Package: pkg,
Version: version,
Channel: channel,
OS: os_,
Arch: arch,
Libc: libc,
Format: format,
Download: fmt.Sprintf("%s/%s/%s", baseURL, rawVersion, filename),
Filename: filename,
LTS: lts,
Date: date,
})
}
return dists
}
// queryLiveAPI queries the live webi.sh API and parses the response header.
func queryLiveAPI(client *http.Client, tc testCase) (*liveResult, error) {
// Build format string matching what the webi.sh bootstrap sends.
// Order: tar,exe,zip,xz,dmg,git (least to most favorable in bootstrap,
// but the API doesn't care about order).
fmtParam := "tar,exe,zip,xz,dmg"
url := fmt.Sprintf("https://webi.sh/api/installers/%s@stable.sh?formats=%s", tc.Package, fmtParam)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", tc.UA)
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return parseLiveResponse(string(body)), nil
}
// parseLiveResponse extracts WEBI_* and PKG_* variables from the shell script.
func parseLiveResponse(body string) *liveResult {
vars := make(map[string]string)
scanner := bufio.NewScanner(strings.NewReader(body))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
for _, prefix := range []string{"WEBI_", "PKG_"} {
if strings.HasPrefix(line, prefix) {
if eq := strings.IndexByte(line, '='); eq > 0 {
key := line[:eq]
val := line[eq+1:]
val = strings.Trim(val, "'\"")
vars[key] = val
}
}
}
}
return &liveResult{
Version: vars["WEBI_VERSION"],
OS: vars["WEBI_OS"],
Arch: vars["WEBI_ARCH"],
Libc: vars["WEBI_LIBC"],
Ext: vars["WEBI_EXT"],
PkgURL: vars["WEBI_PKG_URL"],
PkgFile: vars["WEBI_PKG_FILE"],
Channel: vars["WEBI_CHANNEL"],
Stable: vars["PKG_STABLE"],
Latest: vars["PKG_LATEST"],
Oses: vars["PKG_OSES"],
Arches: vars["PKG_ARCHES"],
Libcs: vars["PKG_LIBCS"],
Formats: vars["PKG_FORMATS"],
}
}
// readAllReleases reads all cached release files.
func readAllReleases(d *rawcache.Dir) (map[string][]byte, error) {
active, err := d.ActivePath()
if err != nil {
return nil, err
}
entries, err := os.ReadDir(active)
if err != nil {
return nil, err
}
result := make(map[string][]byte, len(entries))
for _, e := range entries {
if e.IsDir() || strings.HasPrefix(e.Name(), "_") {
continue
}
data, err := os.ReadFile(filepath.Join(active, e.Name()))
if err != nil {
return nil, err
}
result[e.Name()] = data
}
return result, nil
}
type classResult struct {
os, arch, libc, format string
}
func classifyFilename(name string) classResult {
// Use the classify package.
// Import it indirectly to avoid circular deps — inline the logic
// we need for the e2e test.
lower := strings.ToLower(name)
var r classResult
r.format = detectFormat(name)
// OS detection
switch {
case strings.Contains(lower, "linux"):
r.os = "linux"
case strings.Contains(lower, "darwin") || strings.Contains(lower, "macos") || strings.Contains(lower, "apple"):
r.os = "darwin"
case strings.Contains(lower, "windows") || strings.Contains(lower, "win64") || strings.Contains(lower, "win32"):
r.os = "windows"
case strings.HasSuffix(lower, ".dmg") || strings.HasSuffix(lower, ".app.zip"):
r.os = "darwin"
case strings.HasSuffix(lower, ".exe") || strings.HasSuffix(lower, ".msi"):
r.os = "windows"
case strings.Contains(lower, "freebsd"):
r.os = "freebsd"
}
// Arch detection
switch {
case strings.Contains(lower, "x86_64") || strings.Contains(lower, "amd64") || strings.Contains(lower, "x64"):
r.arch = "x86_64"
case strings.Contains(lower, "aarch64") || strings.Contains(lower, "arm64"):
r.arch = "aarch64"
case strings.Contains(lower, "armv7") || strings.Contains(lower, "armhf"):
r.arch = "armv7"
case strings.Contains(lower, "armv6"):
r.arch = "armv6"
case strings.Contains(lower, "i686") || strings.Contains(lower, "i386") || strings.Contains(lower, "x86") || strings.Contains(lower, "386"):
r.arch = "x86"
case strings.Contains(lower, "ppc64le") || strings.Contains(lower, "powerpc64le"):
r.arch = "ppc64le"
case strings.Contains(lower, "ppc64") || strings.Contains(lower, "powerpc64"):
r.arch = "ppc64"
case strings.Contains(lower, "riscv64"):
r.arch = "riscv64"
case strings.Contains(lower, "s390x"):
r.arch = "s390x"
case strings.Contains(lower, "loong64"):
r.arch = "loong64"
}
// Libc detection
switch {
case strings.Contains(lower, "musl"):
r.libc = "musl"
case strings.Contains(lower, "gnu"):
r.libc = "gnu"
case strings.Contains(lower, "msvc"):
r.libc = "msvc"
}
return r
}
func detectFormat(name string) string {
lower := strings.ToLower(name)
for _, ext := range []string{".tar.gz", ".tar.xz", ".tar.bz2", ".tar.zst", ".exe.xz", ".app.zip"} {
if strings.HasSuffix(lower, ext) {
return ext
}
}
// .tgz is a common alias for .tar.gz
if strings.HasSuffix(lower, ".tgz") {
return ".tar.gz"
}
return filepath.Ext(lower)
}
// detectExtra identifies GPU/vendor-specific variant suffixes in filenames
// like "ollama-linux-amd64-rocm.tar.zst" or "ollama-linux-arm64-jetpack5.tar.zst".
func detectExtra(name string) string {
lower := strings.ToLower(name)
for _, variant := range []string{
"-rocm", "-jetpack", "-cuda", "-vulkan", "-metal",
"-extended", "-static", "-debug", "-nightly",
} {
if strings.Contains(lower, variant) {
return strings.TrimPrefix(variant, "-")
}
}
return ""
}
func isMetaAsset(name string) bool {
lower := strings.ToLower(name)
for _, suffix := range []string{
".sha256", ".sha256sum", ".sha512", ".sha512sum",
".md5", ".md5sum", ".sig", ".asc", ".pem",
"checksums.txt", "sha256sums", "sha512sums",
".sbom", ".spdx", ".json.sig", ".sigstore",
".d.ts", ".pub",
} {
if strings.HasSuffix(lower, suffix) {
return true
}
}
for _, contains := range []string{
"checksums", "sha256sum", "sha512sum",
"buildable-artifact",
} {
if strings.Contains(lower, contains) {
return true
}
}
for _, exact := range []string{
"install.sh", "install.ps1", "compat.json",
} {
if lower == exact {
return true
}
}
return false
}

View File

@@ -1,862 +0,0 @@
// Command fetchraw fetches release histories from upstream APIs and
// merges them into rawcache. Safe to run repeatedly — unchanged releases
// are skipped, new/changed ones are recorded in the audit log.
//
// Reads releases.conf files from package directories to discover what
// to fetch. Adding a new package is just creating a conf file.
//
// Usage:
//
// go run ./cmd/fetchraw -cache ./_cache/raw
// go run ./cmd/fetchraw -cache ./_cache/raw hugo caddy
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"log"
"net/http"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/webinstall/webi-installers/internal/installerconf"
"github.com/webinstall/webi-installers/internal/lexver"
"github.com/webinstall/webi-installers/internal/rawcache"
"github.com/webinstall/webi-installers/internal/releases/chromedist"
"github.com/webinstall/webi-installers/internal/releases/flutterdist"
"github.com/webinstall/webi-installers/internal/releases/gitea"
"github.com/webinstall/webi-installers/internal/releases/github"
"github.com/webinstall/webi-installers/internal/releases/githubish"
"github.com/webinstall/webi-installers/internal/releases/gittag"
"github.com/webinstall/webi-installers/internal/releases/golang"
"github.com/webinstall/webi-installers/internal/releases/gpgdist"
"github.com/webinstall/webi-installers/internal/releases/hashicorp"
"github.com/webinstall/webi-installers/internal/releases/iterm2dist"
"github.com/webinstall/webi-installers/internal/releases/juliadist"
"github.com/webinstall/webi-installers/internal/releases/mariadbdist"
"github.com/webinstall/webi-installers/internal/releases/nodedist"
"github.com/webinstall/webi-installers/internal/releases/zigdist"
)
func main() {
cacheDir := flag.String("cache", "_cache/raw", "root directory for raw cache")
confDir := flag.String("conf", ".", "root directory containing {pkg}/releases.conf files")
token := flag.String("token", os.Getenv("GITHUB_TOKEN"), "GitHub API token")
flag.Parse()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
client := &http.Client{Timeout: 30 * time.Second}
var auth *githubish.Auth
if *token != "" {
auth = &githubish.Auth{Token: *token}
}
// Discover packages from releases.conf files.
packages, err := discover(*confDir)
if err != nil {
log.Fatalf("discover: %v", err)
}
// Filter to requested packages if args given.
args := flag.Args()
if len(args) > 0 {
nameSet := make(map[string]bool, len(args))
for _, a := range args {
nameSet[a] = true
}
var filtered []pkgConf
for _, p := range packages {
if nameSet[p.name] {
filtered = append(filtered, p)
}
}
packages = filtered
}
log.Printf("found %d packages", len(packages))
for _, pkg := range packages {
// Aliases share cache with their target — skip fetching.
if alias := pkg.conf.Extra["alias_of"]; alias != "" {
log.Printf(" %s: alias of %s, skipping", pkg.name, alias)
continue
}
log.Printf("fetching %s...", pkg.name)
var err error
switch pkg.conf.Source {
case "github":
err = fetchGitHub(ctx, client, *cacheDir, pkg.name, pkg.conf, auth)
case "nodedist":
err = fetchNodeDist(ctx, client, *cacheDir, pkg.name, pkg.conf)
case "golang":
err = fetchGolang(ctx, client, *cacheDir, pkg.name)
case "zigdist":
err = fetchZig(ctx, client, *cacheDir, pkg.name)
case "flutterdist":
err = fetchFlutter(ctx, client, *cacheDir, pkg.name)
case "iterm2dist":
err = fetchITerm2(ctx, client, *cacheDir, pkg.name)
case "hashicorp":
err = fetchHashiCorp(ctx, client, *cacheDir, pkg.name, pkg.conf)
case "juliadist":
err = fetchJulia(ctx, client, *cacheDir, pkg.name)
case "gittag":
err = fetchGitTag(ctx, *cacheDir, pkg.name, pkg.conf)
case "gitea":
err = fetchGitea(ctx, client, *cacheDir, pkg.name, pkg.conf)
case "chromedist":
err = fetchChrome(ctx, client, *cacheDir, pkg.name)
case "gpgdist":
err = fetchGPG(ctx, client, *cacheDir, pkg.name)
case "mariadbdist":
err = fetchMariaDB(ctx, client, *cacheDir, pkg.name)
default:
log.Printf(" %s: unknown source %q, skipping", pkg.name, pkg.conf.Source)
continue
}
if err != nil {
log.Printf(" ERROR: %s: %v", pkg.name, err)
}
}
}
type pkgConf struct {
name string
conf *installerconf.Conf
}
// discover finds all {dir}/*/releases.conf files and returns them sorted.
func discover(dir string) ([]pkgConf, error) {
pattern := filepath.Join(dir, "*", "releases.conf")
matches, err := filepath.Glob(pattern)
if err != nil {
return nil, err
}
var packages []pkgConf
for _, path := range matches {
name := filepath.Base(filepath.Dir(path))
// Skip infrastructure dirs (_example, _webi, _common, etc.)
if strings.HasPrefix(name, "_") {
continue
}
conf, err := installerconf.Read(path)
if err != nil {
log.Printf("warning: %s: %v", path, err)
continue
}
packages = append(packages, pkgConf{name: name, conf: conf})
}
sort.Slice(packages, func(i, j int) bool {
return packages[i].name < packages[j].name
})
return packages, nil
}
func fetchNodeDist(ctx context.Context, client *http.Client, cacheRoot, pkgName string, conf *installerconf.Conf) error {
baseURL := conf.BaseURL
if baseURL == "" {
return fmt.Errorf("missing url in releases.conf")
}
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range nodedist.Fetch(ctx, client, baseURL) {
if err != nil {
return fmt.Errorf("%s fetch: %w", pkgName, err)
}
for _, entry := range batch {
tag := entry.Version
data, err := json.Marshal(entry)
if err != nil {
return fmt.Errorf("%s marshal %s: %w", pkgName, tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if latest == "" {
latest = tag
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchGitHub(ctx context.Context, client *http.Client, cacheRoot, pkgName string, conf *installerconf.Conf, auth *githubish.Auth) error {
owner := conf.Owner
repo := conf.Repo
tagPrefix := conf.TagPrefix
if owner == "" || repo == "" {
return fmt.Errorf("missing owner or repo in releases.conf")
}
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range github.Fetch(ctx, client, owner, repo, auth) {
if err != nil {
return fmt.Errorf("github %s/%s: %w", owner, repo, err)
}
for _, rel := range batch {
if rel.Draft {
continue
}
tag := rel.TagName
if tagPrefix != "" {
if !strings.HasPrefix(tag, tagPrefix) {
continue
}
tag = strings.TrimPrefix(tag, tagPrefix)
}
data, err := json.Marshal(rel)
if err != nil {
return fmt.Errorf("marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if latest == "" && !rel.Prerelease {
latest = tag
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func updateLatest(d *rawcache.Dir, candidate string) error {
if candidate == "" {
return nil
}
current := d.Latest()
if current == "" || lexver.Compare(lexver.Parse(candidate), lexver.Parse(current)) > 0 {
return d.SetLatest(candidate)
}
return nil
}
func fetchGolang(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range golang.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("golang: %w", err)
}
for _, rel := range batch {
tag := rel.Version // "go1.24.1"
data, err := json.Marshal(rel)
if err != nil {
return fmt.Errorf("golang marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if latest == "" && rel.Stable {
latest = tag
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchZig(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range zigdist.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("zigdist: %w", err)
}
for _, rel := range batch {
tag := rel.Version
data, err := json.Marshal(rel)
if err != nil {
return fmt.Errorf("zigdist marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
// Stable versions have dots and no dev/pre markers.
isStable := strings.Contains(tag, ".") && !strings.ContainsAny(tag, "+-")
if isStable {
if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
latest = tag
}
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchFlutter(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range flutterdist.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("flutterdist: %w", err)
}
for _, rel := range batch {
// Use version+channel+os+arch as the tag. The arch is embedded
// in the archive path (e.g. flutter_macos_arm64_3.0.0-stable.zip
// vs flutter_macos_3.0.0-stable.zip for universal/x64).
arch := ""
base := filepath.Base(rel.Archive)
prefix := "flutter_" + rel.OS + "_"
if after, ok := strings.CutPrefix(base, prefix); ok {
if !strings.HasPrefix(after, rel.Version) {
// There's an arch segment between OS and version.
if idx := strings.Index(after, "_"); idx > 0 {
arch = after[:idx]
}
}
}
tag := fmt.Sprintf("%s-%s-%s", rel.Version, rel.Channel, rel.OS)
if arch != "" {
tag += "-" + arch
}
data, err := json.Marshal(rel)
if err != nil {
return fmt.Errorf("flutterdist marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if latest == "" && rel.Channel == "stable" {
latest = tag
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchITerm2(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range iterm2dist.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("iterm2dist: %w", err)
}
for _, entry := range batch {
tag := entry.Version
if tag == "" {
continue
}
data, err := json.Marshal(entry)
if err != nil {
return fmt.Errorf("iterm2dist marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if latest == "" && entry.Channel == "stable" {
latest = tag
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchHashiCorp(ctx context.Context, client *http.Client, cacheRoot, pkgName string, conf *installerconf.Conf) error {
product := conf.Extra["product"]
if product == "" {
return fmt.Errorf("missing product in releases.conf")
}
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for idx, err := range hashicorp.Fetch(ctx, client, product) {
if err != nil {
return fmt.Errorf("hashicorp %s: %w", product, err)
}
for tag, ver := range idx.Versions {
data, err := json.Marshal(ver)
if err != nil {
return fmt.Errorf("hashicorp marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
// Stable = no prerelease markers. Compare all to find highest.
isStable := !strings.ContainsAny(tag, "-+")
if isStable {
if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
latest = tag
}
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchJulia(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range juliadist.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("juliadist: %w", err)
}
for _, rel := range batch {
tag := rel.Version
data, err := json.Marshal(rel)
if err != nil {
return fmt.Errorf("juliadist marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if rel.Stable {
if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
latest = tag
}
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchGitTag(ctx context.Context, cacheRoot, pkgName string, conf *installerconf.Conf) error {
gitURL := conf.BaseURL
if gitURL == "" {
return fmt.Errorf("missing url in releases.conf")
}
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
repoDir := filepath.Join(cacheRoot, "_repos")
if err := os.MkdirAll(repoDir, 0o755); err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range gittag.Fetch(ctx, gitURL, repoDir) {
if err != nil {
return fmt.Errorf("gittag %s: %w", pkgName, err)
}
for _, entry := range batch {
tag := entry.Version
if tag == "" {
tag = "HEAD-" + entry.CommitHash
}
data, err := json.Marshal(entry)
if err != nil {
return fmt.Errorf("gittag marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if entry.GitTag != "" && entry.GitTag != "HEAD" {
if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
latest = tag
}
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchGitea(ctx context.Context, client *http.Client, cacheRoot, pkgName string, conf *installerconf.Conf) error {
baseURL := conf.BaseURL
owner := conf.Owner
repo := conf.Repo
if baseURL == "" || owner == "" || repo == "" {
return fmt.Errorf("missing base_url, owner, or repo in releases.conf")
}
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range gitea.Fetch(ctx, client, baseURL, owner, repo, nil) {
if err != nil {
return fmt.Errorf("gitea %s/%s: %w", owner, repo, err)
}
for _, rel := range batch {
if rel.Draft {
continue
}
tag := rel.TagName
data, err := json.Marshal(rel)
if err != nil {
return fmt.Errorf("gitea marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if latest == "" && !rel.Prerelease {
latest = tag
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchChrome(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range chromedist.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("chromedist: %w", err)
}
for _, ver := range batch {
tag := ver.Version
data, err := json.Marshal(ver)
if err != nil {
return fmt.Errorf("chromedist marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
latest = tag
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchGPG(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range gpgdist.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("gpgdist: %w", err)
}
for _, entry := range batch {
tag := entry.Version
data, err := json.Marshal(entry)
if err != nil {
return fmt.Errorf("gpgdist marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
latest = tag
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchMariaDB(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range mariadbdist.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("mariadbdist: %w", err)
}
for _, rel := range batch {
tag := rel.ReleaseID
data, err := json.Marshal(rel)
if err != nil {
return fmt.Errorf("mariadbdist marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
isStable := rel.MajorStatus == "Stable"
if isStable {
if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
latest = tag
}
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}

View File

@@ -1,625 +0,0 @@
// Command inspect downloads release archives, unpacks them, and reports
// their internal structure. This helps discover how packages are laid out
// and whether the layout changes across versions.
//
// Usage:
//
// go run ./cmd/inspect -csv distributables.csv -cache ./_cache/downloads ollama sd
package main
import (
"context"
"encoding/csv"
"flag"
"fmt"
"io"
"log"
"mime"
"net/http"
"os"
"os/exec"
"path/filepath"
"sort"
"strings"
"time"
"github.com/webinstall/webi-installers/internal/httpclient"
)
// Row is one CSV row from distributables.csv.
type Row struct {
Package string
Version string
Channel string
Date string
OS string
Arch string
Libc string
Format string
Download string
Filename string
Extra string
}
// archiveFormats are the formats we download and unpack.
var archiveFormats = map[string]bool{
".tar.gz": true,
".tar.xz": true,
".tar.bz2": true,
".tar.zst": true,
".zip": true,
".dmg": true,
".gz": true,
".xz": true,
}
// inspectOSes are the OSes we inspect.
var inspectOSes = map[string]bool{
"linux": true,
"darwin": true,
"windows": true,
"": true, // source-only packages
}
// preferredArch picks one arch per OS to download.
func preferredArch(os_ string) string {
switch os_ {
case "darwin":
return "aarch64"
default:
return "x86_64"
}
}
func main() {
csvFile := flag.String("csv", "distributables.csv", "path to distributables CSV")
cacheDir := flag.String("cache", "_cache/downloads", "download cache directory")
flag.Parse()
packages := flag.Args()
if len(packages) == 0 {
log.Fatal("usage: inspect [-csv FILE] [-cache DIR] PACKAGE [PACKAGE...]")
}
rows, err := readCSV(*csvFile)
if err != nil {
log.Fatalf("read csv: %v", err)
}
client := httpclient.New()
// Override timeout for large downloads.
client.Timeout = 10 * time.Minute
for _, pkg := range packages {
log.Printf("=== %s ===", pkg)
if err := inspectPackage(client, rows, pkg, *cacheDir); err != nil {
log.Printf("ERROR: %s: %v", pkg, err)
}
}
}
func readCSV(path string) ([]Row, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
r := csv.NewReader(f)
header, err := r.Read()
if err != nil {
return nil, err
}
// Build column index.
idx := make(map[string]int, len(header))
for i, col := range header {
idx[col] = i
}
var rows []Row
for {
record, err := r.Read()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
get := func(col string) string {
if i, ok := idx[col]; ok && i < len(record) {
return record[i]
}
return ""
}
rows = append(rows, Row{
Package: get("package"),
Version: get("version"),
Channel: get("channel"),
Date: get("date"),
OS: get("os"),
Arch: get("arch"),
Libc: get("libc"),
Format: get("format"),
Download: get("download"),
Filename: get("filename"),
Extra: get("extra"),
})
}
return rows, nil
}
func inspectPackage(client *http.Client, allRows []Row, pkg, cacheDir string) error {
// Filter rows for this package.
var pkgRows []Row
for _, r := range allRows {
if r.Package == pkg {
pkgRows = append(pkgRows, r)
}
}
if len(pkgRows) == 0 {
return fmt.Errorf("no rows found")
}
// Find latest stable version, fall back to any version.
versions := findVersionsByDate(pkgRows)
if len(versions) == 0 {
return fmt.Errorf("no versions found")
}
latestVer := versions[0]
log.Printf(" latest version: %s", latestVer)
// Check if latest has assets uploaded (more than just source tarballs).
latestRows := filterVersion(pkgRows, latestVer)
hasRealAssets := false
for _, r := range latestRows {
if r.Extra != "source" && archiveFormats[r.Format] {
hasRealAssets = true
break
}
}
// If latest looks empty, step back one version.
if !hasRealAssets && len(versions) > 1 {
latestVer = versions[1]
latestRows = filterVersion(pkgRows, latestVer)
log.Printf(" latest has no assets, using: %s", latestVer)
}
// Inspect the latest version.
if err := inspectVersion(client, pkg, latestVer, latestRows, cacheDir); err != nil {
return err
}
// Find versions roughly a year apart going back.
yearVersions := findYearlyVersions(pkgRows, latestVer)
for _, v := range yearVersions {
log.Printf(" --- checking %s ---", v)
vRows := filterVersion(pkgRows, v)
if err := inspectVersion(client, pkg, v, vRows, cacheDir); err != nil {
log.Printf(" ERROR: %v", err)
}
}
return nil
}
// findVersionsByDate returns versions sorted newest first, preferring stable.
func findVersionsByDate(rows []Row) []string {
type vInfo struct {
version string
date string
stable bool
}
seen := map[string]*vInfo{}
for _, r := range rows {
if _, ok := seen[r.Version]; !ok {
seen[r.Version] = &vInfo{
version: r.Version,
date: r.Date,
stable: r.Channel == "stable",
}
}
}
var vs []*vInfo
for _, v := range seen {
vs = append(vs, v)
}
// Sort: stable first, then by date descending, then version descending.
sort.Slice(vs, func(i, j int) bool {
if vs[i].stable != vs[j].stable {
return vs[i].stable
}
if vs[i].date != vs[j].date {
return vs[i].date > vs[j].date
}
return vs[i].version > vs[j].version
})
result := make([]string, len(vs))
for i, v := range vs {
result[i] = v.version
}
return result
}
// findYearlyVersions picks versions roughly a year apart before the given version.
func findYearlyVersions(rows []Row, latestVer string) []string {
// Find the date of latest.
var latestDate string
for _, r := range rows {
if r.Version == latestVer && r.Date != "" {
latestDate = r.Date
break
}
}
if latestDate == "" {
return nil
}
latestTime, err := time.Parse("2006-01-02", latestDate)
if err != nil {
return nil
}
// Collect all stable versions with dates.
type vd struct {
version string
date time.Time
}
seen := map[string]bool{}
var all []vd
for _, r := range rows {
if seen[r.Version] || r.Date == "" || r.Channel != "stable" {
continue
}
seen[r.Version] = true
t, err := time.Parse("2006-01-02", r.Date)
if err != nil {
continue
}
if t.Before(latestTime) {
all = append(all, vd{r.Version, t})
}
}
sort.Slice(all, func(i, j int) bool {
return all[i].date.After(all[j].date)
})
// Pick versions roughly a year apart.
var result []string
nextTarget := latestTime.AddDate(-1, 0, 0)
for _, v := range all {
if v.date.Before(nextTarget) || v.date.Equal(nextTarget) {
result = append(result, v.version)
nextTarget = v.date.AddDate(-1, 0, 0)
}
}
return result
}
func filterVersion(rows []Row, version string) []Row {
var result []Row
for _, r := range rows {
if r.Version == version {
result = append(result, r)
}
}
return result
}
// inspectVersion downloads and inspects archives for one version.
func inspectVersion(client *http.Client, pkg, version string, rows []Row, cacheDir string) error {
// Group by OS, pick one arch per OS, pick distinct formats.
type dlKey struct {
os_ string
format string
}
selected := map[dlKey]*Row{}
for i := range rows {
r := &rows[i]
if !inspectOSes[r.OS] {
continue
}
if !archiveFormats[r.Format] {
continue
}
key := dlKey{r.OS, r.Format}
existing := selected[key]
if existing == nil {
selected[key] = r
continue
}
// Prefer the preferred arch.
pref := preferredArch(r.OS)
if r.Arch == pref && existing.Arch != pref {
selected[key] = r
}
// Skip rocm/jetpack variants.
if strings.Contains(r.Filename, "rocm") || strings.Contains(r.Filename, "jetpack") {
if !strings.Contains(existing.Filename, "rocm") && !strings.Contains(existing.Filename, "jetpack") {
continue // keep existing non-special variant
}
}
}
if len(selected) == 0 {
log.Printf(" %s: no downloadable archives", version)
return nil
}
// Sort keys for deterministic output.
var keys []dlKey
for k := range selected {
keys = append(keys, k)
}
sort.Slice(keys, func(i, j int) bool {
if keys[i].os_ != keys[j].os_ {
return keys[i].os_ < keys[j].os_
}
return keys[i].format < keys[j].format
})
for _, key := range keys {
r := selected[key]
os_ := r.OS
if os_ == "" {
os_ = "any"
}
log.Printf(" [%s] %s %s → %s", version, os_, r.Format, r.Filename)
dlPath, err := download(client, r.Download, r.Filename, filepath.Join(cacheDir, pkg, version))
if err != nil {
log.Printf(" download error: %v", err)
continue
}
contents, err := unpackAndList(dlPath, r.Format)
if err != nil {
log.Printf(" unpack error: %v", err)
continue
}
printContents(contents)
}
return nil
}
// download fetches a URL to the cache dir. Returns the path to the cached file.
// Skips download if the file already exists.
func download(client *http.Client, url, hintFilename, dir string) (string, error) {
// Check if already cached by hint filename.
cached := filepath.Join(dir, hintFilename)
if _, err := os.Stat(cached); err == nil {
return cached, nil
}
if err := os.MkdirAll(dir, 0o755); err != nil {
return "", err
}
ctx := context.Background()
resp, err := httpclient.Get(ctx, client, url)
if err != nil {
return "", fmt.Errorf("GET %s: %w", url, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("GET %s: %s", url, resp.Status)
}
// Determine filename from Content-Disposition or hint.
filename := hintFilename
if cd := resp.Header.Get("Content-Disposition"); cd != "" {
_, params, err := mime.ParseMediaType(cd)
if err == nil {
if fn, ok := params["filename"]; ok && fn != "" {
filename = fn
}
}
}
outPath := filepath.Join(dir, filename)
// Atomic write: temp file + rename.
tmp := outPath + ".tmp"
f, err := os.Create(tmp)
if err != nil {
return "", err
}
n, err := io.Copy(f, resp.Body)
if closeErr := f.Close(); closeErr != nil && err == nil {
err = closeErr
}
if err != nil {
os.Remove(tmp)
return "", fmt.Errorf("download %s: %w", url, err)
}
if err := os.Rename(tmp, outPath); err != nil {
os.Remove(tmp)
return "", err
}
log.Printf(" downloaded %s (%d bytes)", filename, n)
return outPath, nil
}
// FileEntry describes one file inside an archive.
type FileEntry struct {
Path string
Size int64
Mode os.FileMode
IsDir bool
IsExec bool
IsSymlink bool
LinkTarget string
}
// unpackAndList extracts an archive to a temp dir and lists contents.
func unpackAndList(archivePath, format string) ([]FileEntry, error) {
tmpDir, err := os.MkdirTemp("", "webi-inspect-*")
if err != nil {
return nil, err
}
defer os.RemoveAll(tmpDir)
switch format {
case ".tar.gz":
err = run("tar", "xzf", archivePath, "-C", tmpDir)
case ".tar.xz":
err = run("tar", "xJf", archivePath, "-C", tmpDir)
case ".tar.bz2":
err = run("tar", "xjf", archivePath, "-C", tmpDir)
case ".tar.zst":
err = run("tar", "--zstd", "-xf", archivePath, "-C", tmpDir)
case ".zip":
err = run("unzip", "-q", "-o", archivePath, "-d", tmpDir)
case ".dmg":
err = extractDMG(archivePath, tmpDir)
case ".gz":
// Single file gzip.
base := filepath.Base(archivePath)
base = strings.TrimSuffix(base, ".gz")
outPath := filepath.Join(tmpDir, base)
err = run("sh", "-c", fmt.Sprintf("gunzip -c %q > %q", archivePath, outPath))
case ".xz":
base := filepath.Base(archivePath)
base = strings.TrimSuffix(base, ".xz")
outPath := filepath.Join(tmpDir, base)
err = run("sh", "-c", fmt.Sprintf("xz -dc %q > %q", archivePath, outPath))
default:
return nil, fmt.Errorf("unsupported format: %s", format)
}
if err != nil {
return nil, fmt.Errorf("extract %s: %w", format, err)
}
return listDir(tmpDir, "")
}
func extractDMG(dmgPath, outDir string) error {
// Try 7z first (doesn't require mounting).
if _, err := exec.LookPath("7z"); err == nil {
return run("7z", "x", "-o"+outDir, dmgPath)
}
// Fall back to hdiutil mount + copy + unmount.
mountPoint, err := os.MkdirTemp("", "webi-dmg-*")
if err != nil {
return err
}
defer os.RemoveAll(mountPoint)
if err := run("hdiutil", "attach", dmgPath, "-mountpoint", mountPoint, "-nobrowse", "-quiet"); err != nil {
return fmt.Errorf("mount dmg: %w", err)
}
defer run("hdiutil", "detach", mountPoint, "-quiet")
// Copy contents.
return run("cp", "-R", mountPoint+"/.", outDir)
}
func run(name string, args ...string) error {
cmd := exec.Command(name, args...)
cmd.Stderr = os.Stderr
return cmd.Run()
}
func listDir(root, prefix string) ([]FileEntry, error) {
entries, err := os.ReadDir(filepath.Join(root, prefix))
if err != nil {
return nil, err
}
var result []FileEntry
for _, e := range entries {
relPath := filepath.Join(prefix, e.Name())
fullPath := filepath.Join(root, relPath)
info, err := e.Info()
if err != nil {
continue
}
entry := FileEntry{
Path: relPath,
Size: info.Size(),
Mode: info.Mode(),
IsDir: e.IsDir(),
}
if info.Mode()&os.ModeSymlink != 0 {
entry.IsSymlink = true
target, _ := os.Readlink(fullPath)
entry.LinkTarget = target
}
if !e.IsDir() && info.Mode()&0o111 != 0 {
entry.IsExec = true
}
result = append(result, entry)
if e.IsDir() {
sub, err := listDir(root, relPath)
if err != nil {
continue
}
result = append(result, sub...)
}
}
return result, nil
}
func printContents(entries []FileEntry) {
for _, e := range entries {
marker := " "
if e.IsExec {
marker = "* "
}
if e.IsDir {
marker = "d "
}
if e.IsSymlink {
marker = "→ "
}
size := ""
if !e.IsDir {
size = formatSize(e.Size)
}
line := fmt.Sprintf(" %s%-50s %8s", marker, e.Path, size)
if e.IsSymlink {
line += " → " + e.LinkTarget
}
log.Print(line)
}
}
func formatSize(n int64) string {
switch {
case n >= 1<<30:
return fmt.Sprintf("%.1fG", float64(n)/float64(1<<30))
case n >= 1<<20:
return fmt.Sprintf("%.1fM", float64(n)/float64(1<<20))
case n >= 1<<10:
return fmt.Sprintf("%.1fK", float64(n)/float64(1<<10))
default:
return fmt.Sprintf("%dB", n)
}
}

View File

@@ -1,356 +0,0 @@
// Command uaparse analyzes User-Agent strings from webi.sh logs.
//
// It reads UA strings (one per line) from stdin or a file, parses each
// through uadetect, and produces summary output showing:
// - unique platform tuples (os, arch, libc) with counts
// - platform hints extracted from kernel version strings (cloud provider,
// container runtime, device info)
// - detection failures and malformed UAs
//
// Usage:
//
// uaparse < LIVE-UAS.txt
// uaparse LIVE-UAS.txt
// uaparse -json LIVE-UAS.txt
// uaparse -fixtures LIVE-UAS.txt # output Go test fixtures
package main
import (
"bufio"
"encoding/json"
"flag"
"fmt"
"os"
"regexp"
"sort"
"strings"
"github.com/webinstall/webi-installers/internal/uadetect"
)
// PlatformKey is the resolution-relevant tuple — everything else is noise
// for artifact selection.
type PlatformKey struct {
OS string `json:"os"`
Arch string `json:"arch"`
Libc string `json:"libc"`
}
func (k PlatformKey) String() string {
return fmt.Sprintf("%-10s %-10s %s", k.OS, k.Arch, k.Libc)
}
// PlatformEntry holds a unique platform and its metadata.
type PlatformEntry struct {
Key PlatformKey `json:"key"`
Count int `json:"count"`
Examples []string `json:"examples"` // up to 3 representative UAs
Hints []string `json:"hints"` // unique platform hints seen
}
// UAIssue records a malformed or undetectable UA.
type UAIssue struct {
Line int `json:"line"`
UA string `json:"ua"`
Reason string `json:"reason"`
}
// Hint is a platform detail extracted from the kernel version string.
type Hint struct {
Tag string // short label: "amzn", "azure", "gcp", "wsl", etc.
Count int
}
func main() {
jsonOut := flag.Bool("json", false, "output as JSON")
fixtures := flag.Bool("fixtures", false, "output Go test fixture table")
flag.Parse()
var scanner *bufio.Scanner
if flag.NArg() > 0 {
f, err := os.Open(flag.Arg(0))
if err != nil {
fmt.Fprintf(os.Stderr, "uaparse: %v\n", err)
os.Exit(1)
}
defer f.Close()
scanner = bufio.NewScanner(f)
} else {
scanner = bufio.NewScanner(os.Stdin)
}
platforms := make(map[PlatformKey]*PlatformEntry)
hints := make(map[string]int)
var issues []UAIssue
lineNum := 0
for scanner.Scan() {
lineNum++
ua := strings.TrimSpace(scanner.Text())
if ua == "" {
continue
}
// Detect corruption: truncated/double-pasted lines.
if isMalformed(ua) {
issues = append(issues, UAIssue{
Line: lineNum,
UA: ua,
Reason: "malformed (truncated or corrupted)",
})
continue
}
// Parse through uadetect.
result := uadetect.Parse(ua)
// Check for detection failures.
if result.OS == "" {
issues = append(issues, UAIssue{
Line: lineNum,
UA: ua,
Reason: "OS not detected",
})
}
if result.Arch == "" {
issues = append(issues, UAIssue{
Line: lineNum,
UA: ua,
Reason: "arch not detected",
})
}
key := PlatformKey{
OS: string(result.OS),
Arch: string(result.Arch),
Libc: string(result.Libc),
}
entry, ok := platforms[key]
if !ok {
entry = &PlatformEntry{Key: key}
platforms[key] = entry
}
entry.Count++
if len(entry.Examples) < 3 {
entry.Examples = append(entry.Examples, ua)
}
// Extract platform hints from kernel version.
for _, h := range extractHints(ua) {
if !containsStr(entry.Hints, h) {
entry.Hints = append(entry.Hints, h)
}
hints[h]++
}
}
if err := scanner.Err(); err != nil {
fmt.Fprintf(os.Stderr, "uaparse: read error: %v\n", err)
os.Exit(1)
}
// Sort platforms by count descending.
entries := make([]*PlatformEntry, 0, len(platforms))
for _, e := range platforms {
entries = append(entries, e)
}
sort.Slice(entries, func(i, j int) bool {
return entries[i].Count > entries[j].Count
})
if *jsonOut {
outputJSON(entries, issues, hints)
} else if *fixtures {
outputFixtures(entries)
} else {
outputTable(entries, issues, hints, lineNum)
}
}
func outputTable(entries []*PlatformEntry, issues []UAIssue, hints map[string]int, total int) {
fmt.Printf("=== UA Analysis: %d lines → %d unique platforms ===\n\n", total, len(entries))
fmt.Printf("%-10s %-10s %-6s %6s %s\n", "OS", "ARCH", "LIBC", "COUNT", "HINTS")
fmt.Println(strings.Repeat("-", 72))
for _, e := range entries {
hintStr := ""
if len(e.Hints) > 0 {
hintStr = strings.Join(e.Hints, ", ")
}
fmt.Printf("%-10s %-10s %-6s %6d %s\n",
displayOS(e.Key.OS), e.Key.Arch, displayLibc(e.Key.Libc),
e.Count, hintStr)
}
if len(hints) > 0 {
fmt.Printf("\n=== Platform Hints (environment signals from kernel strings) ===\n\n")
sortedHints := make([]Hint, 0, len(hints))
for tag, count := range hints {
sortedHints = append(sortedHints, Hint{tag, count})
}
sort.Slice(sortedHints, func(i, j int) bool {
return sortedHints[i].Count > sortedHints[j].Count
})
for _, h := range sortedHints {
fmt.Printf(" %-20s %d\n", h.Tag, h.Count)
}
}
if len(issues) > 0 {
fmt.Printf("\n=== Issues (%d) ===\n\n", len(issues))
for _, iss := range issues {
fmt.Printf(" line %d: %s\n %s\n", iss.Line, iss.Reason, iss.UA)
}
}
}
func outputJSON(entries []*PlatformEntry, issues []UAIssue, hints map[string]int) {
out := struct {
Platforms []*PlatformEntry `json:"platforms"`
Issues []UAIssue `json:"issues"`
Hints map[string]int `json:"hints"`
}{entries, issues, hints}
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
_ = enc.Encode(out)
}
func outputFixtures(entries []*PlatformEntry) {
fmt.Println("// Generated by cmd/uaparse from live UA data.")
fmt.Println("// Each entry represents a unique (os, arch, libc) platform seen in production.")
fmt.Println("var liveUAPlatforms = []struct {")
fmt.Println("\tua string")
fmt.Println("\tos buildmeta.OS")
fmt.Println("\tarch buildmeta.Arch")
fmt.Println("\tlibc buildmeta.Libc")
fmt.Println("}{")
for _, e := range entries {
if e.Key.OS == "" || e.Key.Arch == "" {
continue // skip undetectable
}
ua := e.Examples[0]
fmt.Printf("\t{%q, %s, %s, %s},\n",
ua, goConst("OS", e.Key.OS), goConst("Arch", e.Key.Arch), goConst("Libc", e.Key.Libc))
}
fmt.Println("}")
}
// isMalformed checks for genuinely corrupted UA strings (network truncation).
func isMalformed(ua string) bool {
// Extremely short (less than 10 chars) suggests truncation.
if len(ua) < 10 {
return true
}
return false
}
// extractHints finds environment signals in a UA string.
func extractHints(ua string) []string {
lower := strings.ToLower(ua)
var out []string
patterns := []struct {
substr string
tag string
}{
{"amzn", "amzn"}, // Amazon Linux
{"-azure", "azure"}, // Azure VM
{"-gcp", "gcp"}, // Google Cloud
{"-aws", "aws"}, // AWS kernel
{"-oracle", "oracle"}, // Oracle Cloud
{"el7", "rhel7"}, // RHEL/CentOS 7
{"el8", "rhel8"}, // RHEL/CentOS 8
{"el9", "rhel9"}, // RHEL/CentOS 9
{".fc", "fedora"}, // Fedora
{"+deb", "debian"}, // Debian
{"-generic", "ubuntu"}, // Ubuntu generic kernel
{"-pve", "proxmox"}, // Proxmox VE
{"linuxkit", "docker"}, // Docker Desktop / linuxkit
{"orbstack", "orbstack"},
{"microsoft-standard-wsl", "wsl"},
{"android", "android"},
{"+rpt-rpi", "rpi"}, // Raspberry Pi
{"cygwin", "cygwin"},
{"mingw", "mingw"},
{"msys", "msys"},
{"freebsd", "freebsd"},
{"-nvidia", "nvidia"},
{"gentoo", "gentoo"},
{"coreweave", "coreweave"},
}
for _, p := range patterns {
if strings.Contains(lower, p.substr) {
out = append(out, p.tag)
}
}
return out
}
// androidDeviceRe extracts device/build info from Android kernel strings.
var androidDeviceRe = regexp.MustCompile(`ab[A-Z0-9]+`)
func displayOS(os string) string {
if os == "" {
return "(none)"
}
return os
}
func displayLibc(libc string) string {
if libc == "" {
return "(none)"
}
return libc
}
func goConst(prefix, val string) string {
m := map[string]map[string]string{
"OS": {
"darwin": "buildmeta.OSDarwin",
"linux": "buildmeta.OSLinux",
"windows": "buildmeta.OSWindows",
"freebsd": "buildmeta.OSFreeBSD",
"android": "buildmeta.OSAndroid",
"": `""`,
},
"Arch": {
"aarch64": "buildmeta.ArchARM64",
"x86_64": "buildmeta.ArchAMD64",
"armv7": "buildmeta.ArchARMv7",
"armv6": "buildmeta.ArchARMv6",
"x86": "buildmeta.ArchX86",
"ppc64le": "buildmeta.ArchPPC64LE",
"ppc64": "buildmeta.ArchPPC64",
"s390x": "buildmeta.ArchS390X",
"riscv64": "buildmeta.ArchRISCV64",
"": `""`,
},
"Libc": {
"gnu": "buildmeta.LibcGNU",
"musl": "buildmeta.LibcMusl",
"msvc": "buildmeta.LibcMSVC",
"none": "buildmeta.LibcNone",
"": `""`,
},
}
if v, ok := m[prefix][val]; ok {
return v
}
return fmt.Sprintf("%q /* unmapped */", val)
}
func containsStr(ss []string, s string) bool {
for _, v := range ss {
if v == s {
return true
}
}
return false
}

View File

@@ -54,7 +54,6 @@ import (
"github.com/webinstall/webi-installers/internal/releases/zigdist"
"github.com/webinstall/webi-installers/internal/storage"
"github.com/webinstall/webi-installers/internal/storage/fsstore"
"github.com/webinstall/webi-installers/internal/storage/pgstore"
)
var (
@@ -78,7 +77,6 @@ type MainConfig struct {
envFile string
confDir string
cacheDir string
pgDSN string
rawDir string
token string
once bool
@@ -92,7 +90,7 @@ type MainConfig struct {
// WebiCache holds the configuration for the cache daemon.
type WebiCache struct {
ConfDir string // root directory with {pkg}/releases.conf files
Store storage.Store // classified asset storage (fsstore or pgstore)
Store storage.Store // classified asset storage (fsstore)
RawDir string // raw upstream response cache
Client *http.Client // HTTP client for upstream calls
Auth *githubish.Auth // GitHub API auth (optional)
@@ -157,20 +155,11 @@ func main() {
cfg.token = os.Getenv("GITHUB_TOKEN")
}
var store storage.Store
if cfg.pgDSN != "" {
pg, err := pgstore.New(context.Background(), cfg.pgDSN)
if err != nil {
log.Fatalf("pgstore: %v", err)
}
store = pg
} else {
fs, err := fsstore.New(cfg.cacheDir)
if err != nil {
log.Fatalf("fsstore: %v", err)
}
store = fs
fss, err := fsstore.New(cfg.cacheDir)
if err != nil {
log.Fatalf("fsstore: %v", err)
}
var store storage.Store = fss
var auth *githubish.Auth
if cfg.token != "" {
@@ -271,7 +260,6 @@ func registerFlags(fs *flag.FlagSet, cfg *MainConfig) {
fs.StringVar(&cfg.envFile, "envfile", "", "path to .env file to load before running")
fs.StringVar(&cfg.confDir, "conf", ".", "root directory containing {pkg}/releases.conf files")
fs.StringVar(&cfg.cacheDir, "legacy", "~/.cache/webi/legacy", "legacy cache directory (fsstore root)")
fs.StringVar(&cfg.pgDSN, "pg", "", "PostgreSQL DSN (enables pgstore; mutually exclusive with -legacy)")
fs.StringVar(&cfg.rawDir, "raw", "~/.cache/webi/raw", "raw cache directory for upstream responses")
fs.StringVar(&cfg.token, "token", "", "GitHub API token (or set $GITHUB_TOKEN)")
fs.BoolVar(&cfg.once, "once", false, "run once then exit (no periodic refresh)")
@@ -543,14 +531,7 @@ func (wc *WebiCache) fetchRaw(ctx context.Context, pkg pkgConf, shallow bool) er
// commit hashes. Git entries are classified from this data in
// refreshPackage, not from the main raw cache.
if pkg.conf.GitURL != "" && pkg.conf.Source != "gittag" {
gitShallow := shallow
if !wc.Shallow {
gd, gdErr := rawcache.Open(filepath.Join(wc.RawDir, "_gittag", pkg.name))
if gdErr == nil && !gd.Populated() {
gitShallow = false
}
}
if err := wc.fetchGitTagSupplementary(ctx, pkg.name, pkg.conf.GitURL, gitShallow); err != nil {
if err := wc.fetchGitTagSupplementary(ctx, pkg.name, pkg.conf.GitURL, shallow); err != nil {
log.Printf(" %s: supplementary gittag fetch: %v", pkg.name, err)
}
}

View File

@@ -1,4 +1,3 @@
source = ffmpegdist
github_releases = eugeneware/ffmpeg-static
asset_filter = ffmpeg
version_prefix = b

12
go.mod
View File

@@ -2,14 +2,4 @@ module github.com/webinstall/webi-installers
go 1.26.1
require (
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/pgx/v5 v5.8.0 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/joho/godotenv v1.5.1 // indirect
github.com/jszwec/csvutil v1.10.0 // indirect
github.com/therootcompany/golib/http/middleware/v2 v2.0.1 // indirect
golang.org/x/sync v0.17.0 // indirect
golang.org/x/text v0.29.0 // indirect
)
require github.com/joho/godotenv v1.5.1

23
go.sum
View File

@@ -1,25 +1,2 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo=
github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/jszwec/csvutil v1.10.0 h1:upMDUxhQKqZ5ZDCs/wy+8Kib8rZR8I8lOR34yJkdqhI=
github.com/jszwec/csvutil v1.10.0/go.mod h1:/E4ONrmGkwmWsk9ae9jpXnv9QT8pLHEPcCirMFhxG9I=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/therootcompany/golib/http/middleware/v2 v2.0.1 h1:VNKpHcwyEW7cMct7/eO4fyrxwIQk2ycb6juVXSPs2Sk=
github.com/therootcompany/golib/http/middleware/v2 v2.0.1/go.mod h1:g5gb9qBidw74nW6/mwIauTKMpOKchiN2l0gt5qzJ2aQ=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -166,8 +166,6 @@ func classifySource(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]st
return classifyMariaDBDist(d)
case "zigdist":
return classifyZigDist(d)
case "ffmpegdist":
return classifyFFmpegDist(d)
default:
return nil, nil
}
@@ -466,86 +464,6 @@ func classifyGitHub(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]st
return assets, nil
}
var ffmpegOSMap = map[string]string{
"linux": "linux",
"darwin": "darwin",
"win32": "windows",
}
var ffmpegArchMap = map[string]string{
"x64": "x86_64",
"ia32": "x86",
"arm64": "aarch64",
"arm": "armv7",
}
// classifyFFmpegDist handles eugeneware/ffmpeg-static releases.
// Upstream uses non-standard names (x64, ia32, win32, arm) and ships both
// bare binaries and .gz-compressed copies. Only bare binaries are kept —
// the install template has no handler for single-file .gz extraction.
func classifyFFmpegDist(d *rawcache.Dir) ([]storage.Asset, error) {
releases, err := ReadAllRaw(d)
if err != nil {
return nil, err
}
var assets []storage.Asset
for _, data := range releases {
var rel ghRelease
if err := json.Unmarshal(data, &rel); err != nil {
continue
}
if rel.Draft {
continue
}
version := strings.TrimPrefix(rel.TagName, "b")
channel := "stable"
if rel.Prerelease {
channel = "beta"
}
date := ""
if len(rel.PublishedAt) >= 10 {
date = rel.PublishedAt[:10]
}
for _, a := range rel.Assets {
if strings.Contains(a.Name, ".") {
continue
}
if !strings.HasPrefix(a.Name, "ffmpeg-") {
continue
}
parts := strings.SplitN(a.Name, "-", 3)
if len(parts) != 3 {
continue
}
os, osOK := ffmpegOSMap[parts[1]]
arch, archOK := ffmpegArchMap[parts[2]]
if !osOK || !archOK {
continue
}
assets = append(assets, storage.Asset{
Filename: a.Name,
Version: version,
Channel: channel,
OS: os,
Arch: arch,
Format: "",
Download: a.BrowserDownloadURL,
Date: date,
})
}
}
return assets, nil
}
// classifyServiceman handles serviceman's dual-repo layout: binary releases
// from therootcompany/serviceman (≤v0.8.x) and source-only releases from
// bnnanet/serviceman (v0.9.x+). Emits binary assets where available, plus

View File

@@ -1,154 +0,0 @@
// Package httpclient provides a well-configured [http.Client] for upstream
// API calls. It exists because [http.DefaultClient] has no timeouts, no TLS
// minimum, and follows redirects from HTTPS to HTTP — none of which are
// acceptable for a server calling GitHub, Gitea, etc. on behalf of users.
//
// Use [New] to create a configured client. Use [Do] to execute a request
// with automatic retries for transient failures.
package httpclient
import (
"context"
"crypto/tls"
"errors"
"fmt"
"math/rand/v2"
"net"
"net/http"
"strconv"
"time"
)
const userAgent = "Webi/2.0 (+https://webinstall.dev)"
// New returns an [http.Client] with secure, production-ready defaults:
// TLS 1.2+, timeouts at every level, connection pooling, no HTTPS→HTTP
// redirect, and a Webi User-Agent.
func New() *http.Client {
return &http.Client{
Transport: &http.Transport{
DialContext: (&net.Dialer{
Timeout: 10 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
TLSClientConfig: &tls.Config{
MinVersion: tls.VersionTLS12,
},
TLSHandshakeTimeout: 10 * time.Second,
ResponseHeaderTimeout: 30 * time.Second,
MaxIdleConns: 100,
MaxIdleConnsPerHost: 10,
IdleConnTimeout: 90 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
ForceAttemptHTTP2: true,
},
Timeout: 60 * time.Second,
CheckRedirect: checkRedirect,
}
}
// checkRedirect prevents HTTPS→HTTP downgrades and limits redirect depth.
func checkRedirect(req *http.Request, via []*http.Request) error {
if len(via) >= 10 {
return fmt.Errorf("stopped after %d redirects", len(via))
}
if len(via) > 0 && via[0].URL.Scheme == "https" && req.URL.Scheme == "http" {
return errors.New("refused redirect from https to http")
}
return nil
}
// Get performs a GET request with the Webi User-Agent header.
func Get(ctx context.Context, client *http.Client, url string) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", userAgent)
return client.Do(req)
}
// Do executes a request with automatic retries for transient errors (429,
// 502, 503, 504). Retries up to 3 times with exponential backoff and jitter.
// Respects Retry-After headers. Only retries GET and HEAD (idempotent).
//
// Sets the Webi User-Agent header if not already present.
func Do(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
if req.Header.Get("User-Agent") == "" {
req.Header.Set("User-Agent", userAgent)
}
// Only retry idempotent methods.
idempotent := req.Method == http.MethodGet || req.Method == http.MethodHead
const maxRetries = 3
var resp *http.Response
var err error
for attempt := range maxRetries + 1 {
if attempt > 0 {
if !idempotent {
break
}
delay := backoff(attempt, resp)
timer := time.NewTimer(delay)
select {
case <-ctx.Done():
timer.Stop()
return nil, ctx.Err()
case <-timer.C:
}
if resp != nil {
resp.Body.Close()
}
}
resp, err = client.Do(req)
if err != nil {
if ctx.Err() != nil {
return nil, ctx.Err()
}
continue
}
if !isRetryable(resp.StatusCode) {
return resp, nil
}
}
if err != nil {
return nil, fmt.Errorf("after %d retries: %w", maxRetries, err)
}
return resp, nil
}
func isRetryable(status int) bool {
return status == http.StatusTooManyRequests ||
status == http.StatusBadGateway ||
status == http.StatusServiceUnavailable ||
status == http.StatusGatewayTimeout
}
// backoff returns a delay before the next retry. Respects Retry-After,
// otherwise uses exponential backoff with jitter.
func backoff(attempt int, resp *http.Response) time.Duration {
if resp != nil {
if ra := resp.Header.Get("Retry-After"); ra != "" {
if seconds, err := strconv.Atoi(ra); err == nil && seconds > 0 && seconds < 300 {
return time.Duration(seconds) * time.Second
}
}
}
// 1s, 2s, 4s base delays
base := time.Second << (attempt - 1)
if base > 30*time.Second {
base = 30 * time.Second
}
// Add jitter: 75% to 125% of base
jitter := float64(base) * (0.75 + 0.5*rand.Float64())
return time.Duration(jitter)
}

View File

@@ -162,8 +162,6 @@ func Read(path string) (*Conf, error) {
c := &Conf{}
// Infer source from primary key, falling back to explicit "source".
// When both github_releases and source are set, parse the repo ref
// from github_releases but use the explicit source for classification.
switch {
// GitHub binary releases.
case raw["github_releases"] != "":
@@ -215,13 +213,6 @@ func Read(path string) (*Conf, error) {
default:
}
// Explicit "source" overrides the inferred source when both are present.
// This lets packages like ffmpeg use github_releases for fetching but
// a custom classifier for classification.
if raw["source"] != "" && c.Source != "" {
c.Source = raw["source"]
}
// git_url can appear alongside any source type (e.g. github_sources)
// to provide a git clone fallback. When it's the only key, it's the
// primary source (gittag).

View File

@@ -1,104 +0,0 @@
// Package platlatest tracks the newest release version per build target.
//
// After classification determines which OS/arch/libc targets a release
// covers, this package records the latest version for each target. This
// handles the common case where Windows or macOS releases lag behind
// Linux by several versions.
//
// Storage is a single JSON file per package:
//
// {
// "linux-x86_64-gnu": "v0.145.0",
// "darwin-aarch64-none": "v0.144.1",
// "windows-x86_64-msvc": "v0.143.0"
// }
package platlatest
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sync"
"github.com/webinstall/webi-installers/internal/buildmeta"
)
// Index tracks the latest version for each build target of a package.
type Index struct {
mu sync.RWMutex
path string
m map[string]string // triplet → version
}
// Open loads or creates a per-platform latest index at the given path.
func Open(path string) (*Index, error) {
idx := &Index{
path: path,
m: make(map[string]string),
}
data, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return idx, nil
}
return nil, fmt.Errorf("platlatest: read %s: %w", path, err)
}
if err := json.Unmarshal(data, &idx.m); err != nil {
return nil, fmt.Errorf("platlatest: parse %s: %w", path, err)
}
return idx, nil
}
// Get returns the latest version for a target, or "" if unknown.
func (idx *Index) Get(t buildmeta.Target) string {
idx.mu.RLock()
defer idx.mu.RUnlock()
return idx.m[t.Triplet()]
}
// Set records a version as the latest for a target. Does not persist
// to disk — call Save after all updates.
func (idx *Index) Set(t buildmeta.Target, version string) {
idx.mu.Lock()
defer idx.mu.Unlock()
idx.m[t.Triplet()] = version
}
// All returns a copy of the full triplet→version map.
func (idx *Index) All() map[string]string {
idx.mu.RLock()
defer idx.mu.RUnlock()
out := make(map[string]string, len(idx.m))
for k, v := range idx.m {
out[k] = v
}
return out
}
// Save persists the index to disk (atomic write).
func (idx *Index) Save() error {
idx.mu.RLock()
data, err := json.MarshalIndent(idx.m, "", " ")
idx.mu.RUnlock()
if err != nil {
return fmt.Errorf("platlatest: marshal: %w", err)
}
dir := filepath.Dir(idx.path)
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("platlatest: mkdir: %w", err)
}
tmp := idx.path + ".tmp"
if err := os.WriteFile(tmp, data, 0o644); err != nil {
return fmt.Errorf("platlatest: write %s: %w", tmp, err)
}
if err := os.Rename(tmp, idx.path); err != nil {
os.Remove(tmp)
return fmt.Errorf("platlatest: rename %s: %w", idx.path, err)
}
return nil
}

View File

@@ -1,104 +0,0 @@
package platlatest_test
import (
"path/filepath"
"testing"
"github.com/webinstall/webi-installers/internal/buildmeta"
"github.com/webinstall/webi-installers/internal/platlatest"
)
var (
linuxAMD64 = buildmeta.Target{
OS: buildmeta.OSLinux, Arch: buildmeta.ArchAMD64, Libc: buildmeta.LibcGNU,
}
darwinARM64 = buildmeta.Target{
OS: buildmeta.OSDarwin, Arch: buildmeta.ArchARM64, Libc: buildmeta.LibcNone,
}
windowsAMD64 = buildmeta.Target{
OS: buildmeta.OSWindows, Arch: buildmeta.ArchAMD64, Libc: buildmeta.LibcMSVC,
}
)
func TestSetAndGet(t *testing.T) {
p := filepath.Join(t.TempDir(), "latest.json")
idx, err := platlatest.Open(p)
if err != nil {
t.Fatal(err)
}
if got := idx.Get(linuxAMD64); got != "" {
t.Errorf("Get before Set = %q, want empty", got)
}
idx.Set(linuxAMD64, "v0.145.0")
idx.Set(darwinARM64, "v0.144.1")
idx.Set(windowsAMD64, "v0.143.0")
if got := idx.Get(linuxAMD64); got != "v0.145.0" {
t.Errorf("linux = %q, want v0.145.0", got)
}
if got := idx.Get(darwinARM64); got != "v0.144.1" {
t.Errorf("darwin = %q, want v0.144.1", got)
}
if got := idx.Get(windowsAMD64); got != "v0.143.0" {
t.Errorf("windows = %q, want v0.143.0", got)
}
}
func TestSaveAndReload(t *testing.T) {
p := filepath.Join(t.TempDir(), "latest.json")
idx1, err := platlatest.Open(p)
if err != nil {
t.Fatal(err)
}
idx1.Set(linuxAMD64, "v0.145.0")
idx1.Set(darwinARM64, "v0.144.1")
if err := idx1.Save(); err != nil {
t.Fatal(err)
}
// Reload from disk.
idx2, err := platlatest.Open(p)
if err != nil {
t.Fatal(err)
}
if got := idx2.Get(linuxAMD64); got != "v0.145.0" {
t.Errorf("after reload: linux = %q, want v0.145.0", got)
}
if got := idx2.Get(darwinARM64); got != "v0.144.1" {
t.Errorf("after reload: darwin = %q, want v0.144.1", got)
}
}
func TestAll(t *testing.T) {
p := filepath.Join(t.TempDir(), "latest.json")
idx, err := platlatest.Open(p)
if err != nil {
t.Fatal(err)
}
idx.Set(linuxAMD64, "v1.0.0")
idx.Set(darwinARM64, "v0.9.0")
all := idx.All()
if len(all) != 2 {
t.Fatalf("All() returned %d entries, want 2", len(all))
}
if all[linuxAMD64.Triplet()] != "v1.0.0" {
t.Error("missing linux entry")
}
}
func TestOpenNonexistent(t *testing.T) {
p := filepath.Join(t.TempDir(), "does-not-exist.json")
idx, err := platlatest.Open(p)
if err != nil {
t.Fatal(err)
}
// Should be empty, not nil.
if all := idx.All(); len(all) != 0 {
t.Errorf("new index should be empty, got %v", all)
}
}

View File

@@ -1,264 +0,0 @@
// Package render generates installer scripts by injecting release
// metadata into the package-install template.
//
// The template uses shell-style variable markers:
//
// #WEBI_VERSION= → WEBI_VERSION='1.2.3'
// #export WEBI_PKG_URL= → export WEBI_PKG_URL='https://...'
//
// The package's install.sh is injected at the {{ installer }} marker.
package render
import (
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
)
// Params holds all the values to inject into the installer template.
type Params struct {
// Host is the base URL of the webi server (e.g. "https://webinstall.dev").
Host string
// Checksum is the webi.sh bootstrap script checksum (first 8 hex chars of SHA-1).
Checksum string
// Package name (e.g. "bat", "node").
PkgName string
// Tag is the version selector from the URL (e.g. "20", "stable", "").
Tag string
// OS, Arch, Libc are the detected platform strings.
OS string
Arch string
Libc string
// Resolved release info.
Version string
Major string
Minor string
Patch string
Build string
GitTag string
GitBranch string
LTS string // "true" or "false"
Channel string
Ext string // archive extension (e.g. "tar.gz", "zip")
Formats string // comma-separated format list
// Download info.
PkgURL string // download URL
PkgFile string // filename
// Releases API URL for this request.
ReleasesURL string
// CSV line for WEBI_CSV.
CSV string
// Package catalog info.
PkgStable string
PkgLatest string
PkgOSes string // space-separated
PkgArches string // space-separated
PkgLibcs string // space-separated
PkgFormats string // space-separated
}
// Bash renders a complete bash installer script by injecting params
// into the template and splicing in the package's install.sh.
func Bash(tplPath, installersDir, pkgName string, p Params) (string, error) {
tpl, err := os.ReadFile(tplPath)
if err != nil {
return "", fmt.Errorf("render: read template: %w", err)
}
// Read the package's install.sh.
installPath := filepath.Join(installersDir, pkgName, "install.sh")
installSh, err := os.ReadFile(installPath)
if err != nil {
return "", fmt.Errorf("render: read %s/install.sh: %w", pkgName, err)
}
text := string(tpl)
// Inject environment variables.
vars := []struct {
name string
value string
}{
{"WEBI_CHECKSUM", p.Checksum},
{"WEBI_PKG", p.PkgName + "@" + p.Tag},
{"WEBI_HOST", p.Host},
{"WEBI_OS", p.OS},
{"WEBI_ARCH", p.Arch},
{"WEBI_LIBC", p.Libc},
{"WEBI_TAG", p.Tag},
{"WEBI_RELEASES", p.ReleasesURL},
{"WEBI_CSV", p.CSV},
{"WEBI_VERSION", p.Version},
{"WEBI_MAJOR", p.Major},
{"WEBI_MINOR", p.Minor},
{"WEBI_PATCH", p.Patch},
{"WEBI_BUILD", p.Build},
{"WEBI_GIT_BRANCH", p.GitBranch},
{"WEBI_GIT_TAG", p.GitTag},
{"WEBI_LTS", p.LTS},
{"WEBI_CHANNEL", p.Channel},
{"WEBI_EXT", p.Ext},
{"WEBI_FORMATS", p.Formats},
{"WEBI_PKG_URL", p.PkgURL},
{"WEBI_PKG_PATHNAME", p.PkgFile},
{"WEBI_PKG_FILE", p.PkgFile},
{"PKG_NAME", p.PkgName},
{"PKG_STABLE", p.PkgStable},
{"PKG_LATEST", p.PkgLatest},
{"PKG_OSES", p.PkgOSes},
{"PKG_ARCHES", p.PkgArches},
{"PKG_LIBCS", p.PkgLibcs},
{"PKG_FORMATS", p.PkgFormats},
}
for _, v := range vars {
text = InjectVar(text, v.name, v.value)
}
// Inject the installer script at the {{ installer }} marker.
// The marker sits inside __init_installer() at 8-space indent.
// Production pads every line of install.sh to match, and replaces
// the entire line (including leading whitespace).
padded := padScript(string(installSh), " ")
text = replaceMarkerLine(text, "{{ installer }}", padded)
return text, nil
}
// PowerShell renders a complete PowerShell installer script by injecting
// params into the template and splicing in the package's install.ps1.
func PowerShell(tplPath, installersDir, pkgName string, p Params) (string, error) {
tpl, err := os.ReadFile(tplPath)
if err != nil {
return "", fmt.Errorf("render: read template: %w", err)
}
installPath := filepath.Join(installersDir, pkgName, "install.ps1")
installPs1, err := os.ReadFile(installPath)
if err != nil {
return "", fmt.Errorf("render: read %s/install.ps1: %w", pkgName, err)
}
text := string(tpl)
vars := []struct {
name string
value string
}{
{"WEBI_PKG", p.PkgName + "@" + p.Tag},
{"WEBI_HOST", p.Host},
{"WEBI_VERSION", p.Version},
{"WEBI_GIT_TAG", p.GitTag},
{"WEBI_PKG_URL", p.PkgURL},
{"WEBI_PKG_FILE", p.PkgFile},
{"WEBI_PKG_PATHNAME", p.PkgFile},
{"PKG_NAME", p.PkgName},
}
for _, v := range vars {
text = InjectPSVar(text, v.name, v.value)
}
// PS1 marker is at column 0, no padding needed.
text = replaceMarkerLine(text, "{{ installer }}", string(installPs1))
return text, nil
}
// InjectPSVar replaces a PowerShell template variable line with its value.
// Matches lines like:
//
// #$Env:WEBI_VERSION = v12.16.2
// $Env:WEBI_HOST = 'https://webinstall.dev'
func InjectPSVar(text, name, value string) string {
p := getPSVarPattern(name)
return p.ReplaceAllString(text, "${1}$$Env:"+name+" = '"+sanitizePSValue(value)+"'")
}
var psVarPatterns = map[string]*regexp.Regexp{}
func getPSVarPattern(name string) *regexp.Regexp {
if p, ok := psVarPatterns[name]; ok {
return p
}
// Match: optional leading whitespace, optional #, $Env:NAME, =, rest of line
p := regexp.MustCompile(`(?m)^([ \t]*)#?\$Env:` + regexp.QuoteMeta(name) + `\s*=.*$`)
psVarPatterns[name] = p
return p
}
// sanitizePSValue escapes single quotes for PowerShell single-quoted strings.
// In PowerShell, single quotes inside single-quoted strings are doubled: ''
func sanitizePSValue(s string) string {
return strings.ReplaceAll(s, "'", "''")
}
// varPattern matches shell variable declarations in the template.
// Matches lines like:
//
// #WEBI_VERSION=
// #export WEBI_PKG_URL=
// #WEBI_OS=
var varPatterns = map[string]*regexp.Regexp{}
func getVarPattern(name string) *regexp.Regexp {
if p, ok := varPatterns[name]; ok {
return p
}
// Match: optional leading whitespace, optional #, optional export, the var name, =, rest of line
p := regexp.MustCompile(`(?m)^([ \t]*)#?([ \t]*)(export[ \t]+)?[ \t]*(` + regexp.QuoteMeta(name) + `)=.*$`)
varPatterns[name] = p
return p
}
// InjectVar replaces a template variable line with its value.
// It matches lines like:
//
// #WEBI_VERSION=
// #export WEBI_PKG_URL=
// export WEBI_HOST=
//
// and replaces them with the value in single quotes.
func InjectVar(text, name, value string) string {
p := getVarPattern(name)
return p.ReplaceAllString(text, "${1}${3}"+name+"='"+sanitizeShellValue(value)+"'")
}
// sanitizeShellValue ensures a value is safe to embed in single quotes.
// Single quotes in shell can't be escaped inside single quotes, so we
// close-quote, add escaped quote, re-open quote: 'foo'\''bar'
func sanitizeShellValue(s string) string {
return strings.ReplaceAll(s, "'", `'\''`)
}
// padScript prepends each line of a script with the given indent string.
// This matches production behavior where install.sh content is indented
// to align with the surrounding template code.
func padScript(script, indent string) string {
lines := strings.Split(script, "\n")
for i, line := range lines {
if line != "" {
lines[i] = indent + line
}
}
return strings.Join(lines, "\n")
}
// replaceMarkerLine replaces an entire line containing the marker
// (including any leading whitespace) with the replacement text.
// This matches production's regex: /\s*#?\s*{{ installer }}/
func replaceMarkerLine(text, marker, replacement string) string {
re := regexp.MustCompile(`(?m)^[ \t]*#?[ \t]*` + regexp.QuoteMeta(marker) + `[^\n]*`)
return re.ReplaceAllLiteralString(text, replacement)
}

View File

@@ -1,90 +0,0 @@
package render
import (
"strings"
"testing"
)
func TestInjectVar(t *testing.T) {
tests := []struct {
name string
input string
key string
value string
want string
}{
{
name: "commented var",
input: " #WEBI_VERSION=",
key: "WEBI_VERSION",
value: "1.2.3",
want: " WEBI_VERSION='1.2.3'",
},
{
name: "commented export var",
input: " #export WEBI_PKG_URL=",
key: "WEBI_PKG_URL",
value: "https://example.com/foo.tar.gz",
want: " export WEBI_PKG_URL='https://example.com/foo.tar.gz'",
},
{
name: "existing value replaced",
input: " export WEBI_HOST=",
key: "WEBI_HOST",
value: "https://webinstall.dev",
want: " export WEBI_HOST='https://webinstall.dev'",
},
{
name: "value with single quotes",
input: " #PKG_NAME=",
key: "PKG_NAME",
value: "it's-a-test",
want: " PKG_NAME='it'\\''s-a-test'",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := InjectVar(tt.input, tt.key, tt.value)
if strings.TrimSpace(got) != strings.TrimSpace(tt.want) {
t.Errorf("got %q\nwant %q", got, tt.want)
}
})
}
}
func TestInjectVarInTemplate(t *testing.T) {
tpl := `#!/bin/sh
__bootstrap_webi() {
#PKG_NAME=
#WEBI_OS=
#WEBI_ARCH=
#WEBI_VERSION=
export WEBI_HOST=
WEBI_PKG_DOWNLOAD=""
`
result := tpl
result = InjectVar(result, "PKG_NAME", "bat")
result = InjectVar(result, "WEBI_OS", "linux")
result = InjectVar(result, "WEBI_ARCH", "x86_64")
result = InjectVar(result, "WEBI_VERSION", "0.26.1")
result = InjectVar(result, "WEBI_HOST", "https://webinstall.dev")
if !strings.Contains(result, "PKG_NAME='bat'") {
t.Error("PKG_NAME not injected")
}
if !strings.Contains(result, "WEBI_OS='linux'") {
t.Error("WEBI_OS not injected")
}
if !strings.Contains(result, "WEBI_VERSION='0.26.1'") {
t.Error("WEBI_VERSION not injected")
}
if !strings.Contains(result, "export WEBI_HOST='https://webinstall.dev'") {
t.Error("WEBI_HOST not injected")
}
// Should not have #PKG_NAME= anymore.
if strings.Contains(result, "#PKG_NAME=") {
t.Error("#PKG_NAME= should have been replaced")
}
}

View File

@@ -1,303 +0,0 @@
// Package resolve picks the best release for a given platform query.
//
// Given a set of classified distributables and a target query (OS, arch,
// libc, format preferences, version constraint), it returns the single
// best matching release — or nil if nothing matches.
package resolve
import (
"strings"
"github.com/webinstall/webi-installers/internal/buildmeta"
"github.com/webinstall/webi-installers/internal/lexver"
)
// Dist is one downloadable distributable — matches the CSV row from classify.
type Dist struct {
Package string
Version string
Channel string
OS string
Arch string
Libc string
Format string
Download string
Filename string
SHA256 string
Size int64
LTS bool
Date string
Extra string // extra version info for sorting
GitTag string // original git tag or branch — only for format="git"
GitCommitHash string // short commit hash — only for format="git"
Variants []string // build qualifiers: "installer", "rocm", "fxdependent", etc.
}
// Query describes what the caller wants.
type Query struct {
OS buildmeta.OS
Arch buildmeta.Arch
Libc buildmeta.Libc
Formats []string // acceptable formats (e.g. ".tar.gz", ".zip"), in preference order
Channel string // "stable" (default), "beta", etc.
Version string // version prefix constraint ("24", "24.14", ""), empty = latest
Variants []string // if non-empty, only match assets with these variants
}
// Match is the resolved release.
type Match struct {
Version string
OS string
Arch string
Libc string
Format string
Download string
Filename string
LTS bool
Date string
Channel string
}
// Best finds the single best release matching the query.
// Returns nil if nothing matches.
func Best(dists []Dist, q Query) *Match {
channel := q.Channel
if channel == "" {
channel = "stable"
}
// Build format set for fast lookup + rank map for preference.
formatRank := make(map[string]int, len(q.Formats))
for i, f := range q.Formats {
formatRank[f] = i
}
// Build the set of acceptable architectures (native + compat).
compatArches := buildmeta.CompatArches(q.OS, q.Arch)
archRank := make(map[string]int, len(compatArches))
for i, a := range compatArches {
archRank[string(a)] = i
}
// Parse version prefix for constraint matching.
var versionPrefix lexver.Version
hasVersionConstraint := q.Version != ""
if hasVersionConstraint {
versionPrefix = lexver.Parse(q.Version)
}
var best *candidate
for i := range dists {
d := &dists[i]
// Channel filter.
if channel == "stable" && d.Channel != "stable" && d.Channel != "" {
continue
}
// OS filter: exact match, POSIX fallback, or ANYOS.
if !osMatches(q.OS, d.OS) {
continue
}
// Arch filter (including compat arches).
// Empty arch, ANYARCH, or "*" means "universal/platform-agnostic" —
// accept it but rank it lower than an exact match.
aRank, archOK := archRank[d.Arch]
if !archOK && (d.Arch == "" || d.Arch == "*" || d.Arch == string(buildmeta.ArchAny)) {
// Universal binary — rank after all specific arches.
aRank = len(compatArches)
archOK = true
}
if !archOK {
continue
}
// Libc filter.
if !libcMatches(q.OS, q.Libc, d.Libc) {
continue
}
// Format filter.
// Empty format means bare binary — accept as last resort.
fRank, formatOK := formatRank[d.Format]
if !formatOK && d.Format == "" {
// Bare binary — rank after all explicit formats.
fRank = len(q.Formats)
formatOK = true
}
if !formatOK && len(q.Formats) > 0 {
continue
}
if !formatOK {
fRank = 999
}
// Version constraint.
ver := lexver.Parse(d.Version)
if hasVersionConstraint && !ver.HasPrefix(versionPrefix) {
continue
}
c := &candidate{
dist: d,
ver: ver,
archRank: aRank,
formatRank: fRank,
hasVariants: len(d.Variants) > 0,
}
if best == nil || c.betterThan(best) {
best = c
}
}
if best == nil {
return nil
}
d := best.dist
return &Match{
Version: d.Version,
OS: d.OS,
Arch: d.Arch,
Libc: d.Libc,
Format: d.Format,
Download: d.Download,
Filename: d.Filename,
LTS: d.LTS,
Date: d.Date,
Channel: d.Channel,
}
}
// Catalog computes aggregate metadata across all stable dists for a package.
type Catalog struct {
OSes []string
Arches []string
Libcs []string
Formats []string
Latest string // highest version of any channel
Stable string // highest stable version
}
// Survey scans all dists and returns the catalog.
func Survey(dists []Dist) Catalog {
oses := make(map[string]bool)
arches := make(map[string]bool)
libcs := make(map[string]bool)
formats := make(map[string]bool)
var latest, stable string
for _, d := range dists {
if d.OS != "" {
oses[d.OS] = true
}
if d.Arch != "" {
arches[d.Arch] = true
}
if d.Libc != "" {
libcs[d.Libc] = true
}
if d.Format != "" {
formats[d.Format] = true
}
v := lexver.Parse(d.Version)
if latest == "" || lexver.Compare(v, lexver.Parse(latest)) > 0 {
latest = d.Version
}
if d.Channel == "stable" || d.Channel == "" {
if stable == "" || lexver.Compare(v, lexver.Parse(stable)) > 0 {
stable = d.Version
}
}
}
return Catalog{
OSes: sortedKeys(oses),
Arches: sortedKeys(arches),
Libcs: sortedKeys(libcs),
Formats: sortedKeys(formats),
Latest: latest,
Stable: stable,
}
}
type candidate struct {
dist *Dist
ver lexver.Version
archRank int
formatRank int
hasVariants bool // true if dist has variant qualifiers (GPU, installer, etc.)
}
// betterThan returns true if c is a better match than other.
// Priority: version (higher) > base over variant > arch rank (lower=native) > format rank (lower=preferred).
func (c *candidate) betterThan(other *candidate) bool {
cmp := lexver.Compare(c.ver, other.ver)
if cmp != 0 {
return cmp > 0
}
// Prefer base build over variant builds (rocm, installer, etc.)
if c.hasVariants != other.hasVariants {
return !c.hasVariants
}
if c.archRank != other.archRank {
return c.archRank < other.archRank
}
return c.formatRank < other.formatRank
}
// osMatches checks whether a dist's OS is acceptable for the query.
// Matches exact OS, ANYOS (universal), and POSIX compatibility levels
// (posix_2017 matches any non-Windows OS).
func osMatches(want buildmeta.OS, have string) bool {
if have == string(want) {
return true
}
if have == string(buildmeta.OSAny) {
return true
}
// POSIX assets run on any non-Windows system.
if want != buildmeta.OSWindows {
if have == string(buildmeta.OSPosix2017) || have == string(buildmeta.OSPosix2024) {
return true
}
}
return false
}
// libcMatches checks whether a dist's libc is acceptable for the query.
func libcMatches(os buildmeta.OS, want buildmeta.Libc, have string) bool {
// Darwin and Windows don't use libc tagging — accept anything.
if os == buildmeta.OSDarwin || os == buildmeta.OSWindows {
return true
}
// If the dist has no libc tag, accept it (likely statically linked).
if have == "" || have == "none" || have == string(buildmeta.LibcNone) {
return true
}
// If the query has no libc preference, accept any.
if want == "" || want == buildmeta.LibcNone {
return true
}
return have == string(want)
}
func sortedKeys(m map[string]bool) []string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
// Simple insertion sort — these are tiny sets.
for i := 1; i < len(keys); i++ {
for j := i; j > 0 && strings.Compare(keys[j-1], keys[j]) > 0; j-- {
keys[j-1], keys[j] = keys[j], keys[j-1]
}
}
return keys
}

View File

@@ -1,422 +0,0 @@
package resolve_test
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"github.com/webinstall/webi-installers/internal/buildmeta"
"github.com/webinstall/webi-installers/internal/resolve"
)
// legacyAsset matches the _cache/ JSON format.
type legacyAsset struct {
Name string `json:"name"`
Version string `json:"version"`
LTS bool `json:"lts"`
Channel string `json:"channel"`
Date string `json:"date"`
OS string `json:"os"`
Arch string `json:"arch"`
Libc string `json:"libc"`
Ext string `json:"ext"`
Download string `json:"download"`
}
type legacyCache struct {
Releases []legacyAsset `json:"releases"`
}
func loadCacheDists(t *testing.T, pkg string) []resolve.Dist {
t.Helper()
cacheDir := filepath.Join("..", "..", "_cache", "2026-03")
path := filepath.Join(cacheDir, pkg+".json")
data, err := os.ReadFile(path)
if err != nil {
t.Skipf("no cache file for %s: %v", pkg, err)
}
var lc legacyCache
if err := json.Unmarshal(data, &lc); err != nil {
t.Fatalf("parse %s: %v", pkg, err)
}
dists := make([]resolve.Dist, len(lc.Releases))
for i, la := range lc.Releases {
// Reverse-translate legacy Node.js vocabulary to Go canonical names.
// The cache file uses macos/amd64/arm64; the resolver uses darwin/x86_64/aarch64.
osStr := la.OS
if osStr == "macos" {
osStr = "darwin"
}
archStr := la.Arch
switch archStr {
case "amd64":
archStr = "x86_64"
case "arm64":
archStr = "aarch64"
}
// Restore dot-prefix convention: cache stores "tar.gz", resolver needs ".tar.gz".
// "exe" with no dot in filename = bare binary (Format ""), otherwise ".exe".
format := la.Ext
switch {
case format == "exe" && !strings.Contains(la.Name, "."):
format = ""
case format != "":
format = "." + format
}
dists[i] = resolve.Dist{
Filename: la.Name,
Version: la.Version,
LTS: la.LTS,
Channel: la.Channel,
Date: la.Date,
OS: osStr,
Arch: archStr,
Libc: la.Libc, // "none" = buildmeta.LibcNone (statically linked)
Format: format,
Download: la.Download,
}
}
return dists
}
// platforms is the standard webi target matrix.
var platforms = []struct {
name string
os buildmeta.OS
arch buildmeta.Arch
formats []string
}{
{"darwin-arm64", buildmeta.OSDarwin, buildmeta.ArchARM64, []string{".tar.xz", ".tar.gz", ".zip"}},
{"darwin-amd64", buildmeta.OSDarwin, buildmeta.ArchAMD64, []string{".tar.xz", ".tar.gz", ".zip"}},
{"linux-amd64", buildmeta.OSLinux, buildmeta.ArchAMD64, []string{".tar.xz", ".exe.xz", ".tar.gz", ".xz", ".gz", ".zip"}},
{"linux-arm64", buildmeta.OSLinux, buildmeta.ArchARM64, []string{".tar.xz", ".exe.xz", ".tar.gz", ".xz", ".gz", ".zip"}},
{"linux-armv7", buildmeta.OSLinux, buildmeta.ArchARMv7, []string{".tar.xz", ".tar.gz", ".xz", ".gz", ".zip"}},
{"linux-armv6", buildmeta.OSLinux, buildmeta.ArchARMv6, []string{".tar.xz", ".tar.gz", ".xz", ".gz", ".zip"}},
{"windows-amd64", buildmeta.OSWindows, buildmeta.ArchAMD64, []string{".zip", ".tar.gz", ".exe", ".7z"}},
{"windows-arm64", buildmeta.OSWindows, buildmeta.ArchARM64, []string{".zip", ".tar.gz", ".exe", ".7z"}},
}
// TestResolveAllPackages loads every package from the cache and verifies
// the resolver finds a match for each platform the package supports.
func TestResolveAllPackages(t *testing.T) {
cacheDir := filepath.Join("..", "..", "_cache", "2026-03")
entries, err := os.ReadDir(cacheDir)
if err != nil {
t.Skipf("no cache dir: %v", err)
}
var pkgs []string
for _, e := range entries {
if strings.HasSuffix(e.Name(), ".json") {
pkgs = append(pkgs, strings.TrimSuffix(e.Name(), ".json"))
}
}
if len(pkgs) < 50 {
t.Fatalf("expected at least 50 packages, got %d", len(pkgs))
}
for _, pkg := range pkgs {
t.Run(pkg, func(t *testing.T) {
dists := loadCacheDists(t, pkg)
if len(dists) == 0 {
t.Skip("no releases")
}
// Determine which platforms this package supports.
cat := resolve.Survey(dists)
osSet := make(map[string]bool, len(cat.OSes))
for _, o := range cat.OSes {
osSet[o] = true
}
for _, plat := range platforms {
platOS := string(plat.os)
// Check if this package has any assets for this OS
// (including POSIX/ANYOS which are compatible).
supported := osSet[platOS] ||
osSet[string(buildmeta.OSAny)] ||
(platOS != "windows" && (osSet[string(buildmeta.OSPosix2017)] || osSet[string(buildmeta.OSPosix2024)]))
if !supported {
continue
}
t.Run(plat.name, func(t *testing.T) {
m := resolve.Best(dists, resolve.Query{
OS: plat.os,
Arch: plat.arch,
Formats: plat.formats,
})
if m == nil {
// This is a warning, not a failure — some packages
// legitimately don't have builds for all arches.
// But log it so we can spot unexpected gaps.
t.Logf("WARN: no match for %s on %s (has OSes: %v, Arches: %v)",
pkg, plat.name, cat.OSes, cat.Arches)
return
}
if m.Version == "" {
t.Error("matched but Version is empty")
}
if m.Download == "" {
t.Error("matched but Download is empty")
}
})
}
})
}
}
// Packages with known platform expectations. Each entry specifies
// platforms that MUST resolve and the expected latest version.
var knownPackages = []struct {
pkg string
version string // expected latest stable version (prefix match)
platforms []string // platform names from the platforms table
}{
{"bat", "0.26", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"caddy", "2.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "linux-armv7", "linux-armv6", "windows-amd64"}},
{"delta", "0.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"fd", "10.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "linux-armv7", "windows-amd64"}},
{"fzf", "0.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "linux-armv7", "windows-amd64"}},
{"gh", "2.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "linux-armv6", "windows-amd64"}},
{"rg", "", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"shellcheck", "0.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "linux-armv6", "windows-amd64"}},
{"shfmt", "3.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "linux-armv6", "windows-amd64"}},
{"xz", "", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"yq", "4.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "linux-armv6", "windows-amd64"}},
{"zoxide", "0.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "linux-armv7", "windows-amd64"}},
{"aliasman", "", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64"}},
{"comrak", "0.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "windows-amd64"}},
{"hugo", "0.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"node", "", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"terraform", "", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"zig", "", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
}
// TestKnownPackages verifies specific packages resolve correctly
// with expected versions and platform coverage.
func TestKnownPackages(t *testing.T) {
platMap := make(map[string]struct {
os buildmeta.OS
arch buildmeta.Arch
formats []string
})
for _, p := range platforms {
platMap[p.name] = struct {
os buildmeta.OS
arch buildmeta.Arch
formats []string
}{p.os, p.arch, p.formats}
}
for _, kp := range knownPackages {
t.Run(kp.pkg, func(t *testing.T) {
dists := loadCacheDists(t, kp.pkg)
for _, platName := range kp.platforms {
plat := platMap[platName]
t.Run(platName, func(t *testing.T) {
m := resolve.Best(dists, resolve.Query{
OS: plat.os,
Arch: plat.arch,
Formats: plat.formats,
})
if m == nil {
t.Skipf("no build available for %s on %s — upstream gap", kp.pkg, platName)
return
}
if kp.version != "" {
// Strip leading "v" for prefix comparison.
v := strings.TrimPrefix(m.Version, "v")
if !strings.HasPrefix(v, kp.version) {
t.Errorf("Version = %q, want prefix %q", m.Version, kp.version)
}
}
})
}
})
}
}
// TestResolveVersionConstraints tests version pinning across real packages.
func TestResolveVersionConstraints(t *testing.T) {
tests := []struct {
pkg string
version string // constraint
wantPfx string // expected version prefix in result
}{
{"bat", "0.25", "0.25"},
{"bat", "0.26", "0.26"},
{"gh", "2.40", "2.40"},
{"node", "20", "20."},
{"node", "22", "22."},
{"hugo", "0.121", "0.121"},
}
for _, tt := range tests {
name := fmt.Sprintf("%s@%s", tt.pkg, tt.version)
t.Run(name, func(t *testing.T) {
dists := loadCacheDists(t, tt.pkg)
m := resolve.Best(dists, resolve.Query{
OS: buildmeta.OSLinux,
Arch: buildmeta.ArchAMD64,
Formats: []string{".tar.xz", ".tar.gz", ".zip"},
Version: tt.version,
})
if m == nil {
t.Fatalf("no match for %s@%s", tt.pkg, tt.version)
}
v := strings.TrimPrefix(m.Version, "v")
if !strings.HasPrefix(v, tt.wantPfx) {
t.Errorf("Version = %q, want prefix %q", m.Version, tt.wantPfx)
}
})
}
}
// TestResolveArchFallbackReal tests arch fallback with real package data.
func TestResolveArchFallbackReal(t *testing.T) {
// awless only has amd64 builds — macOS ARM64 should fall back.
dists := loadCacheDists(t, "awless")
m := resolve.Best(dists, resolve.Query{
OS: buildmeta.OSDarwin,
Arch: buildmeta.ArchARM64,
Formats: []string{".tar.gz", ".zip"},
})
if m == nil {
t.Fatal("expected Rosetta 2 fallback for awless")
}
if m.Arch != "x86_64" {
t.Errorf("Arch = %q, want x86_64", m.Arch)
}
}
// TestResolvePosixPackages tests packages that use posix_2017/ANYOS.
func TestResolvePosixPackages(t *testing.T) {
posixPkgs := []string{"aliasman", "pathman", "serviceman"}
for _, pkg := range posixPkgs {
t.Run(pkg, func(t *testing.T) {
dists := loadCacheDists(t, pkg)
if len(dists) == 0 {
t.Skip("no releases")
}
// Should resolve on Linux.
m := resolve.Best(dists, resolve.Query{
OS: buildmeta.OSLinux,
Arch: buildmeta.ArchAMD64,
Formats: []string{".tar.xz", ".tar.gz", ".zip", ".xz", ".gz"},
})
if m == nil {
t.Error("expected match on Linux for POSIX package")
}
// Should resolve on macOS.
m = resolve.Best(dists, resolve.Query{
OS: buildmeta.OSDarwin,
Arch: buildmeta.ArchARM64,
Formats: []string{".tar.xz", ".tar.gz", ".zip"},
})
if m == nil {
t.Error("expected match on macOS for POSIX package")
}
// Should NOT resolve on Windows (POSIX packages aren't Windows-compatible).
m = resolve.Best(dists, resolve.Query{
OS: buildmeta.OSWindows,
Arch: buildmeta.ArchAMD64,
Formats: []string{".zip", ".tar.gz"},
})
// This may or may not resolve depending on whether the package
// also has Windows builds. Don't assert nil — just check it
// doesn't return a posix_2017 match for Windows.
if m != nil && (m.OS == "posix_2017" || m.OS == "posix_2024") {
t.Errorf("POSIX package should not match Windows, got OS=%q", m.OS)
}
})
}
}
// TestResolveLibcPreference tests libc selection.
// bat is a Rust project — its musl builds are static (libc='none').
// pwsh has hard musl dependencies (libc='musl').
func TestResolveLibcPreference(t *testing.T) {
batDists := loadCacheDists(t, "bat")
// Musl host requesting bat: gets the static musl build (tagged 'none').
m := resolve.Best(batDists, resolve.Query{
OS: buildmeta.OSLinux,
Arch: buildmeta.ArchAMD64,
Libc: buildmeta.LibcMusl,
Formats: []string{".tar.gz"},
})
if m == nil {
t.Fatal("expected match for musl host")
}
// Rust musl builds are static — tagged as 'none', not 'musl'.
if m.Libc != "none" {
t.Errorf("bat musl request: Libc = %q, want none (static musl)", m.Libc)
}
// Explicit gnu request.
m = resolve.Best(batDists, resolve.Query{
OS: buildmeta.OSLinux,
Arch: buildmeta.ArchAMD64,
Libc: buildmeta.LibcGNU,
Formats: []string{".tar.gz"},
})
if m == nil {
t.Fatal("expected gnu match")
}
if m.Libc != "gnu" {
t.Errorf("Libc = %q, want gnu", m.Libc)
}
// No preference — should still match (accepts any).
m = resolve.Best(batDists, resolve.Query{
OS: buildmeta.OSLinux,
Arch: buildmeta.ArchAMD64,
Formats: []string{".tar.gz"},
})
if m == nil {
t.Fatal("expected match with no libc preference")
}
// pwsh has hard musl builds (dynamically linked, requires musl runtime).
pwshDists := loadCacheDists(t, "pwsh")
m = resolve.Best(pwshDists, resolve.Query{
OS: buildmeta.OSLinux,
Arch: buildmeta.ArchAMD64,
Libc: buildmeta.LibcMusl,
Formats: []string{".tar.gz"},
})
if m == nil {
t.Fatal("expected pwsh musl match")
}
if m.Libc != "musl" {
t.Errorf("pwsh musl request: Libc = %q, want musl", m.Libc)
}
}
// TestResolveFormatFallback tests format preference cascading.
func TestResolveFormatFallback(t *testing.T) {
// Request .tar.xz first, fall back to .tar.gz.
dists := loadCacheDists(t, "bat")
m := resolve.Best(dists, resolve.Query{
OS: buildmeta.OSLinux,
Arch: buildmeta.ArchAMD64,
Formats: []string{".tar.xz", ".tar.gz", ".zip"},
})
if m == nil {
t.Fatal("expected match")
}
// bat only has .tar.gz — should fall back from .tar.xz.
if m.Format != ".tar.gz" {
t.Errorf("Format = %q, want .tar.gz (fallback from .tar.xz)", m.Format)
}
}

View File

@@ -1,250 +0,0 @@
package resolve
import (
"testing"
"github.com/webinstall/webi-installers/internal/buildmeta"
)
// bat-style dists: standard goreleaser output.
var batDists = []Dist{
{Version: "0.26.1", Channel: "stable", OS: "darwin", Arch: "aarch64", Format: ".tar.gz", Filename: "bat-v0.26.1-aarch64-apple-darwin.tar.gz"},
{Version: "0.26.1", Channel: "stable", OS: "darwin", Arch: "x86_64", Format: ".tar.gz", Filename: "bat-v0.26.1-x86_64-apple-darwin.tar.gz"},
{Version: "0.26.1", Channel: "stable", OS: "linux", Arch: "aarch64", Libc: "gnu", Format: ".tar.gz", Filename: "bat-v0.26.1-aarch64-unknown-linux-gnu.tar.gz"},
{Version: "0.26.1", Channel: "stable", OS: "linux", Arch: "aarch64", Libc: "musl", Format: ".tar.gz", Filename: "bat-v0.26.1-aarch64-unknown-linux-musl.tar.gz"},
{Version: "0.26.1", Channel: "stable", OS: "linux", Arch: "x86_64", Libc: "gnu", Format: ".tar.gz", Filename: "bat-v0.26.1-x86_64-unknown-linux-gnu.tar.gz"},
{Version: "0.26.1", Channel: "stable", OS: "linux", Arch: "x86_64", Libc: "musl", Format: ".tar.gz", Filename: "bat-v0.26.1-x86_64-unknown-linux-musl.tar.gz"},
{Version: "0.26.1", Channel: "stable", OS: "windows", Arch: "aarch64", Format: ".zip", Filename: "bat-v0.26.1-aarch64-pc-windows-msvc.zip"},
{Version: "0.26.1", Channel: "stable", OS: "windows", Arch: "x86_64", Libc: "gnu", Format: ".zip", Filename: "bat-v0.26.1-x86_64-pc-windows-gnu.zip"},
{Version: "0.26.1", Channel: "stable", OS: "windows", Arch: "x86_64", Libc: "msvc", Format: ".zip", Filename: "bat-v0.26.1-x86_64-pc-windows-msvc.zip"},
// Older version.
{Version: "0.25.0", Channel: "stable", OS: "darwin", Arch: "aarch64", Format: ".tar.gz", Filename: "bat-v0.25.0-aarch64-apple-darwin.tar.gz"},
{Version: "0.25.0", Channel: "stable", OS: "linux", Arch: "x86_64", Libc: "gnu", Format: ".tar.gz", Filename: "bat-v0.25.0-x86_64-unknown-linux-gnu.tar.gz"},
}
func TestBestExactMatch(t *testing.T) {
m := Best(batDists, Query{
OS: buildmeta.OSLinux,
Arch: buildmeta.ArchAMD64,
Formats: []string{".tar.gz"},
})
if m == nil {
t.Fatal("expected match")
}
if m.Version != "0.26.1" {
t.Errorf("Version = %q, want 0.26.1", m.Version)
}
if m.Filename != "bat-v0.26.1-x86_64-unknown-linux-gnu.tar.gz" {
t.Errorf("Filename = %q", m.Filename)
}
}
func TestBestVersionConstraint(t *testing.T) {
m := Best(batDists, Query{
OS: buildmeta.OSDarwin,
Arch: buildmeta.ArchARM64,
Formats: []string{".tar.gz"},
Version: "0.25",
})
if m == nil {
t.Fatal("expected match")
}
if m.Version != "0.25.0" {
t.Errorf("Version = %q, want 0.25.0", m.Version)
}
}
func TestBestArchFallback(t *testing.T) {
// macOS ARM64 should fall back to x86_64 via Rosetta 2
// when no ARM64 build exists.
dists := []Dist{
{Version: "1.0.0", Channel: "stable", OS: "darwin", Arch: "x86_64", Format: ".tar.gz", Filename: "tool-darwin-amd64.tar.gz"},
}
m := Best(dists, Query{
OS: buildmeta.OSDarwin,
Arch: buildmeta.ArchARM64,
Formats: []string{".tar.gz"},
})
if m == nil {
t.Fatal("expected match via Rosetta 2 fallback")
}
if m.Arch != "x86_64" {
t.Errorf("Arch = %q, want x86_64", m.Arch)
}
}
func TestBestPrefersNativeOverCompat(t *testing.T) {
// When both native and compat builds exist, prefer native.
dists := []Dist{
{Version: "1.0.0", Channel: "stable", OS: "darwin", Arch: "x86_64", Format: ".tar.gz", Filename: "tool-darwin-amd64.tar.gz"},
{Version: "1.0.0", Channel: "stable", OS: "darwin", Arch: "aarch64", Format: ".tar.gz", Filename: "tool-darwin-arm64.tar.gz"},
}
m := Best(dists, Query{
OS: buildmeta.OSDarwin,
Arch: buildmeta.ArchARM64,
Formats: []string{".tar.gz"},
})
if m == nil {
t.Fatal("expected match")
}
if m.Arch != "aarch64" {
t.Errorf("Arch = %q, want aarch64 (native)", m.Arch)
}
}
func TestBestFormatPreference(t *testing.T) {
dists := []Dist{
{Version: "1.0.0", Channel: "stable", OS: "linux", Arch: "x86_64", Format: ".zip", Filename: "tool.zip"},
{Version: "1.0.0", Channel: "stable", OS: "linux", Arch: "x86_64", Format: ".tar.gz", Filename: "tool.tar.gz"},
{Version: "1.0.0", Channel: "stable", OS: "linux", Arch: "x86_64", Format: ".tar.xz", Filename: "tool.tar.xz"},
}
m := Best(dists, Query{
OS: buildmeta.OSLinux,
Arch: buildmeta.ArchAMD64,
Formats: []string{".tar.xz", ".tar.gz", ".zip"},
})
if m == nil {
t.Fatal("expected match")
}
if m.Format != ".tar.xz" {
t.Errorf("Format = %q, want .tar.xz", m.Format)
}
}
func TestBestNoMatch(t *testing.T) {
m := Best(batDists, Query{
OS: buildmeta.OSFreeBSD,
Arch: buildmeta.ArchAMD64,
Formats: []string{".tar.gz"},
})
if m != nil {
t.Errorf("expected nil, got %+v", m)
}
}
func TestBestLibcMusl(t *testing.T) {
m := Best(batDists, Query{
OS: buildmeta.OSLinux,
Arch: buildmeta.ArchAMD64,
Libc: buildmeta.LibcMusl,
Formats: []string{".tar.gz"},
})
if m == nil {
t.Fatal("expected match")
}
if m.Libc != "musl" {
t.Errorf("Libc = %q, want musl", m.Libc)
}
}
func TestBestPrefersBaseOverVariant(t *testing.T) {
dists := []Dist{
{Version: "1.0.0", Channel: "stable", OS: "linux", Arch: "x86_64", Format: ".tar.gz", Filename: "tool.tar.gz"},
{Version: "1.0.0", Channel: "stable", OS: "linux", Arch: "x86_64", Format: ".tar.gz", Filename: "tool-rocm.tar.gz", Variants: []string{"rocm"}},
}
m := Best(dists, Query{
OS: buildmeta.OSLinux,
Arch: buildmeta.ArchAMD64,
Formats: []string{".tar.gz"},
})
if m == nil {
t.Fatal("expected match")
}
if m.Filename != "tool.tar.gz" {
t.Errorf("got variant build %q, want base", m.Filename)
}
}
func TestBestPosixFallback(t *testing.T) {
dists := []Dist{
{Version: "1.0.0", Channel: "stable", OS: "posix_2017", Format: ".tar.gz", Filename: "script.tar.gz"},
}
m := Best(dists, Query{
OS: buildmeta.OSLinux,
Arch: buildmeta.ArchAMD64,
Formats: []string{".tar.gz"},
})
if m == nil {
t.Fatal("expected match via POSIX fallback")
}
if m.OS != "posix_2017" {
t.Errorf("OS = %q, want posix_2017", m.OS)
}
}
func TestBestAnyOS(t *testing.T) {
dists := []Dist{
{Version: "1.0.0", Channel: "stable", OS: "ANYOS", Format: ".tar.gz", Filename: "tool.tar.gz"},
}
m := Best(dists, Query{
OS: buildmeta.OSWindows,
Arch: buildmeta.ArchAMD64,
Formats: []string{".tar.gz"},
})
if m == nil {
t.Fatal("expected match via ANYOS")
}
}
func TestBestAnyArch(t *testing.T) {
dists := []Dist{
{Version: "1.0.0", Channel: "stable", OS: "linux", Arch: "ANYARCH", Format: ".tar.gz", Filename: "tool.tar.gz"},
}
m := Best(dists, Query{
OS: buildmeta.OSLinux,
Arch: buildmeta.ArchARM64,
Formats: []string{".tar.gz"},
})
if m == nil {
t.Fatal("expected match via ANYARCH")
}
}
func TestBestWindowsArchFallback(t *testing.T) {
// Windows ARM64 should fall back to x86_64 via emulation.
dists := []Dist{
{Version: "1.0.0", Channel: "stable", OS: "windows", Arch: "x86_64", Format: ".zip", Filename: "tool-win64.zip"},
}
m := Best(dists, Query{
OS: buildmeta.OSWindows,
Arch: buildmeta.ArchARM64,
Formats: []string{".zip"},
})
if m == nil {
t.Fatal("expected match via Windows ARM64 emulation")
}
if m.Arch != "x86_64" {
t.Errorf("Arch = %q, want x86_64", m.Arch)
}
}
func TestBestMicroArchFallback(t *testing.T) {
// amd64v3 query should fall back to amd64 baseline.
dists := []Dist{
{Version: "1.0.0", Channel: "stable", OS: "linux", Arch: "x86_64", Format: ".tar.gz", Filename: "tool-amd64.tar.gz"},
}
m := Best(dists, Query{
OS: buildmeta.OSLinux,
Arch: buildmeta.ArchAMD64v3,
Formats: []string{".tar.gz"},
})
if m == nil {
t.Fatal("expected match via micro-arch fallback")
}
if m.Arch != "x86_64" {
t.Errorf("Arch = %q, want x86_64 (baseline)", m.Arch)
}
}
func TestSurvey(t *testing.T) {
cat := Survey(batDists)
if cat.Stable != "0.26.1" {
t.Errorf("Stable = %q, want 0.26.1", cat.Stable)
}
if cat.Latest != "0.26.1" {
t.Errorf("Latest = %q, want 0.26.1", cat.Latest)
}
if len(cat.OSes) != 3 {
t.Errorf("OSes = %v, want 3", cat.OSes)
}
}

View File

@@ -1,416 +0,0 @@
// Package resolver selects the best release asset for a given platform
// and version constraint.
//
// The resolver takes a package's full asset list and a request describing
// what the client needs (OS, arch, libc, version prefix, channel, format
// preferences). It returns the single best matching asset or an error.
//
// Resolution order:
// 1. Filter assets by channel (inclusive: @stable includes stable+lts)
// 2. Sort versions descending, filter by version prefix if given
// 3. For each candidate version, try compatible platform triplets
// (OS × CompatArches fallback × libc) in preference order
// 4. Among platform matches, pick the best format
// 5. Among format matches, prefer assets without build variants
package resolver
import (
"errors"
"slices"
"strings"
"github.com/webinstall/webi-installers/internal/buildmeta"
"github.com/webinstall/webi-installers/internal/lexver"
"github.com/webinstall/webi-installers/internal/storage"
)
// ErrNoMatch is returned when no asset matches the request.
var ErrNoMatch = errors.New("resolver: no matching asset")
// Request describes what the client is looking for.
type Request struct {
// OS is the target operating system (e.g. "linux", "darwin", "windows").
OS string
// Arch is the target architecture (e.g. "aarch64", "x86_64").
Arch string
// Libc is the preferred C library (e.g. "gnu", "musl", "msvc").
// Empty means no preference — the resolver tries all libc values.
Libc string
// Version is a version prefix constraint (e.g. "1.20", "1", "").
// Empty means latest. Exact versions like "1.20.3" also work.
Version string
// Channel selects the release stability level. Values:
// ""/"stable" — stable and LTS only (default)
// "lts" — LTS releases only
// "rc" — rc + stable + LTS
// "beta" — beta + rc + stable + LTS
// "alpha" — everything (alpha + beta + rc + stable + LTS)
// "pre" — alias for beta (package-specific meaning)
Channel string
// LTS when true selects only LTS-flagged releases.
LTS bool
// Formats lists acceptable archive formats in preference order.
// If empty, a default preference order is used.
Formats []string
// Variant selects a specific build variant (e.g. "rocm", "jetpack6").
// If empty, assets with variants are deprioritized.
Variant string
}
// Result holds the resolved asset and metadata about the match.
type Result struct {
// Asset is the selected download.
Asset storage.Asset
// Version is the matched version string.
Version string
// Triplet is the matched platform triplet (os-arch-libc).
Triplet string
}
// Resolve finds the best matching asset for the given request.
func Resolve(assets []storage.Asset, req Request) (Result, error) {
if len(assets) == 0 {
return Result{}, ErrNoMatch
}
// Parse the version prefix for filtering.
var versionPrefix lexver.Version
hasPrefix := req.Version != ""
if hasPrefix {
versionPrefix = lexver.Parse(req.Version)
}
// Build the channel filter.
channelOK := channelFilter(req.Channel, req.LTS)
// Parse and sort all unique versions descending.
type versionEntry struct {
parsed lexver.Version
raw string
}
seen := make(map[string]bool)
var versions []versionEntry
for _, a := range assets {
if seen[a.Version] {
continue
}
seen[a.Version] = true
v := lexver.Parse(a.Version)
v.Raw = a.Version
versions = append(versions, versionEntry{parsed: v, raw: a.Version})
}
slices.SortFunc(versions, func(a, b versionEntry) int {
return lexver.Compare(b.parsed, a.parsed) // descending
})
// Build platform fallback list: ordered (os, arch, libc) combinations.
triplets := enumerateTriplets(req.OS, req.Arch, req.Libc)
// Build format preference list.
formats := req.Formats
if len(formats) == 0 {
formats = defaultFormats(req.OS)
}
// Index assets by version+triplet for fast lookup.
// Assets with empty OS/Arch (like git repos) use "" keys.
type tripletKey struct {
version string
os string
arch string
libc string
}
index := make(map[tripletKey][]storage.Asset)
for _, a := range assets {
key := tripletKey{
version: a.Version,
os: a.OS,
arch: a.Arch,
libc: a.Libc,
}
index[key] = append(index[key], a)
}
// Walk versions in descending order.
for _, ve := range versions {
// Check version prefix.
if hasPrefix && !ve.parsed.HasPrefix(versionPrefix) {
continue
}
// Check channel.
if !channelOK(ve.parsed.Channel, ve.raw) {
continue
}
// Try each compatible triplet.
for _, tri := range triplets {
key := tripletKey{
version: ve.raw,
os: tri.os,
arch: tri.arch,
libc: tri.libc,
}
candidates := index[key]
if len(candidates) == 0 {
continue
}
// Pick the best asset from candidates.
best, ok := pickBest(candidates, formats, req.Variant, req.LTS)
if !ok {
continue
}
triplet := tri.os + "-" + tri.arch + "-" + tri.libc
return Result{
Asset: best,
Version: ve.raw,
Triplet: triplet,
}, nil
}
}
return Result{}, ErrNoMatch
}
// channelFilter returns a function that checks whether a given channel
// is acceptable for the requested channel level.
func channelFilter(requested string, ltsOnly bool) func(channel string, version string) bool {
if ltsOnly {
return func(_ string, _ string) bool {
// LTS filtering happens at the asset level, not version level.
// We let all versions through and filter by LTS flag later.
// Actually, LTS is per-asset, so we handle it in pickBest.
return true
}
}
requested = strings.ToLower(requested)
if requested == "" {
requested = "stable"
}
if requested == "pre" {
requested = "beta"
}
if requested == "latest" {
requested = "stable"
}
// channelRank maps channel names to a numeric rank.
// Higher rank = less stable. A request for rank N accepts
// everything at rank N or below.
rank := func(ch string) int {
ch = strings.ToLower(ch)
switch ch {
case "", "stable":
return 0
case "rc":
return 1
case "beta", "preview":
return 2
case "alpha", "dev":
return 3
default:
return 2 // unknown pre-release channels default to beta-level
}
}
maxRank := rank(requested)
return func(channel string, _ string) bool {
return rank(channel) <= maxRank
}
}
type platformTriple struct {
os string
arch string
libc string
}
// enumerateTriplets builds the ordered list of platform combinations to try.
// It uses CompatArches for arch fallback and tries multiple libc values.
func enumerateTriplets(osStr, archStr, libcStr string) []platformTriple {
// OS candidates: specific OS first, then POSIX compat, then any.
var oses []string
switch osStr {
case "windows":
oses = []string{"windows", "ANYOS", ""}
case "android":
oses = []string{"android", "linux", "posix_2024", "posix_2017", "ANYOS", ""}
case "":
oses = []string{"ANYOS", ""}
default:
oses = []string{osStr, "posix_2024", "posix_2017", "ANYOS", ""}
}
// Arch candidates: use CompatArches for fallback chain.
arches := buildmeta.CompatArches(buildmeta.OS(osStr), buildmeta.Arch(archStr))
var archStrs []string
for _, a := range arches {
archStrs = append(archStrs, string(a))
}
// Also try ANYARCH and empty (for platform-agnostic assets like git repos).
archStrs = append(archStrs, "ANYARCH", "")
// Libc candidates.
var libcs []string
if libcStr != "" {
libcs = []string{libcStr, "none", ""}
} else {
// No preference: try all common options.
switch osStr {
case "linux":
// none first (static, no deps), then gnu, musl, empty.
libcs = []string{"none", "gnu", "musl", ""}
case "windows":
// none first (no deps), msvc last (needs vcredist).
libcs = []string{"none", "msvc", ""}
default:
libcs = []string{"none", ""}
}
}
var triplets []platformTriple
for _, os := range oses {
for _, arch := range archStrs {
for _, libc := range libcs {
triplets = append(triplets, platformTriple{
os: os,
arch: arch,
libc: libc,
})
}
}
}
return triplets
}
// pickBest selects the best asset from a set of candidates for the same
// version and platform. Prefers the requested variant (or no-variant if
// none requested), then picks by format preference.
func pickBest(candidates []storage.Asset, formats []string, wantVariant string, ltsOnly bool) (storage.Asset, bool) {
// Filter by LTS if requested.
if ltsOnly {
var lts []storage.Asset
for _, a := range candidates {
if a.LTS {
lts = append(lts, a)
}
}
if len(lts) == 0 {
return storage.Asset{}, false
}
candidates = lts
}
// Separate into variant-matched and non-variant pools.
var preferred []storage.Asset
var fallback []storage.Asset
for _, a := range candidates {
if wantVariant != "" {
// User requested a specific variant.
if hasVariant(a.Variants, wantVariant) {
preferred = append(preferred, a)
} else if len(a.Variants) == 0 {
fallback = append(fallback, a)
}
} else {
// No variant requested: prefer no-variant assets.
if len(a.Variants) == 0 {
preferred = append(preferred, a)
} else {
fallback = append(fallback, a)
}
}
}
// Try preferred pool first, then fallback.
for _, pool := range [][]storage.Asset{preferred, fallback} {
if len(pool) == 0 {
continue
}
if best, ok := pickByFormat(pool, formats); ok {
return best, true
}
}
return storage.Asset{}, false
}
// pickByFormat selects the asset with the most preferred format.
func pickByFormat(assets []storage.Asset, formats []string) (storage.Asset, bool) {
for _, fmt := range formats {
for _, a := range assets {
if a.Format == fmt {
return a, true
}
}
}
// No format match — return the first asset as last resort.
if len(assets) > 0 {
return assets[0], true
}
return storage.Asset{}, false
}
func hasVariant(variants []string, want string) bool {
for _, v := range variants {
if v == want {
return true
}
}
return false
}
// defaultFormats returns the format preference order for an OS.
// zst is preferred as the modern standard, but availability varies.
func defaultFormats(os string) []string {
switch os {
case "windows":
return []string{
".tar.zst",
".tar.xz",
".zip",
".tar.gz",
".exe.xz",
".7z",
".exe",
".msi",
"git",
}
case "darwin":
return []string{
".tar.zst",
".tar.xz",
".zip",
".tar.gz",
".gz",
".app.zip",
".dmg",
".pkg",
"git",
}
default:
// Linux and other POSIX.
return []string{
".tar.zst",
".tar.xz",
".tar.gz",
".gz",
".zip",
".xz",
"git",
}
}
}

View File

@@ -1,290 +0,0 @@
package resolver_test
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"github.com/webinstall/webi-installers/internal/resolver"
"github.com/webinstall/webi-installers/internal/storage"
)
func loadAssets(t *testing.T, pkg string) []storage.Asset {
t.Helper()
cacheDir := filepath.Join("..", "..", "_cache", "2026-03")
path := filepath.Join(cacheDir, pkg+".json")
data, err := os.ReadFile(path)
if err != nil {
t.Skipf("no cache file for %s: %v", pkg, err)
}
var lc storage.LegacyCache
if err := json.Unmarshal(data, &lc); err != nil {
t.Fatalf("parse %s: %v", pkg, err)
}
pd := storage.ImportLegacy(lc)
return pd.Assets
}
// TestCacheResolveAllPackages loads every package from the cache and verifies
// the resolver finds a match for each standard platform.
func TestCacheResolveAllPackages(t *testing.T) {
cacheDir := filepath.Join("..", "..", "_cache", "2026-03")
entries, err := os.ReadDir(cacheDir)
if err != nil {
t.Skipf("no cache dir: %v", err)
}
var pkgs []string
for _, e := range entries {
if strings.HasSuffix(e.Name(), ".json") {
pkgs = append(pkgs, strings.TrimSuffix(e.Name(), ".json"))
}
}
if len(pkgs) < 50 {
t.Fatalf("expected at least 50 packages, got %d", len(pkgs))
}
platforms := []struct {
name string
os string
arch string
}{
{"darwin-arm64", "darwin", "aarch64"},
{"darwin-amd64", "darwin", "x86_64"},
{"linux-amd64", "linux", "x86_64"},
{"linux-arm64", "linux", "aarch64"},
{"windows-amd64", "windows", "x86_64"},
}
for _, pkg := range pkgs {
t.Run(pkg, func(t *testing.T) {
assets := loadAssets(t, pkg)
if len(assets) == 0 {
t.Skip("no releases")
}
// Determine which OSes this package has.
osSet := make(map[string]bool)
for _, a := range assets {
if a.OS != "" {
osSet[a.OS] = true
}
}
// Also check for platform-agnostic assets.
hasAgnostic := false
for _, a := range assets {
if a.OS == "" {
hasAgnostic = true
break
}
}
for _, plat := range platforms {
supported := osSet[plat.os] ||
osSet["ANYOS"] ||
hasAgnostic ||
(plat.os != "windows" && (osSet["posix_2017"] || osSet["posix_2024"]))
if !supported {
continue
}
t.Run(plat.name, func(t *testing.T) {
res, err := resolver.Resolve(assets, resolver.Request{
OS: plat.os,
Arch: plat.arch,
})
if err != nil {
// Not a test failure — some packages don't have
// all arch builds. Log for visibility.
t.Logf("WARN: no match for %s on %s (has OSes: %v)",
pkg, plat.name, sortedOSes(osSet))
return
}
if res.Version == "" {
t.Error("matched but Version is empty")
}
if res.Asset.Download == "" {
t.Error("matched but Download is empty")
}
})
}
})
}
}
// TestCacheKnownPackages verifies specific packages resolve correctly.
var knownPackages = []struct {
pkg string
version string // expected latest stable version prefix
platforms []string
}{
{"bat", "0.26", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"caddy", "2.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"delta", "0.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"fd", "10.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"fzf", "0.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"gh", "2.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"rg", "", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"node", "", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"terraform", "", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"zig", "", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
}
func TestCacheKnownPackages(t *testing.T) {
platMap := map[string]resolver.Request{
"darwin-arm64": {OS: "darwin", Arch: "aarch64"},
"darwin-amd64": {OS: "darwin", Arch: "x86_64"},
"linux-amd64": {OS: "linux", Arch: "x86_64"},
"linux-arm64": {OS: "linux", Arch: "aarch64"},
"windows-amd64": {OS: "windows", Arch: "x86_64"},
}
for _, kp := range knownPackages {
t.Run(kp.pkg, func(t *testing.T) {
assets := loadAssets(t, kp.pkg)
for _, platName := range kp.platforms {
req := platMap[platName]
t.Run(platName, func(t *testing.T) {
res, err := resolver.Resolve(assets, req)
if err != nil {
t.Fatalf("no match for %s on %s", kp.pkg, platName)
}
if kp.version != "" {
v := strings.TrimPrefix(res.Version, "v")
if !strings.HasPrefix(v, kp.version) {
t.Errorf("Version = %q, want prefix %q", res.Version, kp.version)
}
}
})
}
})
}
}
// TestCacheVersionConstraints tests version pinning with real data.
func TestCacheVersionConstraints(t *testing.T) {
tests := []struct {
pkg string
version string
wantPfx string
}{
{"bat", "0.25", "0.25"},
{"bat", "0.26", "0.26"},
{"gh", "2.40", "2.40"},
{"node", "20", "20."},
{"node", "22", "22."},
}
for _, tt := range tests {
t.Run(tt.pkg+"@"+tt.version, func(t *testing.T) {
assets := loadAssets(t, tt.pkg)
res, err := resolver.Resolve(assets, resolver.Request{
OS: "linux",
Arch: "x86_64",
Version: tt.version,
})
if err != nil {
t.Fatalf("no match for %s@%s", tt.pkg, tt.version)
}
v := strings.TrimPrefix(res.Version, "v")
if !strings.HasPrefix(v, tt.wantPfx) {
t.Errorf("Version = %q, want prefix %q", res.Version, tt.wantPfx)
}
})
}
}
// TestCacheArchFallback verifies Rosetta-style fallback with real data.
func TestCacheArchFallback(t *testing.T) {
// awless only has amd64 builds — macOS ARM64 should fall back.
assets := loadAssets(t, "awless")
res, err := resolver.Resolve(assets, resolver.Request{
OS: "darwin",
Arch: "aarch64",
})
if err != nil {
t.Fatal("expected Rosetta 2 fallback for awless")
}
if res.Asset.Arch != "x86_64" {
t.Errorf("Arch = %q, want x86_64", res.Asset.Arch)
}
}
// TestCacheGitPackages verifies git-only packages resolve on any platform.
func TestCacheGitPackages(t *testing.T) {
gitPkgs := []string{"vim-essentials", "vim-spell"}
for _, pkg := range gitPkgs {
t.Run(pkg, func(t *testing.T) {
assets := loadAssets(t, pkg)
if len(assets) == 0 {
t.Skip("no releases")
}
// Should work on any platform.
for _, plat := range []struct {
os, arch string
}{
{"linux", "x86_64"},
{"darwin", "aarch64"},
{"windows", "x86_64"},
} {
res, err := resolver.Resolve(assets, resolver.Request{
OS: plat.os,
Arch: plat.arch,
})
if err != nil {
t.Errorf("expected match on %s-%s", plat.os, plat.arch)
continue
}
if res.Asset.Format != "git" {
t.Errorf("format = %q, want git", res.Asset.Format)
}
}
})
}
}
// TestCacheLibcPreference tests explicit libc selection.
// bat is Rust — its musl builds are static (tagged 'none').
func TestCacheLibcPreference(t *testing.T) {
assets := loadAssets(t, "bat")
// Musl host requesting bat: gets static musl build (tagged 'none').
res, err := resolver.Resolve(assets, resolver.Request{
OS: "linux",
Arch: "x86_64",
Libc: "musl",
})
if err != nil {
t.Fatal("expected match for musl host")
}
if res.Asset.Libc != "none" {
t.Errorf("Libc = %q, want none (static musl)", res.Asset.Libc)
}
// Explicit gnu.
res, err = resolver.Resolve(assets, resolver.Request{
OS: "linux",
Arch: "x86_64",
Libc: "gnu",
})
if err != nil {
t.Fatal("expected gnu match")
}
if res.Asset.Libc != "gnu" {
t.Errorf("Libc = %q, want gnu", res.Asset.Libc)
}
}
func sortedOSes(m map[string]bool) []string {
var keys []string
for k := range m {
keys = append(keys, k)
}
return keys
}

View File

@@ -1,397 +0,0 @@
package resolver
import (
"testing"
"github.com/webinstall/webi-installers/internal/storage"
)
func TestResolveSimple(t *testing.T) {
assets := []storage.Asset{
{
Filename: "bat-v0.25.0-x86_64-unknown-linux-musl.tar.gz",
Version: "0.25.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Libc: "musl",
Format: ".tar.gz",
Download: "https://example.com/bat-0.25.0-linux-x86_64.tar.gz",
},
{
Filename: "bat-v0.26.0-x86_64-unknown-linux-musl.tar.gz",
Version: "0.26.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Libc: "musl",
Format: ".tar.gz",
Download: "https://example.com/bat-0.26.0-linux-x86_64.tar.gz",
},
{
Filename: "bat-v0.26.0-aarch64-unknown-linux-musl.tar.gz",
Version: "0.26.0",
Channel: "stable",
OS: "linux",
Arch: "aarch64",
Libc: "musl",
Format: ".tar.gz",
Download: "https://example.com/bat-0.26.0-linux-aarch64.tar.gz",
},
{
Filename: "bat-v0.26.0-x86_64-pc-windows-msvc.zip",
Version: "0.26.0",
Channel: "stable",
OS: "windows",
Arch: "x86_64",
Libc: "msvc",
Format: ".zip",
Download: "https://example.com/bat-0.26.0-windows-x86_64.zip",
},
{
Filename: "bat-v0.26.0-x86_64-apple-darwin.tar.gz",
Version: "0.26.0",
Channel: "stable",
OS: "darwin",
Arch: "x86_64",
Format: ".tar.gz",
Download: "https://example.com/bat-0.26.0-darwin-x86_64.tar.gz",
},
}
t.Run("latest linux x86_64", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
})
if err != nil {
t.Fatal(err)
}
if res.Version != "0.26.0" {
t.Errorf("version = %q, want 0.26.0", res.Version)
}
if res.Asset.OS != "linux" {
t.Errorf("os = %q, want linux", res.Asset.OS)
}
if res.Asset.Arch != "x86_64" {
t.Errorf("arch = %q, want x86_64", res.Asset.Arch)
}
})
t.Run("latest linux aarch64", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "aarch64",
})
if err != nil {
t.Fatal(err)
}
if res.Version != "0.26.0" {
t.Errorf("version = %q, want 0.26.0", res.Version)
}
if res.Asset.Arch != "aarch64" {
t.Errorf("arch = %q, want aarch64", res.Asset.Arch)
}
})
t.Run("version prefix 0.25", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
Version: "0.25",
})
if err != nil {
t.Fatal(err)
}
if res.Version != "0.25.0" {
t.Errorf("version = %q, want 0.25.0", res.Version)
}
})
t.Run("darwin arm64 falls back to x86_64", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "darwin",
Arch: "aarch64",
})
if err != nil {
t.Fatal(err)
}
if res.Asset.Arch != "x86_64" {
t.Errorf("arch = %q, want x86_64 (Rosetta fallback)", res.Asset.Arch)
}
})
t.Run("no match returns error", func(t *testing.T) {
_, err := Resolve(assets, Request{
OS: "freebsd",
Arch: "x86_64",
})
if err != ErrNoMatch {
t.Errorf("err = %v, want ErrNoMatch", err)
}
})
t.Run("windows gets zip", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "windows",
Arch: "x86_64",
})
if err != nil {
t.Fatal(err)
}
if res.Asset.Format != ".zip" {
t.Errorf("format = %q, want .zip", res.Asset.Format)
}
})
}
func TestResolveChannels(t *testing.T) {
assets := []storage.Asset{
{
Filename: "tool-v2.0.0-rc1-linux-x86_64.tar.gz",
Version: "2.0.0-rc1",
Channel: "rc",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
},
{
Filename: "tool-v1.5.0-linux-x86_64.tar.gz",
Version: "1.5.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
},
{
Filename: "tool-v2.0.0-beta2-linux-x86_64.tar.gz",
Version: "2.0.0-beta2",
Channel: "beta",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
},
}
t.Run("stable skips rc and beta", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
})
if err != nil {
t.Fatal(err)
}
if res.Version != "1.5.0" {
t.Errorf("version = %q, want 1.5.0", res.Version)
}
})
t.Run("rc includes rc and stable", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
Channel: "rc",
})
if err != nil {
t.Fatal(err)
}
if res.Version != "2.0.0-rc1" {
t.Errorf("version = %q, want 2.0.0-rc1", res.Version)
}
})
t.Run("beta includes beta, rc, and stable", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
Channel: "beta",
})
if err != nil {
t.Fatal(err)
}
// beta2 sorts after rc1 for the same numeric version (2.0.0),
// but rc1 is more stable. However, the user asked for beta channel
// which includes everything — and beta sorts before rc alphabetically.
// With lexver: 2.0.0-rc1 > 2.0.0-beta2 (rc > beta alphabetically).
if res.Version != "2.0.0-rc1" {
t.Errorf("version = %q, want 2.0.0-rc1", res.Version)
}
})
}
func TestResolveVariants(t *testing.T) {
assets := []storage.Asset{
{
Filename: "ollama-linux-amd64.tgz",
Version: "0.6.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
},
{
Filename: "ollama-linux-amd64-rocm.tgz",
Version: "0.6.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
Variants: []string{"rocm"},
},
}
t.Run("no variant prefers plain", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
})
if err != nil {
t.Fatal(err)
}
if len(res.Asset.Variants) != 0 {
t.Errorf("variants = %v, want empty", res.Asset.Variants)
}
})
t.Run("explicit variant selects it", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
Variant: "rocm",
})
if err != nil {
t.Fatal(err)
}
if !hasVariant(res.Asset.Variants, "rocm") {
t.Errorf("variants = %v, want [rocm]", res.Asset.Variants)
}
})
}
func TestResolveFormatPreference(t *testing.T) {
assets := []storage.Asset{
{
Filename: "tool-v1.0.0-linux-x86_64.tar.gz",
Version: "1.0.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
},
{
Filename: "tool-v1.0.0-linux-x86_64.tar.xz",
Version: "1.0.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.xz",
},
{
Filename: "tool-v1.0.0-linux-x86_64.tar.zst",
Version: "1.0.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.zst",
},
}
t.Run("default prefers zst", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
})
if err != nil {
t.Fatal(err)
}
if res.Asset.Format != ".tar.zst" {
t.Errorf("format = %q, want .tar.zst", res.Asset.Format)
}
})
t.Run("explicit format preference", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
Formats: []string{".tar.gz"},
})
if err != nil {
t.Fatal(err)
}
if res.Asset.Format != ".tar.gz" {
t.Errorf("format = %q, want .tar.gz", res.Asset.Format)
}
})
}
func TestResolveGitAssets(t *testing.T) {
assets := []storage.Asset{
{
Filename: "vim-commentary-v1.2",
Version: "1.2",
Channel: "stable",
Format: "git",
Download: "https://github.com/tpope/vim-commentary.git",
},
{
Filename: "vim-commentary-v1.1",
Version: "1.1",
Channel: "stable",
Format: "git",
Download: "https://github.com/tpope/vim-commentary.git",
},
}
t.Run("git assets match any platform", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
})
if err != nil {
t.Fatal(err)
}
if res.Version != "1.2" {
t.Errorf("version = %q, want 1.2", res.Version)
}
if res.Asset.Format != "git" {
t.Errorf("format = %q, want git", res.Asset.Format)
}
})
}
func TestResolveLTS(t *testing.T) {
assets := []storage.Asset{
{
Filename: "node-v22.0.0-linux-x64.tar.gz",
Version: "22.0.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
LTS: false,
},
{
Filename: "node-v20.15.0-linux-x64.tar.gz",
Version: "20.15.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
LTS: true,
},
}
t.Run("LTS selects older LTS version", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
LTS: true,
})
if err != nil {
t.Fatal(err)
}
if res.Version != "20.15.0" {
t.Errorf("version = %q, want 20.15.0", res.Version)
}
})
}

View File

@@ -1,295 +0,0 @@
// Package pgstore implements [storage.Store] on PostgreSQL.
//
// Schema uses double-buffering: two asset generations per package (0 and 1).
// The active generation pointer in webi_packages is updated atomically on
// Commit, so readers always see a complete consistent snapshot.
//
// Write path:
//
// BeginRefresh → clears inactive generation, returns tx
// Put → stages assets in-memory
// Commit → bulk-inserts assets (COPY), swaps generation pointer
//
// Read path:
//
// Load → reads active generation from webi_packages, fetches assets
//
// Connection string format: standard libpq / pgx DSN, e.g.:
//
// postgres://user:pass@host/dbname?sslmode=require
// host=localhost user=webi dbname=webi sslmode=disable
package pgstore
import (
"context"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/webinstall/webi-installers/internal/storage"
)
// Schema holds the DDL for creating the required tables.
// Run once on startup or deploy to ensure the schema exists.
const Schema = `
CREATE TABLE IF NOT EXISTS webi_packages (
name TEXT NOT NULL PRIMARY KEY,
active_gen SMALLINT NOT NULL DEFAULT 0,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS webi_assets (
id BIGSERIAL PRIMARY KEY,
pkg TEXT NOT NULL,
gen SMALLINT NOT NULL,
filename TEXT NOT NULL DEFAULT '',
version TEXT NOT NULL DEFAULT '',
lts BOOLEAN NOT NULL DEFAULT FALSE,
channel TEXT NOT NULL DEFAULT '',
date TEXT NOT NULL DEFAULT '',
os TEXT NOT NULL DEFAULT '',
arch TEXT NOT NULL DEFAULT '',
libc TEXT NOT NULL DEFAULT '',
format TEXT NOT NULL DEFAULT '',
download TEXT NOT NULL DEFAULT '',
extra TEXT NOT NULL DEFAULT '',
variants TEXT[] NOT NULL DEFAULT '{}'
);
CREATE INDEX IF NOT EXISTS webi_assets_pkg_gen ON webi_assets (pkg, gen);
`
// Store is a PostgreSQL-backed asset store.
type Store struct {
pool *pgxpool.Pool
}
// New opens a connection pool to the given DSN and applies the schema.
// Returns an error if the connection or schema creation fails.
func New(ctx context.Context, dsn string) (*Store, error) {
cfg, err := pgxpool.ParseConfig(dsn)
if err != nil {
return nil, fmt.Errorf("pgstore: parse dsn: %w", err)
}
pool, err := pgxpool.NewWithConfig(ctx, cfg)
if err != nil {
return nil, fmt.Errorf("pgstore: connect: %w", err)
}
if err := applySchema(ctx, pool); err != nil {
pool.Close()
return nil, err
}
return &Store{pool: pool}, nil
}
// Close releases the connection pool.
func (s *Store) Close() {
s.pool.Close()
}
// ListPackages returns the names of all packages in the store.
func (s *Store) ListPackages(ctx context.Context) ([]string, error) {
rows, err := s.pool.Query(ctx,
`SELECT name FROM webi_packages ORDER BY name`,
)
if err != nil {
return nil, fmt.Errorf("pgstore: list packages: %w", err)
}
defer rows.Close()
var pkgs []string
for rows.Next() {
var name string
if err := rows.Scan(&name); err != nil {
return nil, fmt.Errorf("pgstore: scan package name: %w", err)
}
pkgs = append(pkgs, name)
}
return pkgs, rows.Err()
}
// Load returns all assets for a package using the active generation.
// Returns nil (not an error) if the package is not cached.
func (s *Store) Load(ctx context.Context, pkg string) (*storage.PackageData, error) {
// Fetch active generation and updated_at.
var gen int16
var updatedAt time.Time
err := s.pool.QueryRow(ctx,
`SELECT active_gen, updated_at FROM webi_packages WHERE name = $1`,
pkg,
).Scan(&gen, &updatedAt)
if err == pgx.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("pgstore: load %s: %w", pkg, err)
}
// Fetch all assets for this generation.
rows, err := s.pool.Query(ctx, `
SELECT filename, version, lts, channel, date,
os, arch, libc, format, download, extra, variants
FROM webi_assets
WHERE pkg = $1 AND gen = $2
ORDER BY id
`, pkg, gen)
if err != nil {
return nil, fmt.Errorf("pgstore: load assets %s: %w", pkg, err)
}
defer rows.Close()
var assets []storage.Asset
for rows.Next() {
var a storage.Asset
if err := rows.Scan(
&a.Filename, &a.Version, &a.LTS, &a.Channel, &a.Date,
&a.OS, &a.Arch, &a.Libc, &a.Format, &a.Download,
&a.Extra, &a.Variants,
); err != nil {
return nil, fmt.Errorf("pgstore: scan asset %s: %w", pkg, err)
}
assets = append(assets, a)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("pgstore: rows %s: %w", pkg, err)
}
return &storage.PackageData{
Assets: assets,
UpdatedAt: updatedAt,
}, nil
}
// BeginRefresh starts a write transaction for a package.
// It determines the inactive generation and clears it, ready for new data.
func (s *Store) BeginRefresh(ctx context.Context, pkg string) (storage.RefreshTx, error) {
// Determine which generation to write into (the inactive one).
var activeGen int16
err := s.pool.QueryRow(ctx,
`SELECT active_gen FROM webi_packages WHERE name = $1`,
pkg,
).Scan(&activeGen)
if err != nil && err != pgx.ErrNoRows {
return nil, fmt.Errorf("pgstore: begin refresh %s: %w", pkg, err)
}
// If package doesn't exist yet, activeGen defaults to 0 and we write to gen 1.
// If package exists, we write to the inactive generation (1 - activeGen).
var writeGen int16
if err == pgx.ErrNoRows {
writeGen = 1
} else {
writeGen = 1 - activeGen
}
// Clear the write generation so we start fresh.
if _, err := s.pool.Exec(ctx,
`DELETE FROM webi_assets WHERE pkg = $1 AND gen = $2`,
pkg, writeGen,
); err != nil {
return nil, fmt.Errorf("pgstore: clear gen %d for %s: %w", writeGen, pkg, err)
}
return &refreshTx{
pool: s.pool,
pkg: pkg,
gen: writeGen,
}, nil
}
// refreshTx is an in-progress write for one package.
type refreshTx struct {
pool *pgxpool.Pool
pkg string
gen int16
assets []storage.Asset
}
// Put stages assets for writing. May be called multiple times.
func (tx *refreshTx) Put(assets []storage.Asset) error {
tx.assets = append(tx.assets, assets...)
return nil
}
// Commit bulk-inserts all staged assets, then atomically swaps the
// active generation pointer in webi_packages.
func (tx *refreshTx) Commit(ctx context.Context) error {
if len(tx.assets) == 0 {
return tx.swapGeneration(ctx)
}
// Build rows for pgx.CopyFromRows.
rows := make([][]any, len(tx.assets))
for i, a := range tx.assets {
variants := a.Variants
if variants == nil {
variants = []string{}
}
rows[i] = []any{
tx.pkg,
tx.gen,
a.Filename,
a.Version,
a.LTS,
a.Channel,
a.Date,
a.OS,
a.Arch,
a.Libc,
a.Format,
a.Download,
a.Extra,
variants,
}
}
cols := []string{
"pkg", "gen",
"filename", "version", "lts", "channel", "date",
"os", "arch", "libc", "format", "download", "extra", "variants",
}
_, err := tx.pool.CopyFrom(ctx,
pgx.Identifier{"webi_assets"},
cols,
pgx.CopyFromRows(rows),
)
if err != nil {
return fmt.Errorf("pgstore: copy assets %s: %w", tx.pkg, err)
}
return tx.swapGeneration(ctx)
}
// swapGeneration atomically updates the active generation pointer.
func (tx *refreshTx) swapGeneration(ctx context.Context) error {
_, err := tx.pool.Exec(ctx, `
INSERT INTO webi_packages (name, active_gen, updated_at)
VALUES ($1, $2, now())
ON CONFLICT (name)
DO UPDATE SET active_gen = $2, updated_at = now()
`, tx.pkg, tx.gen)
if err != nil {
return fmt.Errorf("pgstore: swap gen %s: %w", tx.pkg, err)
}
tx.assets = nil
return nil
}
// Rollback discards all staged assets without writing anything.
func (tx *refreshTx) Rollback() error {
tx.assets = nil
return nil
}
// applySchema runs the schema DDL idempotently.
func applySchema(ctx context.Context, pool *pgxpool.Pool) error {
if _, err := pool.Exec(ctx, Schema); err != nil {
return fmt.Errorf("pgstore: apply schema: %w", err)
}
return nil
}

View File

@@ -1,247 +0,0 @@
// Package uadetect identifies the requesting agent's OS, CPU architecture,
// and libc so the server can select the correct release artifact.
//
// An agent identifies itself through multiple signals:
// - The User-Agent header: Webi's bootstrap scripts send "$(uname -srm)",
// e.g. "Darwin 23.1.0 arm64". Browsers, curl, and PowerShell send their
// own UA strings.
// - Query parameters: ?os=linux&arch=arm64 are an explicit declaration
// that takes precedence over the header.
//
// Use [FromRequest] to detect from an HTTP request (preferred).
// Use [Parse] to detect from a raw UA string.
package uadetect
import (
"net/http"
"strings"
"github.com/webinstall/webi-installers/internal/buildmeta"
)
// Result holds the detected platform info from a User-Agent string.
type Result struct {
OS buildmeta.OS
Arch buildmeta.Arch
Libc buildmeta.Libc
}
// FromRequest detects the agent's platform from an HTTP request.
// Query parameters ?os and ?arch override the User-Agent header.
func FromRequest(r *http.Request) Result {
qOS := r.URL.Query().Get("os")
qArch := r.URL.Query().Get("arch")
var ua string
switch {
case qOS != "" && qArch != "":
ua = qOS + " " + qArch
case qOS != "":
ua = qOS
case qArch != "":
ua = qArch
default:
ua = r.Header.Get("User-Agent")
}
return Parse(ua)
}
// Parse extracts OS, arch, and libc from a User-Agent string.
func Parse(ua string) Result {
if ua == "-" {
return Result{}
}
tokens := tokenize(ua)
return Result{
OS: matchOS(tokens),
Arch: matchArch(tokens),
Libc: matchLibc(tokens),
}
}
// tokenize splits a User-Agent into lowercase tokens for matching.
// Splits on whitespace, '/', and ';', since UAs come in various forms:
//
// "Darwin 23.1.0 arm64" (uname -srm)
// "PowerShell/7.3.0" (PowerShell)
// "MS AMD64" (Windows shorthand)
// "Macintosh; Intel Mac OS X 10_15_7" (browser)
func tokenize(ua string) []string {
// Strip xnu kernel info that can mislead arch detection under Rosetta.
// "xnu-7195.60.75~1/RELEASE_ARM64_T8101" contains ARM64 even when
// running as x86_64. This only appears in verbose uname output.
if i := strings.Index(ua, "xnu-"); i >= 0 {
end := strings.IndexByte(ua[i:], ' ')
if end < 0 {
ua = ua[:i]
} else {
ua = ua[:i] + ua[i+end:]
}
}
return strings.FieldsFunc(strings.ToLower(ua), func(r rune) bool {
return r == ' ' || r == '/' || r == ';' || r == '\t'
})
}
// matchOS identifies the operating system from tokens.
// Order matters: Android before Linux, Linux before Windows (for WSL).
func matchOS(tokens []string) buildmeta.OS {
has := func(s string) bool {
for _, t := range tokens {
if strings.Contains(t, s) {
return true
}
}
return false
}
// Android must be checked before Linux.
if has("android") {
return buildmeta.OSAndroid
}
if has("darwin") || has("macos") || has("macintosh") || has("iphone") || has("ios") || has("ipad") {
return buildmeta.OSDarwin
}
// "mac" alone (not in "macintosh" which is already matched)
for _, t := range tokens {
if t == "mac" {
return buildmeta.OSDarwin
}
}
// FreeBSD before Linux (both are POSIX, but FreeBSD never reports "linux").
if has("freebsd") {
return buildmeta.OSFreeBSD
}
// Linux before Windows because WSL UAs contain both "linux" and "microsoft".
// But exclude Cygwin/Msys/MINGW which report Linux-like strings on Windows.
if has("linux") && !has("cygwin") && !has("msysgit") && !has("msys") && !has("mingw") {
return buildmeta.OSLinux
}
// Cygwin, Msys, and MINGW are Windows environments.
if has("windows") || has("win32") || has("microsoft") || has("powershell") ||
has("cygwin") || has("msys") || has("mingw") {
return buildmeta.OSWindows
}
for _, t := range tokens {
if t == "ms" || t == "win" {
return buildmeta.OSWindows
}
}
// Fallback: curl and wget imply a POSIX system, almost always Linux.
if has("curl") || has("wget") {
return buildmeta.OSLinux
}
return ""
}
// matchArch identifies the CPU architecture from tokens.
// More specific patterns are checked before less specific ones.
func matchArch(tokens []string) buildmeta.Arch {
has := func(s string) bool {
for _, t := range tokens {
if strings.Contains(t, s) {
return true
}
}
return false
}
exact := func(s string) bool {
for _, t := range tokens {
if t == s {
return true
}
}
return false
}
// ARM 64-bit (most specific first)
if has("aarch64") || has("arm64") || has("armv8") {
return buildmeta.ArchARM64
}
// ARM 32-bit variants
if has("armv7") || has("arm32") {
return buildmeta.ArchARMv7
}
if has("armv6") {
return buildmeta.ArchARMv6
}
// Bare "arm" without a version qualifier → armv6 (conservative).
if exact("arm") {
return buildmeta.ArchARMv6
}
// POWER (check before generic 64-bit)
if has("ppc64le") {
return buildmeta.ArchPPC64LE
}
if has("ppc64") {
return buildmeta.ArchPPC64
}
// s390x (IBM Z)
if has("s390x") {
return buildmeta.ArchS390X
}
// RISC-V
if has("riscv64") {
return buildmeta.ArchRISCV64
}
// MIPS (check before generic 64-bit)
if has("mips64") {
return buildmeta.ArchMIPS64
}
if has("mips") {
return buildmeta.ArchMIPS
}
// x86-64
if has("x86_64") || has("amd64") || exact("x64") {
return buildmeta.ArchAMD64
}
// x86 32-bit (after x86_64 to avoid false match)
if has("i386") || has("i686") || exact("x86") {
return buildmeta.ArchX86
}
return ""
}
// matchLibc identifies the C library from tokens.
func matchLibc(tokens []string) buildmeta.Libc {
has := func(s string) bool {
for _, t := range tokens {
if strings.Contains(t, s) {
return true
}
}
return false
}
if has("musl") {
return buildmeta.LibcMusl
}
// Don't match "microsoft" — it appears in WSL kernel version strings
// (e.g. "5.15.146.1-microsoft-standard-WSL2") and doesn't indicate MSVC.
if has("msvc") || has("windows") {
return buildmeta.LibcMSVC
}
if has("gnu") || has("glibc") || has("linux") {
return buildmeta.LibcGNU
}
return buildmeta.LibcNone
}

View File

@@ -1,190 +0,0 @@
package uadetect_test
import (
"net/http"
"testing"
"github.com/webinstall/webi-installers/internal/buildmeta"
"github.com/webinstall/webi-installers/internal/uadetect"
)
func TestOS(t *testing.T) {
tests := []struct {
ua string
want buildmeta.OS
}{
// uname -srm style
{"Darwin 23.1.0 arm64", buildmeta.OSDarwin},
{"Darwin 20.2.0 x86_64", buildmeta.OSDarwin},
{"Linux 6.1.0-18-amd64 x86_64", buildmeta.OSLinux},
{"Linux 5.15.0 aarch64", buildmeta.OSLinux},
// WSL: Linux, not Windows (contains "microsoft" in kernel release)
{"Linux 5.15.146.1-microsoft-standard-WSL2 x86_64", buildmeta.OSLinux},
// Windows
{"MS AMD64", buildmeta.OSWindows},
{"PowerShell/7.3.0", buildmeta.OSWindows},
{"Microsoft Windows 10.0.19045", buildmeta.OSWindows},
// Msys/MINGW/Cygwin → Windows
{"webi/curl x86_64/unknown Msys/MINGW64_NT-10.0-19045/3.5.7-463ebcdc.x86_64 libc", buildmeta.OSWindows},
{"webi/curl+wget x86_64/unknown Msys/MSYS_NT-10.0-26200/3.6.6-1cdd4371.x86_64 libc", buildmeta.OSWindows},
{"webi/curl x86_64/unknown Cygwin/CYGWIN_NT-10.0/2.10.0(0.325/5/3) libc", buildmeta.OSWindows},
// FreeBSD
{"webi/curl amd64/unknown FreeBSD/14.3-RELEASE-p8 libc", buildmeta.OSFreeBSD},
// Android before Linux
{"Android 13 aarch64", buildmeta.OSAndroid},
{"webi/curl aarch64/unknown Android/Linux/6.6.77-android15-8 libc", buildmeta.OSAndroid},
// WSL: Linux, not Windows (kernel contains "microsoft")
{"webi/curl+wget x86_64/unknown GNU/Linux/5.15.146.1-microsoft-standard-WSL2 libc", buildmeta.OSLinux},
// Browser-style
{"Macintosh; Intel Mac OS X 10_15_7", buildmeta.OSDarwin},
// Minimal agents → assume Linux
{"curl/8.1.2", buildmeta.OSLinux},
{"wget/1.21", buildmeta.OSLinux},
// Explicit unknown
{"-", ""},
}
for _, tt := range tests {
t.Run(tt.ua, func(t *testing.T) {
got := uadetect.Parse(tt.ua).OS
if got != tt.want {
t.Errorf("Parse(%q).OS = %q, want %q", tt.ua, got, tt.want)
}
})
}
}
func TestArch(t *testing.T) {
tests := []struct {
ua string
want buildmeta.Arch
}{
{"Darwin 23.1.0 arm64", buildmeta.ArchARM64},
{"Linux 6.1.0 aarch64", buildmeta.ArchARM64},
{"Linux 5.4.0 x86_64", buildmeta.ArchAMD64},
{"MS AMD64", buildmeta.ArchAMD64},
{"Linux 5.10.0 armv7l", buildmeta.ArchARMv7},
{"Linux 5.10.0 armv6l", buildmeta.ArchARMv6},
{"Linux 5.4.0 ppc64le", buildmeta.ArchPPC64LE},
{"webi/curl+wget s390x/unknown GNU/Linux/6.4.0-150700.53.6-default libc", buildmeta.ArchS390X},
// FreeBSD uses "amd64" not "x86_64"
{"webi/curl amd64/unknown FreeBSD/14.3-RELEASE-p8 libc", buildmeta.ArchAMD64},
// Rosetta: xnu kernel info says ARM64 but actual arch is x86_64
{"Darwin 20.2.0 Darwin Kernel Version 20.2.0; root:xnu-7195.60.75~1/RELEASE_ARM64_T8101 x86_64", buildmeta.ArchAMD64},
{"-", ""},
}
for _, tt := range tests {
t.Run(tt.ua, func(t *testing.T) {
got := uadetect.Parse(tt.ua).Arch
if got != tt.want {
t.Errorf("Parse(%q).Arch = %q, want %q", tt.ua, got, tt.want)
}
})
}
}
func TestLibc(t *testing.T) {
tests := []struct {
ua string
want buildmeta.Libc
}{
{"Linux 6.1.0 x86_64 musl", buildmeta.LibcMusl},
{"Linux 6.1.0 x86_64 gnu", buildmeta.LibcGNU},
{"Linux 6.1.0 x86_64 linux", buildmeta.LibcGNU},
{"MS AMD64 msvc", buildmeta.LibcMSVC},
{"Microsoft Windows", buildmeta.LibcMSVC},
{"Darwin 23.1.0 arm64", buildmeta.LibcNone},
// WSL: kernel version contains "microsoft" but libc is gnu, not msvc
{"webi/curl+wget x86_64/unknown GNU/Linux/5.15.146.1-microsoft-standard-WSL2 libc", buildmeta.LibcGNU},
{"-", ""},
}
for _, tt := range tests {
t.Run(tt.ua, func(t *testing.T) {
got := uadetect.Parse(tt.ua).Libc
if got != tt.want {
t.Errorf("Parse(%q).Libc = %q, want %q", tt.ua, got, tt.want)
}
})
}
}
func TestFromRequest(t *testing.T) {
tests := []struct {
name string
ua string // User-Agent header
query string // raw query string
wantOS buildmeta.OS
wantAr buildmeta.Arch
}{
{
name: "UA header only",
ua: "Darwin 23.1.0 arm64",
wantOS: buildmeta.OSDarwin,
wantAr: buildmeta.ArchARM64,
},
{
name: "query params override UA",
ua: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
query: "os=linux&arch=aarch64",
wantOS: buildmeta.OSLinux,
wantAr: buildmeta.ArchARM64,
},
{
name: "os param only",
ua: "curl/8.1.2",
query: "os=windows",
wantOS: buildmeta.OSWindows,
},
{
name: "arch param only",
ua: "curl/8.1.2",
query: "arch=arm64",
wantAr: buildmeta.ArchARM64,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
req, _ := http.NewRequest("GET", "http://example.com/api?"+tt.query, nil)
if tt.ua != "" {
req.Header.Set("User-Agent", tt.ua)
}
got := uadetect.FromRequest(req)
if tt.wantOS != "" && got.OS != tt.wantOS {
t.Errorf("OS = %q, want %q", got.OS, tt.wantOS)
}
if tt.wantAr != "" && got.Arch != tt.wantAr {
t.Errorf("Arch = %q, want %q", got.Arch, tt.wantAr)
}
})
}
}
func TestFullParse(t *testing.T) {
r := uadetect.Parse("Darwin 23.1.0 arm64")
if r.OS != buildmeta.OSDarwin {
t.Errorf("OS = %q, want %q", r.OS, buildmeta.OSDarwin)
}
if r.Arch != buildmeta.ArchARM64 {
t.Errorf("Arch = %q, want %q", r.Arch, buildmeta.ArchARM64)
}
if r.Libc != buildmeta.LibcNone {
t.Errorf("Libc = %q, want %q", r.Libc, buildmeta.LibcNone)
}
}

View File

@@ -1,4 +1,5 @@
#!/bin/sh
# shellcheck disable=SC2029,SC2088
set -e
set -u
@@ -10,13 +11,13 @@ g_out="agents/tmp/${g_bin}"
g_remote_bin="~/bin/${g_bin}"
case "${g_host}" in
beta.webi.sh) g_remote_conf="~/srv/beta.webinstall.dev/installers/" ;;
next.webi.sh) g_remote_conf="~/srv/next.webinstall.dev/installers/" ;;
*) g_remote_conf="~/srv/webid/installers/" ;;
beta.webi.sh) g_remote_conf="~/srv/beta.webinstall.dev/installers/" ;;
next.webi.sh) g_remote_conf="~/srv/next.webinstall.dev/installers/" ;;
*) g_remote_conf="~/srv/webid/installers/" ;;
esac
fn_build() {
b_version="$(git describe --tags --always 2> /dev/null || echo '0.0.0-dev')"
b_version="$(git describe --tags --always 2>/dev/null || echo '0.0.0-dev')"
b_commit="$(git rev-parse --short HEAD)"
b_date="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
b_ldflags="-X main.version=${b_version} -X main.commit=${b_commit} -X main.date=${b_date}"
@@ -28,7 +29,7 @@ fn_build() {
fn_deploy() {
printf 'Stopping %s on %s...\n' "${g_bin}" "${g_host}"
ssh "${g_host}" "~/.local/bin/serviceman stop ${g_bin}" 2> /dev/null || true
ssh "${g_host}" "~/.local/bin/serviceman stop ${g_bin}" 2>/dev/null || true
printf 'Uploading binary...\n'
scp "${g_out}" "${g_host}:${g_remote_bin}"