github/vim-ale
1
0
Fork 0
mirror of https://github.com/webinstall/webi-installers.git synced 2026-08-23 22:16:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(tools): add development and testing utilities (classify, comparecache, e2etest, fetchraw, inspect, uaparse)

Browse Source
This commit is contained in:
AJ ONeal
2026-05-14 17:49:29 -06:00
parent 1bc9e40bf6
commit f5a8a0cd5a
6 changed files with 4896 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/classify/main.go
+1293
View File
File diff suppressed because it is too large Load Diff
cmd/comparecache/main.go
+914
View File
@@ -0,0 +1,914 @@
// Command comparecache compares Go-generated cache output against the
// Node.js LIVE_cache. It identifies categorical differences in asset
// selection — which filenames appear in one cache but not the other.
//
// The comparison is done at the filename level (not OS/arch/ext fields)
// because the Node.js cache leaves those empty (normalize.js fills them
// at serve time), while the Go pipeline classifies at write time.
//
// Usage:
//
// go run ./cmd/comparecache -live ./LIVE_cache -go ./_cache
// go run ./cmd/comparecache -live ./LIVE_cache -go ./_cache bat jq
// go run ./cmd/comparecache -live ./LIVE_cache -go ./_cache -summary
package main
import (
"encoding/json"
"flag"
"fmt"
"log"
"math/rand/v2"
"os"
"path/filepath"
"slices"
"sort"
"strings"
"time"
"github.com/webinstall/webi-installers/internal/classify"
"github.com/webinstall/webi-installers/internal/lexver"
)
type cacheEntry struct {
Releases []cacheRelease `json:"releases"`
}
type cacheRelease struct {
Name string `json:"name"`
Filename string `json:"_filename"` // Node.js uses _filename for some sources
Version string `json:"version"`
Download string `json:"download"`
Channel string `json:"channel"`
OS string `json:"os"`
Arch string `json:"arch"`
Libc string `json:"libc"`
Ext string `json:"ext"`
}
// fieldDiff records a field-level difference for an asset that exists
// in both caches (same filename) but has different classification.
type fieldDiff struct {
Filename string
Field string // "os", "arch", "libc", "ext", "channel"
Live string
Go string
BothSet bool // true when both live and go have non-empty values
}
type packageDiff struct {
Name string
LiveCount int
GoCount int
OnlyInLive []string // filenames only in Node.js cache
OnlyInGo []string // filenames only in Go cache
FieldDiffs []fieldDiff // classification differences on shared assets
VersionsLive []string // unique versions in live
VersionsGo []string // unique versions in go
GoMissing bool // true if Go didn't produce output for this package
LiveMissing bool // true if no live cache for this package
Categories []string // categorical difference labels
}
func main() {
liveDir := flag.String("live", "./LIVE_cache", "path to Node.js LIVE_cache directory")
goDir := flag.String("go", "./_cache", "path to Go cache directory")
summary := flag.Bool("summary", false, "only print summary, not per-package details")
diffsOnly := flag.Bool("diffs", false, "only show packages with asset differences (skip matches)")
latest := flag.Bool("latest", false, "only compare latest version in each cache")
windowed := flag.Bool("windowed", false, "limit Go versions to the Node.js version range (2nd to 2nd-to-last)")
sample := flag.Int("sample", 0, "for each package diff, show N randomly sampled assets (implies -windowed -diffs)")
flag.Parse()
filterPkgs := flag.Args()
// -sample implies -windowed and -diffs so we focus on real classification
// differences, not version-depth noise.
if *sample > 0 {
*windowed = true
*diffsOnly = true
}
totalStart := time.Now()
// Find the most recent month directory in each cache.
liveMonth := findLatestMonth(*liveDir)
goMonth := findLatestMonth(*goDir)
if liveMonth == "" {
log.Fatalf("no month directories found in %s", *liveDir)
}
livePath := filepath.Join(*liveDir, liveMonth)
goPath := ""
if goMonth != "" {
goPath = filepath.Join(*goDir, goMonth)
}
// Discover all packages across both caches.
discoverStart := time.Now()
allPkgs := discoverPackages(livePath, goPath)
if len(filterPkgs) > 0 {
nameSet := make(map[string]bool, len(filterPkgs))
for _, n := range filterPkgs {
nameSet[n] = true
}
var filtered []string
for _, p := range allPkgs {
if nameSet[p] {
filtered = append(filtered, p)
}
}
allPkgs = filtered
}
log.Printf("discovered %d packages in %s", len(allPkgs), time.Since(discoverStart))
compareStart := time.Now()
var diffs []packageDiff
for _, pkg := range allPkgs {
d := compare(livePath, goPath, pkg, *latest, *windowed)
categorize(&d)
diffs = append(diffs, d)
}
log.Printf("compared %d packages in %s", len(diffs), time.Since(compareStart))
if *summary {
printSummary(diffs)
} else {
printDetails(diffs, *diffsOnly, *sample)
}
log.Printf("total: %s", time.Since(totalStart))
}
func findLatestMonth(dir string) string {
entries, err := os.ReadDir(dir)
if err != nil {
return ""
}
var months []string
for _, e := range entries {
if e.IsDir() && len(e.Name()) == 7 && e.Name()[4] == '-' {
months = append(months, e.Name())
}
}
if len(months) == 0 {
return ""
}
sort.Strings(months)
return months[len(months)-1]
}
func discoverPackages(livePath, goPath string) []string {
seen := make(map[string]bool)
for _, dir := range []string{livePath, goPath} {
if dir == "" {
continue
}
entries, err := os.ReadDir(dir)
if err != nil {
continue
}
for _, e := range entries {
name := e.Name()
if strings.HasSuffix(name, ".json") && !strings.HasSuffix(name, ".updated.txt") {
pkg := strings.TrimSuffix(name, ".json")
seen[pkg] = true
}
}
}
var pkgs []string
for p := range seen {
pkgs = append(pkgs, p)
}
sort.Strings(pkgs)
return pkgs
}
func loadCache(dir, pkg string) *cacheEntry {
if dir == "" {
return nil
}
data, err := os.ReadFile(filepath.Join(dir, pkg+".json"))
if err != nil {
return nil
}
var entry cacheEntry
if err := json.Unmarshal(data, &entry); err != nil {
return nil
}
return &entry
}
// effectiveName returns the best available filename for a release entry.
// Node.js sometimes uses _filename (a path) instead of name.
func effectiveName(name, filename, download string) string {
if name != "" {
return name
}
if filename != "" {
// _filename may be a path like "stable/macos/flutter_macos_3.41.4.zip"
if i := strings.LastIndex(filename, "/"); i >= 0 {
return filename[i+1:]
}
return filename
}
// Last resort: basename of download URL.
if download != "" {
if i := strings.LastIndex(download, "/"); i >= 0 {
return download[i+1:]
}
}
return ""
}
// versionWindow returns the 2nd and 2nd-to-last versions from a sorted
// version list. This trims the edges where Node.js may have a newer fetch
// or Go may have deeper history, focusing on the overlapping middle.
func versionWindow(versions []string) (low, high string) {
if len(versions) <= 2 {
// Too few versions to window — use all.
if len(versions) > 0 {
return versions[0], versions[len(versions)-1]
}
return "", ""
}
// 2nd version (skip oldest) and 2nd-to-last (skip newest).
return versions[1], versions[len(versions)-2]
}
// filterVersionRange returns only the versions in sorted order that fall
// within [low, high] inclusive (by lexver comparison).
func filterVersionRange(vf map[string]map[string]bool, versions []string, low, high string) (map[string]bool, []string) {
lowV := lexver.Parse(low)
highV := lexver.Parse(high)
files := make(map[string]bool)
var kept []string
for _, v := range versions {
pv := lexver.Parse(v)
if lexver.Compare(pv, lowV) >= 0 && lexver.Compare(pv, highV) <= 0 {
kept = append(kept, v)
for f := range vf[v] {
files[f] = true
}
}
}
return files, kept
}
func compare(livePath, goPath, pkg string, latestOnly, windowed bool) packageDiff {
live := loadCache(livePath, pkg)
goCache := loadCache(goPath, pkg)
d := packageDiff{Name: pkg}
if live == nil {
d.LiveMissing = true
}
if goCache == nil {
d.GoMissing = true
}
if d.LiveMissing && d.GoMissing {
return d
}
normVersion := normalizeVersionFunc(pkg)
// Collect filenames by version. If filter is non-nil, skip filenames it rejects.
extractVersionFiles := func(ce *cacheEntry, filter func(string) bool) (map[string]map[string]bool, []string) {
vf := make(map[string]map[string]bool)
for _, r := range ce.Releases {
name := effectiveName(r.Name, r.Filename, r.Download)
if filter != nil && !filter(name) {
continue
}
ver := normVersion(r.Version)
if vf[ver] == nil {
vf[ver] = make(map[string]bool)
}
vf[ver][name] = true
}
var versions []string
for v := range vf {
versions = append(versions, v)
}
slices.SortFunc(versions, func(a, b string) int {
return lexver.Compare(lexver.Parse(a), lexver.Parse(b))
})
return vf, versions
}
notNoise := func(name string) bool { return !isLiveNoise(name) }
var liveFiles, goFiles map[string]bool
// Parse live cache.
var liveVF map[string]map[string]bool
var liveVersions []string
if live != nil {
liveVF, liveVersions = extractVersionFiles(live, notNoise)
d.VersionsLive = liveVersions
d.LiveCount = len(live.Releases)
}
// Parse Go cache.
var goVF map[string]map[string]bool
var goVersions []string
if goCache != nil {
goVF, goVersions = extractVersionFiles(goCache, notNoise)
d.VersionsGo = goVersions
d.GoCount = len(goCache.Releases)
}
// Determine which files to compare based on mode.
if latestOnly {
// Compare only the latest version from each cache.
if live != nil && len(liveVersions) > 0 {
liveFiles = liveVF[liveVersions[len(liveVersions)-1]]
}
if goCache != nil && len(goVersions) > 0 {
goFiles = goVF[goVersions[len(goVersions)-1]]
}
} else if windowed && live != nil && len(liveVersions) > 0 {
// Use the Node.js version range (2nd to 2nd-to-last) to establish
// the window. Include ALL Node.js versions in the window (so missing
// Go versions are visible), but exclude Go-only versions (those are
// just deeper history, not real gaps).
low, high := versionWindow(liveVersions)
lowV := lexver.Parse(low)
highV := lexver.Parse(high)
// Collect all live files in the window.
liveFiles = make(map[string]bool)
liveInWindow := make(map[string]bool)
for _, v := range liveVersions {
pv := lexver.Parse(v)
if lexver.Compare(pv, lowV) >= 0 && lexver.Compare(pv, highV) <= 0 {
liveInWindow[v] = true
for f := range liveVF[v] {
liveFiles[f] = true
}
}
}
// For Go, only include versions that Node.js also has in the window.
// Go-only versions are hidden (deeper history, not gaps).
goFiles = make(map[string]bool)
for _, v := range goVersions {
if !liveInWindow[v] {
continue
}
for f := range goVF[v] {
goFiles[f] = true
}
}
} else {
// Compare all versions — use pre-filtered version maps.
if live != nil {
liveFiles = make(map[string]bool)
for _, files := range liveVF {
for f := range files {
liveFiles[f] = true
}
}
}
if goCache != nil {
goFiles = make(map[string]bool)
for _, files := range goVF {
for f := range files {
goFiles[f] = true
}
}
}
}
if liveFiles == nil {
liveFiles = make(map[string]bool)
}
if goFiles == nil {
goFiles = make(map[string]bool)
}
for f := range liveFiles {
if !goFiles[f] {
d.OnlyInLive = append(d.OnlyInLive, f)
}
}
for f := range goFiles {
if !liveFiles[f] {
d.OnlyInGo = append(d.OnlyInGo, f)
}
}
sort.Strings(d.OnlyInLive)
sort.Strings(d.OnlyInGo)
// Field-level comparison on assets that exist in both caches.
// Build version+filename → fields maps from each cache.
if live != nil && goCache != nil {
type assetKey struct {
version string
filename string
}
liveByKey := make(map[assetKey]cacheRelease)
for _, r := range live.Releases {
name := effectiveName(r.Name, r.Filename, r.Download)
ver := normVersion(r.Version)
liveByKey[assetKey{ver, name}] = r
}
for _, r := range goCache.Releases {
name := effectiveName(r.Name, r.Filename, r.Download)
ver := normVersion(r.Version)
lr, ok := liveByKey[assetKey{ver, name}]
if !ok {
continue
}
// Compare classification fields.
// Use equivalence checks for os/arch/ext so naming
// convention differences don't mask real classification bugs.
for _, cmp := range []struct {
field string
live string
go_ string
equiv bool
}{
{"os", lr.OS, r.OS, equivOS(lr.OS, r.OS)},
{"arch", lr.Arch, r.Arch, equivArch(lr.Arch, r.Arch)},
{"libc", lr.Libc, r.Libc, lr.Libc == r.Libc},
{"ext", lr.Ext, r.Ext, equivExt(lr.Ext, r.Ext)},
{"channel", lr.Channel, r.Channel, lr.Channel == r.Channel},
} {
if cmp.equiv {
continue
}
d.FieldDiffs = append(d.FieldDiffs, fieldDiff{
Filename: name,
Field: cmp.field,
Live: cmp.live,
Go: cmp.go_,
BothSet: cmp.live != "" && cmp.go_ != "",
})
}
}
sort.Slice(d.FieldDiffs, func(i, j int) bool {
if d.FieldDiffs[i].Field != d.FieldDiffs[j].Field {
return d.FieldDiffs[i].Field < d.FieldDiffs[j].Field
}
return d.FieldDiffs[i].Filename < d.FieldDiffs[j].Filename
})
}
return d
}
// equivOS returns true if two OS values are equivalent across naming conventions.
func equivOS(a, b string) bool {
return a == b || canonicalOS(a) == canonicalOS(b)
}
func canonicalOS(s string) string {
switch strings.ToLower(s) {
case "darwin", "macos", "mac", "osx":
return "darwin"
case "win", "windows":
return "windows"
default:
return strings.ToLower(s)
}
}
// equivArch returns true if two arch values are equivalent.
func equivArch(a, b string) bool {
return a == b || canonicalArch(a) == canonicalArch(b)
}
func canonicalArch(s string) string {
switch strings.ToLower(s) {
case "x86_64", "amd64", "x64":
return "x86_64"
case "aarch64", "arm64":
return "aarch64"
case "armv7", "armv7l":
return "armv7"
case "armv6", "armv6l":
return "armv6"
case "x86", "i386", "i686", "386":
return "x86"
default:
return strings.ToLower(s)
}
}
// equivExt returns true if two extension values are equivalent.
func equivExt(a, b string) bool {
// Normalize: strip leading dot, handle common aliases.
return a == b || canonicalExt(a) == canonicalExt(b)
}
func canonicalExt(s string) string {
s = strings.TrimPrefix(s, ".")
switch s {
case "tgz":
return "tar.gz"
default:
return s
}
}
func categorize(d *packageDiff) {
if d.GoMissing {
d.Categories = append(d.Categories, "go-missing")
return
}
if d.LiveMissing {
d.Categories = append(d.Categories, "live-missing")
return
}
if len(d.OnlyInLive) == 0 && len(d.OnlyInGo) == 0 && len(d.FieldDiffs) == 0 {
d.Categories = append(d.Categories, "match")
return
}
if len(d.OnlyInLive) == 0 && len(d.OnlyInGo) == 0 && len(d.FieldDiffs) > 0 {
d.Categories = append(d.Categories, "fields-only")
}
// Check if differences are only version depth (Go has more history).
liveVersionSet := make(map[string]bool, len(d.VersionsLive))
for _, v := range d.VersionsLive {
liveVersionSet[v] = true
}
goVersionSet := make(map[string]bool, len(d.VersionsGo))
for _, v := range d.VersionsGo {
goVersionSet[v] = true
}
goExtraVersions := 0
for _, v := range d.VersionsGo {
if !liveVersionSet[v] {
goExtraVersions++
}
}
liveExtraVersions := 0
for _, v := range d.VersionsLive {
if !goVersionSet[v] {
liveExtraVersions++
}
}
if goExtraVersions > 0 {
d.Categories = append(d.Categories, fmt.Sprintf("go-extra-versions(%d)", goExtraVersions))
}
if liveExtraVersions > 0 {
d.Categories = append(d.Categories, fmt.Sprintf("live-extra-versions(%d)", liveExtraVersions))
}
// Check for meta-asset filtering differences.
metaOnlyInLive := 0
nonMetaOnlyInLive := 0
for _, f := range d.OnlyInLive {
if classify.IsMetaAsset(f) {
metaOnlyInLive++
} else {
nonMetaOnlyInLive++
}
}
metaOnlyInGo := 0
nonMetaOnlyInGo := 0
for _, f := range d.OnlyInGo {
if classify.IsMetaAsset(f) {
metaOnlyInGo++
} else {
nonMetaOnlyInGo++
}
}
if metaOnlyInLive > 0 {
d.Categories = append(d.Categories, fmt.Sprintf("live-has-meta(%d)", metaOnlyInLive))
}
if metaOnlyInGo > 0 {
d.Categories = append(d.Categories, fmt.Sprintf("go-has-meta(%d)", metaOnlyInGo))
}
// Check for source tarball differences.
srcOnlyInGo := 0
for _, f := range d.OnlyInGo {
if strings.HasSuffix(f, ".tar.gz") || strings.HasSuffix(f, ".zip") {
if strings.HasPrefix(f, "v") || strings.HasPrefix(f, "refs/") {
srcOnlyInGo++
}
}
}
if srcOnlyInGo > 0 {
d.Categories = append(d.Categories, fmt.Sprintf("go-has-source-tarballs(%d)", srcOnlyInGo))
}
if nonMetaOnlyInLive > 0 {
d.Categories = append(d.Categories, fmt.Sprintf("live-extra-assets(%d)", nonMetaOnlyInLive))
}
if nonMetaOnlyInGo > 0 {
d.Categories = append(d.Categories, fmt.Sprintf("go-extra-assets(%d)", nonMetaOnlyInGo))
}
// Count field diffs by field name, separating real disagreements
// from expected "live empty, Go classified" differences.
type fieldCount struct {
bothSet int // both caches have a value but they disagree
oneEmpty int // one side is empty (typically live — normalize.js fills at serve time)
}
fieldCounts := make(map[string]fieldCount)
for _, fd := range d.FieldDiffs {
fc := fieldCounts[fd.Field]
if fd.BothSet {
fc.bothSet++
} else {
fc.oneEmpty++
}
fieldCounts[fd.Field] = fc
}
for _, field := range []string{"os", "arch", "libc", "ext", "channel"} {
fc := fieldCounts[field]
if fc.bothSet > 0 {
d.Categories = append(d.Categories, fmt.Sprintf("diff-%s(%d)", field, fc.bothSet))
}
if fc.oneEmpty > 0 {
d.Categories = append(d.Categories, fmt.Sprintf("fill-%s(%d)", field, fc.oneEmpty))
}
}
}
// isLiveNoise returns true for filenames that the Node.js cache keeps
// but Go intentionally filters out. Pre-filtering these from the live
// side prevents them from appearing as live-extra-assets noise.
//
// This includes everything classify.IsMetaAsset catches plus formats
// that Go's legacy export strips (.deb, .rpm, etc.).
func isLiveNoise(name string) bool {
if classify.IsMetaAsset(name) {
return true
}
lower := strings.ToLower(name)
// Formats Go filters from legacy export but Node.js keeps.
for _, suffix := range []string{
".deb", ".rpm", ".gpg",
} {
if strings.HasSuffix(lower, suffix) {
return true
}
}
// Source tarballs (e.g. gitea-src-1.25.4.tar.gz, caddy_2.10.0_src.tar.gz, go1.26.1.src.tar.gz).
if strings.Contains(lower, "-src-") || strings.Contains(lower, "_src.") || strings.Contains(lower, ".src.") || strings.HasPrefix(lower, "src-") {
return true
}
// Docs tarballs (e.g. gitea-docs-1.22.3.tar.gz).
if strings.Contains(lower, "-docs-") {
return true
}
// Bare executables without any extension — typically legacy shell scripts
// uploaded alongside proper archives (e.g. kubectx, kubens).
if !strings.Contains(name, ".") {
return true
}
// GPU accelerator / hardware variants that Go tags as variants
// but Node.js keeps with special arch names.
for _, v := range []string{"-rocm", "-jetpack"} {
if strings.Contains(lower, v) {
return true
}
}
// Linux binaries for packages where Node.js only kept macOS .app.zip.
// Go correctly includes these as installable on Linux.
if strings.HasPrefix(lower, "fish-") && strings.Contains(lower, "-linux-") {
return true
}
return false
}
// normalizeVersionFunc returns a version normalizer for a given package.
// Most packages return the identity function. Some (like git) need
// version string normalization to match across Go and Node.js caches.
func normalizeVersionFunc(pkg string) func(string) string {
switch pkg {
case "git":
return func(v string) string {
// Git for Windows: v2.53.0.windows.1 → v2.53.0
// v2.53.0.windows.2 → v2.53.0.2
idx := strings.Index(v, ".windows.")
if idx < 0 {
return v
}
suffix := v[idx+len(".windows."):]
base := v[:idx]
if suffix == "1" {
return base
}
return base + "." + suffix
}
case "lf":
return func(v string) string {
// lf: r21 → 0.21.0
if strings.HasPrefix(v, "r") {
return "0." + v[1:] + ".0"
}
return v
}
case "bun":
return func(v string) string {
// bun: bun-v1.3.9 → v1.3.9
return strings.TrimPrefix(v, "bun-")
}
case "watchexec":
return func(v string) string {
// watchexec monorepo: cli-v1.20.5 → v1.20.5
return strings.TrimPrefix(v, "cli-")
}
case "go":
return func(v string) string {
// Go: go1.10 → 1.10.0 (pad to 3 parts)
v = strings.TrimPrefix(v, "go")
parts := strings.SplitN(v, ".", 3)
for len(parts) < 3 {
parts = append(parts, "0")
}
return strings.Join(parts, ".")
}
default:
return func(v string) string { return v }
}
}
func printSummary(diffs []packageDiff) {
// Count by category.
categoryCounts := make(map[string]int)
for _, d := range diffs {
for _, c := range d.Categories {
// Strip the count suffix for grouping.
base := c
if idx := strings.Index(c, "("); idx != -1 {
base = c[:idx]
}
categoryCounts[base]++
}
}
fmt.Println("=== COMPARISON SUMMARY ===")
fmt.Printf("Total packages: %d\n\n", len(diffs))
var cats []string
for c := range categoryCounts {
cats = append(cats, c)
}
sort.Strings(cats)
for _, c := range cats {
fmt.Printf(" %-30s %d\n", c, categoryCounts[c])
}
fmt.Println("\n=== PER-PACKAGE CATEGORIES ===")
for _, d := range diffs {
fmt.Printf("%-25s %s\n", d.Name, strings.Join(d.Categories, ", "))
}
}
func printDetails(diffs []packageDiff, diffsOnly bool, sampleN int) {
for _, d := range diffs {
if diffsOnly && len(d.OnlyInLive) == 0 && len(d.OnlyInGo) == 0 && len(d.FieldDiffs) == 0 {
continue
}
fmt.Printf("=== %s ===\n", d.Name)
fmt.Printf(" Categories: %s\n", strings.Join(d.Categories, ", "))
fmt.Printf(" Live: %d assets, %d versions | Go: %d assets, %d versions\n",
d.LiveCount, len(d.VersionsLive), d.GoCount, len(d.VersionsGo))
printAssetList("Only in LIVE", d.OnlyInLive, sampleN)
printAssetList("Only in Go", d.OnlyInGo, sampleN)
printFieldDiffs(d.FieldDiffs, sampleN)
fmt.Println()
}
}
// printFieldDiffs shows classification differences on shared assets.
// Shows "real" diffs (both sides non-empty) first, then "fill" diffs
// (one side empty) as a summary count only.
func printFieldDiffs(diffs []fieldDiff, sampleN int) {
if len(diffs) == 0 {
return
}
// Separate real disagreements from fill diffs.
var real, fill []fieldDiff
for _, fd := range diffs {
if fd.BothSet {
real = append(real, fd)
} else {
fill = append(fill, fd)
}
}
// Show real disagreements in detail.
if len(real) > 0 {
byField := make(map[string][]fieldDiff)
for _, fd := range real {
byField[fd.Field] = append(byField[fd.Field], fd)
}
for _, field := range []string{"os", "arch", "libc", "ext", "channel"} {
fds := byField[field]
if len(fds) == 0 {
continue
}
fmt.Printf(" DISAGREE %s (%d):\n", field, len(fds))
printFieldDiffItems(fds, sampleN)
}
}
// Summarize fill diffs (live empty, Go classified) as counts.
if len(fill) > 0 {
byField := make(map[string]int)
for _, fd := range fill {
byField[fd.Field]++
}
var parts []string
for _, field := range []string{"os", "arch", "libc", "ext", "channel"} {
if n := byField[field]; n > 0 {
parts = append(parts, fmt.Sprintf("%s(%d)", field, n))
}
}
if len(parts) > 0 {
fmt.Printf(" Go fills empty: %s\n", strings.Join(parts, ", "))
}
}
}
func printFieldDiffItems(fds []fieldDiff, sampleN int) {
items := fds
if sampleN > 0 && len(items) > sampleN {
sampled := make([]fieldDiff, len(items))
copy(sampled, items)
rand.Shuffle(len(sampled), func(i, j int) {
sampled[i], sampled[j] = sampled[j], sampled[i]
})
items = sampled[:sampleN]
sort.Slice(items, func(i, j int) bool {
return items[i].Filename < items[j].Filename
})
}
limit := 20
for i, fd := range items {
if sampleN == 0 && i >= limit {
fmt.Printf(" ... and %d more\n", len(fds)-limit)
break
}
fmt.Printf(" - %s: live=%q go=%q\n", fd.Filename, fd.Live, fd.Go)
}
if sampleN > 0 && len(fds) > sampleN {
fmt.Printf(" ... sampled %d of %d\n", sampleN, len(fds))
}
}
// printAssetList prints a list of asset filenames, optionally sampling N at
// random. When sampleN > 0 and the list is longer, it picks N random items
// so you can spot classification bugs across the full range instead of only
// seeing the first alphabetical entries.
func printAssetList(label string, items []string, sampleN int) {
if len(items) == 0 {
return
}
fmt.Printf(" %s (%d):\n", label, len(items))
if sampleN > 0 && len(items) > sampleN {
// Shuffle a copy, take first N, then sort for readable output.
sampled := make([]string, len(items))
copy(sampled, items)
rand.Shuffle(len(sampled), func(i, j int) {
sampled[i], sampled[j] = sampled[j], sampled[i]
})
picked := sampled[:sampleN]
sort.Strings(picked)
for _, f := range picked {
fmt.Printf(" - %s\n", f)
}
fmt.Printf(" ... sampled %d of %d (run again for different sample)\n", sampleN, len(items))
return
}
limit := 20
for i, f := range items {
if i >= limit {
fmt.Printf(" ... and %d more\n", len(items)-limit)
break
}
fmt.Printf(" - %s\n", f)
}
}
cmd/e2etest/main.go
+846
View File
@@ -0,0 +1,846 @@
// Command e2etest runs the full release pipeline for selected packages
// and compares results against the live webi.sh API.
//
// It fetches from upstream, classifies assets, resolves the best match
// for a set of test queries, then fetches the same queries from the live
// API and reports any differences.
//
// Usage:
//
// go run ./cmd/e2etest
// go run ./cmd/e2etest -packages goreleaser,ollama,node
// go run ./cmd/e2etest -cache ./_cache/raw # reuse existing cache
package main
import (
"bufio"
"context"
"encoding/json"
"flag"
"fmt"
"io"
"log"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/webinstall/webi-installers/internal/buildmeta"
"github.com/webinstall/webi-installers/internal/installerconf"
"github.com/webinstall/webi-installers/internal/lexver"
"github.com/webinstall/webi-installers/internal/rawcache"
"github.com/webinstall/webi-installers/internal/releases/github"
"github.com/webinstall/webi-installers/internal/releases/githubish"
"github.com/webinstall/webi-installers/internal/releases/nodedist"
"github.com/webinstall/webi-installers/internal/resolve"
)
// testCase is one query to resolve and compare against the live API.
type testCase struct {
Name string
Package string
OS buildmeta.OS
Arch buildmeta.Arch
Libc buildmeta.Libc
Formats []string
UA string // User-Agent for live API query
}
// liveResult holds parsed fields from the live webi API response.
type liveResult struct {
Version string
OS string
Arch string
Libc string
Ext string
PkgURL string
PkgFile string
Channel string
Stable string
Latest string
Oses string
Arches string
Libcs string
Formats string
}
// UA format from webi.sh bootstrap: "curl {uname -s}/{uname -r} {uname -m}/unknown {libc}"
// libc is "gnu", "musl", or "libc" (for darwin/other)
var cases = []testCase{
{
Name: "goreleaser/linux/x86_64", Package: "goreleaser",
OS: buildmeta.OSLinux, Arch: buildmeta.ArchAMD64, Libc: buildmeta.LibcGNU,
Formats: []string{".tar.gz", ".tar.xz", ".zip"},
UA: "curl Linux/6.6.123 x86_64/unknown gnu",
},
{
Name: "goreleaser/darwin/arm64", Package: "goreleaser",
OS: buildmeta.OSDarwin, Arch: buildmeta.ArchARM64, Libc: "",
Formats: []string{".tar.gz", ".tar.xz", ".zip"},
UA: "curl Darwin/25.2.0 arm64/unknown libc",
},
{
Name: "goreleaser/windows/x86_64", Package: "goreleaser",
OS: buildmeta.OSWindows, Arch: buildmeta.ArchAMD64, Libc: "",
Formats: []string{".zip", ".exe"},
UA: "PowerShell/7.0 Windows/10.0 x86_64/unknown msvc",
},
{
Name: "ollama/linux/x86_64", Package: "ollama",
OS: buildmeta.OSLinux, Arch: buildmeta.ArchAMD64, Libc: buildmeta.LibcGNU,
Formats: []string{".tar.gz", ".tar.xz", ".tar.zst", ".zip"},
UA: "curl Linux/6.6.123 x86_64/unknown gnu",
},
{
Name: "ollama/darwin/arm64", Package: "ollama",
OS: buildmeta.OSDarwin, Arch: buildmeta.ArchARM64, Libc: "",
Formats: []string{".tar.gz", ".tar.xz", ".tar.zst", ".zip", ".dmg"},
UA: "curl Darwin/25.2.0 arm64/unknown libc",
},
{
Name: "ollama/linux/arm64", Package: "ollama",
OS: buildmeta.OSLinux, Arch: buildmeta.ArchARM64, Libc: buildmeta.LibcGNU,
Formats: []string{".tar.gz", ".tar.xz", ".tar.zst", ".zip"},
UA: "curl Linux/6.6.123 aarch64/unknown gnu",
},
{
Name: "node/linux/x86_64", Package: "node",
OS: buildmeta.OSLinux, Arch: buildmeta.ArchAMD64, Libc: buildmeta.LibcGNU,
Formats: []string{".tar.xz", ".tar.gz", ".zip"},
UA: "curl Linux/6.6.123 x86_64/unknown gnu",
},
{
Name: "node/darwin/arm64", Package: "node",
OS: buildmeta.OSDarwin, Arch: buildmeta.ArchARM64, Libc: "",
Formats: []string{".tar.xz", ".tar.gz", ".zip"},
UA: "curl Darwin/25.2.0 arm64/unknown libc",
},
{
Name: "node/linux/arm64", Package: "node",
OS: buildmeta.OSLinux, Arch: buildmeta.ArchARM64, Libc: buildmeta.LibcGNU,
Formats: []string{".tar.xz", ".tar.gz", ".zip"},
UA: "curl Linux/6.6.123 aarch64/unknown gnu",
},
}
func main() {
cacheDir := flag.String("cache", "_cache/raw", "root directory for raw cache")
confDir := flag.String("conf", ".", "root directory containing {pkg}/releases.conf files")
token := flag.String("token", os.Getenv("GITHUB_TOKEN"), "GitHub API token")
skipFetch := flag.Bool("skip-fetch", false, "skip fetching, use existing cache")
skipLive := flag.Bool("skip-live", false, "skip live API comparison")
packages := flag.String("packages", "goreleaser,ollama,node", "comma-separated packages to test")
flag.Parse()
pkgList := strings.Split(*packages, ",")
pkgSet := make(map[string]bool, len(pkgList))
for _, p := range pkgList {
pkgSet[strings.TrimSpace(p)] = true
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
client := &http.Client{Timeout: 30 * time.Second}
var auth *githubish.Auth
if *token != "" {
auth = &githubish.Auth{Token: *token}
}
// Step 1: Fetch raw releases.
if !*skipFetch {
log.Println("=== Step 1: Fetching releases ===")
for _, pkg := range pkgList {
if err := fetchPackage(ctx, client, *cacheDir, *confDir, pkg, auth); err != nil {
log.Fatalf("fetch %s: %v", pkg, err)
}
}
} else {
log.Println("=== Step 1: Skipping fetch (using cache) ===")
}
// Step 2: Classify releases.
log.Println("=== Step 2: Classifying releases ===")
allDists := make(map[string][]resolve.Dist)
for _, pkg := range pkgList {
conf, err := installerconf.Read(filepath.Join(*confDir, pkg, "releases.conf"))
if err != nil {
log.Fatalf("read conf %s: %v", pkg, err)
}
d, err := rawcache.Open(filepath.Join(*cacheDir, pkg))
if err != nil {
log.Fatalf("open cache %s: %v", pkg, err)
}
dists, err := classifyFromCache(pkg, conf, d)
if err != nil {
log.Fatalf("classify %s: %v", pkg, err)
}
allDists[pkg] = dists
log.Printf(" %s: %d distributables", pkg, len(dists))
// Show catalog.
cat := resolve.Survey(dists)
log.Printf(" oses=%v arches=%v libcs=%v formats=%v", cat.OSes, cat.Arches, cat.Libcs, cat.Formats)
log.Printf(" latest=%s stable=%s", cat.Latest, cat.Stable)
}
// Step 3: Resolve best match for each test case.
log.Println("=== Step 3: Resolving best matches ===")
type result struct {
tc testCase
match *resolve.Match
live *liveResult
}
var results []result
for _, tc := range cases {
if !pkgSet[tc.Package] {
continue
}
dists := allDists[tc.Package]
q := resolve.Query{
OS: tc.OS,
Arch: tc.Arch,
Libc: tc.Libc,
Formats: tc.Formats,
Channel: "stable",
}
m := resolve.Best(dists, q)
results = append(results, result{tc: tc, match: m})
}
// Step 4: Compare with live API.
if !*skipLive {
log.Println("=== Step 4: Comparing with live API ===")
for i := range results {
tc := results[i].tc
live, err := queryLiveAPI(client, tc)
if err != nil {
log.Printf(" %s: live API error: %v", tc.Name, err)
continue
}
results[i].live = live
}
}
// Step 5: Report.
log.Println("")
log.Println("=== Results ===")
log.Println("")
pass, fail, warn := 0, 0, 0
for _, r := range results {
tc := r.tc
m := r.match
live := r.live
if m == nil {
log.Printf("FAIL %s: no match found", tc.Name)
fail++
continue
}
log.Printf("--- %s ---", tc.Name)
log.Printf(" Go: version=%s file=%s ext=%s url=%s", m.Version, m.Filename, m.Format, m.Download)
if live != nil {
log.Printf(" Live: version=%s file=%s ext=%s url=%s", live.Version, live.PkgFile, live.Ext, live.PkgURL)
if live.Version == "0.0.0" {
log.Printf(" WARN: live API returned error (no match)")
warn++
} else if m.Version == live.Version && m.Filename == live.PkgFile {
log.Printf(" PASS: exact match")
pass++
} else if m.Version == live.Version && m.Download == live.PkgURL {
log.Printf(" PASS: same URL (filename display differs: go=%s live=%s)", m.Filename, live.PkgFile)
pass++
} else if m.Version == live.Version {
log.Printf(" WARN: same version, different file (go=%s live=%s)", m.Filename, live.PkgFile)
warn++
} else {
log.Printf(" DIFF: version mismatch (go=%s live=%s)", m.Version, live.Version)
fail++
}
} else {
log.Printf(" (no live comparison)")
pass++
}
}
log.Println("")
log.Printf("Summary: %d pass, %d fail, %d warn (live API errors)", pass, fail, warn)
if fail > 0 {
os.Exit(1)
}
}
// fetchPackage fetches raw releases for one package.
func fetchPackage(ctx context.Context, client *http.Client, cacheRoot, confDir, pkg string, auth *githubish.Auth) error {
conf, err := installerconf.Read(filepath.Join(confDir, pkg, "releases.conf"))
if err != nil {
return fmt.Errorf("read conf: %w", err)
}
source := conf.Source
log.Printf(" %s: source=%s", pkg, source)
switch source {
case "github":
return fetchGitHub(ctx, client, cacheRoot, pkg, conf, auth)
case "nodedist":
return fetchNodeDist(ctx, client, cacheRoot, pkg, conf)
default:
return fmt.Errorf("unsupported source %q (only github and nodedist for e2e test)", source)
}
}
func fetchGitHub(ctx context.Context, client *http.Client, cacheRoot, pkg string, conf *installerconf.Conf, auth *githubish.Auth) error {
owner := conf.Owner
repo := conf.Repo
tagPrefix := conf.TagPrefix
d, err := rawcache.Open(filepath.Join(cacheRoot, pkg))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range github.Fetch(ctx, client, owner, repo, auth) {
if err != nil {
return fmt.Errorf("github %s/%s: %w", owner, repo, err)
}
for _, rel := range batch {
if rel.Draft {
continue
}
tag := rel.TagName
if tagPrefix != "" {
if !strings.HasPrefix(tag, tagPrefix) {
continue
}
tag = strings.TrimPrefix(tag, tagPrefix)
}
data, err := json.Marshal(rel)
if err != nil {
return fmt.Errorf("marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if latest == "" && !rel.Prerelease {
latest = tag
}
}
}
if latest != "" {
current := d.Latest()
if current == "" || lexver.Compare(lexver.Parse(latest), lexver.Parse(current)) > 0 {
d.SetLatest(latest)
}
}
log.Printf(" +%d ~%d =%d latest=%s", added, changed, skipped, d.Latest())
return nil
}
func fetchNodeDist(ctx context.Context, client *http.Client, cacheRoot, pkg string, conf *installerconf.Conf) error {
baseURL := conf.BaseURL
d, err := rawcache.Open(filepath.Join(cacheRoot, pkg))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range nodedist.Fetch(ctx, client, baseURL) {
if err != nil {
return fmt.Errorf("nodedist: %w", err)
}
for _, entry := range batch {
tag := entry.Version
data, err := json.Marshal(entry)
if err != nil {
return fmt.Errorf("marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if latest == "" {
latest = tag
}
}
}
if latest != "" {
current := d.Latest()
if current == "" || lexver.Compare(lexver.Parse(latest), lexver.Parse(current)) > 0 {
d.SetLatest(latest)
}
}
log.Printf(" +%d ~%d =%d latest=%s", added, changed, skipped, d.Latest())
return nil
}
// classifyFromCache reads the raw cache and produces classified dists.
func classifyFromCache(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]resolve.Dist, error) {
source := conf.Source
switch source {
case "github":
return classifyGitHub(pkg, conf, d)
case "nodedist":
return classifyNodeDist(pkg, conf, d)
default:
return nil, fmt.Errorf("unsupported source %q", source)
}
}
func classifyGitHub(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]resolve.Dist, error) {
tagPrefix := conf.TagPrefix
releases, err := readAllReleases(d)
if err != nil {
return nil, err
}
var dists []resolve.Dist
for _, data := range releases {
var rel struct {
TagName string `json:"tag_name"`
Prerelease bool `json:"prerelease"`
Draft bool `json:"draft"`
PublishedAt string `json:"published_at"`
Assets []struct {
Name string `json:"name"`
BrowserDownloadURL string `json:"browser_download_url"`
Size int64 `json:"size"`
} `json:"assets"`
}
if err := json.Unmarshal(data, &rel); err != nil {
continue
}
if rel.Draft {
continue
}
version := rel.TagName
if tagPrefix != "" {
version = strings.TrimPrefix(version, tagPrefix)
}
// Strip leading "v" for version normalization.
version = strings.TrimPrefix(version, "v")
channel := "stable"
if rel.Prerelease {
channel = "beta"
}
date := ""
if len(rel.PublishedAt) >= 10 {
date = rel.PublishedAt[:10]
}
for _, asset := range rel.Assets {
if isMetaAsset(asset.Name) {
continue
}
r := classifyFilename(asset.Name)
extra := detectExtra(asset.Name)
dists = append(dists, resolve.Dist{
Package: pkg,
Version: version,
Channel: channel,
OS: r.os,
Arch: r.arch,
Libc: r.libc,
Format: r.format,
Download: asset.BrowserDownloadURL,
Filename: asset.Name,
Size: asset.Size,
Date: date,
Extra: extra,
})
}
}
return dists, nil
}
func classifyNodeDist(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]resolve.Dist, error) {
baseURL := conf.BaseURL
releases, err := readAllReleases(d)
if err != nil {
return nil, err
}
var dists []resolve.Dist
for _, data := range releases {
var entry struct {
Version string `json:"version"`
Date string `json:"date"`
Files []string `json:"files"`
LTS json.RawMessage `json:"lts"`
Security bool `json:"security"`
}
if err := json.Unmarshal(data, &entry); err != nil {
continue
}
lts := string(entry.LTS) != "false" && string(entry.LTS) != ""
version := strings.TrimPrefix(entry.Version, "v")
// Webi treats even major versions as "stable" (LTS-eligible).
channel := "stable"
parts := strings.SplitN(version, ".", 2)
if len(parts) > 0 {
var major int
fmt.Sscanf(parts[0], "%d", &major)
if major%2 != 0 {
channel = "beta"
}
}
for _, file := range entry.Files {
if file == "src" || file == "headers" {
continue
}
fileDists := expandNodeFile(pkg, entry.Version, version, channel, entry.Date, lts, baseURL, file)
dists = append(dists, fileDists...)
}
}
return dists, nil
}
func expandNodeFile(pkg, rawVersion, version, channel, date string, lts bool, baseURL, file string) []resolve.Dist {
parts := strings.Split(file, "-")
if len(parts) < 2 {
return nil
}
osMap := map[string]string{
"osx": "darwin", "linux": "linux", "win": "windows",
"sunos": "sunos", "aix": "aix",
}
archMap := map[string]string{
"x64": "x86_64", "x86": "x86", "arm64": "aarch64",
"armv7l": "armv7", "armv6l": "armv6",
"ppc64": "ppc64", "ppc64le": "ppc64le", "s390x": "s390x",
"loong64": "loong64", "riscv64": "riscv64",
}
os_ := osMap[parts[0]]
arch := archMap[parts[1]]
if os_ == "" || arch == "" {
return nil
}
libc := ""
pkgType := ""
if len(parts) > 2 {
pkgType = parts[2]
}
var formats []string
switch pkgType {
case "musl":
libc = "musl"
formats = []string{".tar.gz", ".tar.xz"}
case "tar":
formats = []string{".tar.gz", ".tar.xz"}
case "zip":
formats = []string{".zip"}
case "7z":
formats = []string{".7z"}
case "pkg":
formats = []string{".pkg"}
case "msi":
formats = []string{".msi"}
case "exe":
formats = []string{".exe"}
case "":
formats = []string{".tar.gz", ".tar.xz"}
default:
return nil
}
if libc == "" && os_ == "linux" {
libc = "gnu"
}
osPart := parts[0]
if osPart == "osx" {
osPart = "darwin"
}
archPart := parts[1]
muslExtra := ""
if libc == "musl" {
muslExtra = "-musl"
}
var dists []resolve.Dist
for _, format := range formats {
var filename string
if format == ".msi" {
filename = fmt.Sprintf("node-%s-%s%s%s", rawVersion, archPart, muslExtra, format)
} else {
filename = fmt.Sprintf("node-%s-%s-%s%s%s", rawVersion, osPart, archPart, muslExtra, format)
}
dists = append(dists, resolve.Dist{
Package: pkg,
Version: version,
Channel: channel,
OS: os_,
Arch: arch,
Libc: libc,
Format: format,
Download: fmt.Sprintf("%s/%s/%s", baseURL, rawVersion, filename),
Filename: filename,
LTS: lts,
Date: date,
})
}
return dists
}
// queryLiveAPI queries the live webi.sh API and parses the response header.
func queryLiveAPI(client *http.Client, tc testCase) (*liveResult, error) {
// Build format string matching what the webi.sh bootstrap sends.
// Order: tar,exe,zip,xz,dmg,git (least to most favorable in bootstrap,
// but the API doesn't care about order).
fmtParam := "tar,exe,zip,xz,dmg"
url := fmt.Sprintf("https://webi.sh/api/installers/%s@stable.sh?formats=%s", tc.Package, fmtParam)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", tc.UA)
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return parseLiveResponse(string(body)), nil
}
// parseLiveResponse extracts WEBI_* and PKG_* variables from the shell script.
func parseLiveResponse(body string) *liveResult {
vars := make(map[string]string)
scanner := bufio.NewScanner(strings.NewReader(body))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
for _, prefix := range []string{"WEBI_", "PKG_"} {
if strings.HasPrefix(line, prefix) {
if eq := strings.IndexByte(line, '='); eq > 0 {
key := line[:eq]
val := line[eq+1:]
val = strings.Trim(val, "'\"")
vars[key] = val
}
}
}
}
return &liveResult{
Version: vars["WEBI_VERSION"],
OS: vars["WEBI_OS"],
Arch: vars["WEBI_ARCH"],
Libc: vars["WEBI_LIBC"],
Ext: vars["WEBI_EXT"],
PkgURL: vars["WEBI_PKG_URL"],
PkgFile: vars["WEBI_PKG_FILE"],
Channel: vars["WEBI_CHANNEL"],
Stable: vars["PKG_STABLE"],
Latest: vars["PKG_LATEST"],
Oses: vars["PKG_OSES"],
Arches: vars["PKG_ARCHES"],
Libcs: vars["PKG_LIBCS"],
Formats: vars["PKG_FORMATS"],
}
}
// readAllReleases reads all cached release files.
func readAllReleases(d *rawcache.Dir) (map[string][]byte, error) {
active, err := d.ActivePath()
if err != nil {
return nil, err
}
entries, err := os.ReadDir(active)
if err != nil {
return nil, err
}
result := make(map[string][]byte, len(entries))
for _, e := range entries {
if e.IsDir() || strings.HasPrefix(e.Name(), "_") {
continue
}
data, err := os.ReadFile(filepath.Join(active, e.Name()))
if err != nil {
return nil, err
}
result[e.Name()] = data
}
return result, nil
}
type classResult struct {
os, arch, libc, format string
}
func classifyFilename(name string) classResult {
// Use the classify package.
// Import it indirectly to avoid circular deps — inline the logic
// we need for the e2e test.
lower := strings.ToLower(name)
var r classResult
r.format = detectFormat(name)
// OS detection
switch {
case strings.Contains(lower, "linux"):
r.os = "linux"
case strings.Contains(lower, "darwin") || strings.Contains(lower, "macos") || strings.Contains(lower, "apple"):
r.os = "darwin"
case strings.Contains(lower, "windows") || strings.Contains(lower, "win64") || strings.Contains(lower, "win32"):
r.os = "windows"
case strings.HasSuffix(lower, ".dmg") || strings.HasSuffix(lower, ".app.zip"):
r.os = "darwin"
case strings.HasSuffix(lower, ".exe") || strings.HasSuffix(lower, ".msi"):
r.os = "windows"
case strings.Contains(lower, "freebsd"):
r.os = "freebsd"
}
// Arch detection
switch {
case strings.Contains(lower, "x86_64") || strings.Contains(lower, "amd64") || strings.Contains(lower, "x64"):
r.arch = "x86_64"
case strings.Contains(lower, "aarch64") || strings.Contains(lower, "arm64"):
r.arch = "aarch64"
case strings.Contains(lower, "armv7") || strings.Contains(lower, "armhf"):
r.arch = "armv7"
case strings.Contains(lower, "armv6"):
r.arch = "armv6"
case strings.Contains(lower, "i686") || strings.Contains(lower, "i386") || strings.Contains(lower, "x86") || strings.Contains(lower, "386"):
r.arch = "x86"
case strings.Contains(lower, "ppc64le") || strings.Contains(lower, "powerpc64le"):
r.arch = "ppc64le"
case strings.Contains(lower, "ppc64") || strings.Contains(lower, "powerpc64"):
r.arch = "ppc64"
case strings.Contains(lower, "riscv64"):
r.arch = "riscv64"
case strings.Contains(lower, "s390x"):
r.arch = "s390x"
case strings.Contains(lower, "loong64"):
r.arch = "loong64"
}
// Libc detection
switch {
case strings.Contains(lower, "musl"):
r.libc = "musl"
case strings.Contains(lower, "gnu"):
r.libc = "gnu"
case strings.Contains(lower, "msvc"):
r.libc = "msvc"
}
return r
}
func detectFormat(name string) string {
lower := strings.ToLower(name)
for _, ext := range []string{".tar.gz", ".tar.xz", ".tar.bz2", ".tar.zst", ".exe.xz", ".app.zip"} {
if strings.HasSuffix(lower, ext) {
return ext
}
}
// .tgz is a common alias for .tar.gz
if strings.HasSuffix(lower, ".tgz") {
return ".tar.gz"
}
return filepath.Ext(lower)
}
// detectExtra identifies GPU/vendor-specific variant suffixes in filenames
// like "ollama-linux-amd64-rocm.tar.zst" or "ollama-linux-arm64-jetpack5.tar.zst".
func detectExtra(name string) string {
lower := strings.ToLower(name)
for _, variant := range []string{
"-rocm", "-jetpack", "-cuda", "-vulkan", "-metal",
"-extended", "-static", "-debug", "-nightly",
} {
if strings.Contains(lower, variant) {
return strings.TrimPrefix(variant, "-")
}
}
return ""
}
func isMetaAsset(name string) bool {
lower := strings.ToLower(name)
for _, suffix := range []string{
".sha256", ".sha256sum", ".sha512", ".sha512sum",
".md5", ".md5sum", ".sig", ".asc", ".pem",
"checksums.txt", "sha256sums", "sha512sums",
".sbom", ".spdx", ".json.sig", ".sigstore",
".d.ts", ".pub",
} {
if strings.HasSuffix(lower, suffix) {
return true
}
}
for _, contains := range []string{
"checksums", "sha256sum", "sha512sum",
"buildable-artifact",
} {
if strings.Contains(lower, contains) {
return true
}
}
for _, exact := range []string{
"install.sh", "install.ps1", "compat.json",
} {
if lower == exact {
return true
}
}
return false
}
cmd/fetchraw/main.go
+862
View File
@@ -0,0 +1,862 @@
// Command fetchraw fetches release histories from upstream APIs and
// merges them into rawcache. Safe to run repeatedly — unchanged releases
// are skipped, new/changed ones are recorded in the audit log.
//
// Reads releases.conf files from package directories to discover what
// to fetch. Adding a new package is just creating a conf file.
//
// Usage:
//
// go run ./cmd/fetchraw -cache ./_cache/raw
// go run ./cmd/fetchraw -cache ./_cache/raw hugo caddy
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"log"
"net/http"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/webinstall/webi-installers/internal/installerconf"
"github.com/webinstall/webi-installers/internal/lexver"
"github.com/webinstall/webi-installers/internal/rawcache"
"github.com/webinstall/webi-installers/internal/releases/chromedist"
"github.com/webinstall/webi-installers/internal/releases/flutterdist"
"github.com/webinstall/webi-installers/internal/releases/gitea"
"github.com/webinstall/webi-installers/internal/releases/github"
"github.com/webinstall/webi-installers/internal/releases/githubish"
"github.com/webinstall/webi-installers/internal/releases/gittag"
"github.com/webinstall/webi-installers/internal/releases/golang"
"github.com/webinstall/webi-installers/internal/releases/gpgdist"
"github.com/webinstall/webi-installers/internal/releases/hashicorp"
"github.com/webinstall/webi-installers/internal/releases/iterm2dist"
"github.com/webinstall/webi-installers/internal/releases/juliadist"
"github.com/webinstall/webi-installers/internal/releases/mariadbdist"
"github.com/webinstall/webi-installers/internal/releases/nodedist"
"github.com/webinstall/webi-installers/internal/releases/zigdist"
)
func main() {
cacheDir := flag.String("cache", "_cache/raw", "root directory for raw cache")
confDir := flag.String("conf", ".", "root directory containing {pkg}/releases.conf files")
token := flag.String("token", os.Getenv("GITHUB_TOKEN"), "GitHub API token")
flag.Parse()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
client := &http.Client{Timeout: 30 * time.Second}
var auth *githubish.Auth
if *token != "" {
auth = &githubish.Auth{Token: *token}
}
// Discover packages from releases.conf files.
packages, err := discover(*confDir)
if err != nil {
log.Fatalf("discover: %v", err)
}
// Filter to requested packages if args given.
args := flag.Args()
if len(args) > 0 {
nameSet := make(map[string]bool, len(args))
for _, a := range args {
nameSet[a] = true
}
var filtered []pkgConf
for _, p := range packages {
if nameSet[p.name] {
filtered = append(filtered, p)
}
}
packages = filtered
}
log.Printf("found %d packages", len(packages))
for _, pkg := range packages {
// Aliases share cache with their target — skip fetching.
if alias := pkg.conf.Extra["alias_of"]; alias != "" {
log.Printf(" %s: alias of %s, skipping", pkg.name, alias)
continue
}
log.Printf("fetching %s...", pkg.name)
var err error
switch pkg.conf.Source {
case "github":
err = fetchGitHub(ctx, client, *cacheDir, pkg.name, pkg.conf, auth)
case "nodedist":
err = fetchNodeDist(ctx, client, *cacheDir, pkg.name, pkg.conf)
case "golang":
err = fetchGolang(ctx, client, *cacheDir, pkg.name)
case "zigdist":
err = fetchZig(ctx, client, *cacheDir, pkg.name)
case "flutterdist":
err = fetchFlutter(ctx, client, *cacheDir, pkg.name)
case "iterm2dist":
err = fetchITerm2(ctx, client, *cacheDir, pkg.name)
case "hashicorp":
err = fetchHashiCorp(ctx, client, *cacheDir, pkg.name, pkg.conf)
case "juliadist":
err = fetchJulia(ctx, client, *cacheDir, pkg.name)
case "gittag":
err = fetchGitTag(ctx, *cacheDir, pkg.name, pkg.conf)
case "gitea":
err = fetchGitea(ctx, client, *cacheDir, pkg.name, pkg.conf)
case "chromedist":
err = fetchChrome(ctx, client, *cacheDir, pkg.name)
case "gpgdist":
err = fetchGPG(ctx, client, *cacheDir, pkg.name)
case "mariadbdist":
err = fetchMariaDB(ctx, client, *cacheDir, pkg.name)
default:
log.Printf(" %s: unknown source %q, skipping", pkg.name, pkg.conf.Source)
continue
}
if err != nil {
log.Printf(" ERROR: %s: %v", pkg.name, err)
}
}
}
type pkgConf struct {
name string
conf *installerconf.Conf
}
// discover finds all {dir}/*/releases.conf files and returns them sorted.
func discover(dir string) ([]pkgConf, error) {
pattern := filepath.Join(dir, "*", "releases.conf")
matches, err := filepath.Glob(pattern)
if err != nil {
return nil, err
}
var packages []pkgConf
for _, path := range matches {
name := filepath.Base(filepath.Dir(path))
// Skip infrastructure dirs (_example, _webi, _common, etc.)
if strings.HasPrefix(name, "_") {
continue
}
conf, err := installerconf.Read(path)
if err != nil {
log.Printf("warning: %s: %v", path, err)
continue
}
packages = append(packages, pkgConf{name: name, conf: conf})
}
sort.Slice(packages, func(i, j int) bool {
return packages[i].name < packages[j].name
})
return packages, nil
}
func fetchNodeDist(ctx context.Context, client *http.Client, cacheRoot, pkgName string, conf *installerconf.Conf) error {
baseURL := conf.BaseURL
if baseURL == "" {
return fmt.Errorf("missing url in releases.conf")
}
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range nodedist.Fetch(ctx, client, baseURL) {
if err != nil {
return fmt.Errorf("%s fetch: %w", pkgName, err)
}
for _, entry := range batch {
tag := entry.Version
data, err := json.Marshal(entry)
if err != nil {
return fmt.Errorf("%s marshal %s: %w", pkgName, tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if latest == "" {
latest = tag
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchGitHub(ctx context.Context, client *http.Client, cacheRoot, pkgName string, conf *installerconf.Conf, auth *githubish.Auth) error {
owner := conf.Owner
repo := conf.Repo
tagPrefix := conf.TagPrefix
if owner == "" || repo == "" {
return fmt.Errorf("missing owner or repo in releases.conf")
}
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range github.Fetch(ctx, client, owner, repo, auth) {
if err != nil {
return fmt.Errorf("github %s/%s: %w", owner, repo, err)
}
for _, rel := range batch {
if rel.Draft {
continue
}
tag := rel.TagName
if tagPrefix != "" {
if !strings.HasPrefix(tag, tagPrefix) {
continue
}
tag = strings.TrimPrefix(tag, tagPrefix)
}
data, err := json.Marshal(rel)
if err != nil {
return fmt.Errorf("marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if latest == "" && !rel.Prerelease {
latest = tag
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func updateLatest(d *rawcache.Dir, candidate string) error {
if candidate == "" {
return nil
}
current := d.Latest()
if current == "" || lexver.Compare(lexver.Parse(candidate), lexver.Parse(current)) > 0 {
return d.SetLatest(candidate)
}
return nil
}
func fetchGolang(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range golang.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("golang: %w", err)
}
for _, rel := range batch {
tag := rel.Version // "go1.24.1"
data, err := json.Marshal(rel)
if err != nil {
return fmt.Errorf("golang marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if latest == "" && rel.Stable {
latest = tag
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchZig(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range zigdist.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("zigdist: %w", err)
}
for _, rel := range batch {
tag := rel.Version
data, err := json.Marshal(rel)
if err != nil {
return fmt.Errorf("zigdist marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
// Stable versions have dots and no dev/pre markers.
isStable := strings.Contains(tag, ".") && !strings.ContainsAny(tag, "+-")
if isStable {
if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
latest = tag
}
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchFlutter(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range flutterdist.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("flutterdist: %w", err)
}
for _, rel := range batch {
// Use version+channel+os+arch as the tag. The arch is embedded
// in the archive path (e.g. flutter_macos_arm64_3.0.0-stable.zip
// vs flutter_macos_3.0.0-stable.zip for universal/x64).
arch := ""
base := filepath.Base(rel.Archive)
prefix := "flutter_" + rel.OS + "_"
if after, ok := strings.CutPrefix(base, prefix); ok {
if !strings.HasPrefix(after, rel.Version) {
// There's an arch segment between OS and version.
if idx := strings.Index(after, "_"); idx > 0 {
arch = after[:idx]
}
}
}
tag := fmt.Sprintf("%s-%s-%s", rel.Version, rel.Channel, rel.OS)
if arch != "" {
tag += "-" + arch
}
data, err := json.Marshal(rel)
if err != nil {
return fmt.Errorf("flutterdist marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if latest == "" && rel.Channel == "stable" {
latest = tag
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchITerm2(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range iterm2dist.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("iterm2dist: %w", err)
}
for _, entry := range batch {
tag := entry.Version
if tag == "" {
continue
}
data, err := json.Marshal(entry)
if err != nil {
return fmt.Errorf("iterm2dist marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if latest == "" && entry.Channel == "stable" {
latest = tag
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchHashiCorp(ctx context.Context, client *http.Client, cacheRoot, pkgName string, conf *installerconf.Conf) error {
product := conf.Extra["product"]
if product == "" {
return fmt.Errorf("missing product in releases.conf")
}
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for idx, err := range hashicorp.Fetch(ctx, client, product) {
if err != nil {
return fmt.Errorf("hashicorp %s: %w", product, err)
}
for tag, ver := range idx.Versions {
data, err := json.Marshal(ver)
if err != nil {
return fmt.Errorf("hashicorp marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
// Stable = no prerelease markers. Compare all to find highest.
isStable := !strings.ContainsAny(tag, "-+")
if isStable {
if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
latest = tag
}
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchJulia(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range juliadist.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("juliadist: %w", err)
}
for _, rel := range batch {
tag := rel.Version
data, err := json.Marshal(rel)
if err != nil {
return fmt.Errorf("juliadist marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if rel.Stable {
if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
latest = tag
}
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchGitTag(ctx context.Context, cacheRoot, pkgName string, conf *installerconf.Conf) error {
gitURL := conf.BaseURL
if gitURL == "" {
return fmt.Errorf("missing url in releases.conf")
}
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
repoDir := filepath.Join(cacheRoot, "_repos")
if err := os.MkdirAll(repoDir, 0o755); err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range gittag.Fetch(ctx, gitURL, repoDir) {
if err != nil {
return fmt.Errorf("gittag %s: %w", pkgName, err)
}
for _, entry := range batch {
tag := entry.Version
if tag == "" {
tag = "HEAD-" + entry.CommitHash
}
data, err := json.Marshal(entry)
if err != nil {
return fmt.Errorf("gittag marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if entry.GitTag != "" && entry.GitTag != "HEAD" {
if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
latest = tag
}
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchGitea(ctx context.Context, client *http.Client, cacheRoot, pkgName string, conf *installerconf.Conf) error {
baseURL := conf.BaseURL
owner := conf.Owner
repo := conf.Repo
if baseURL == "" || owner == "" || repo == "" {
return fmt.Errorf("missing base_url, owner, or repo in releases.conf")
}
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range gitea.Fetch(ctx, client, baseURL, owner, repo, nil) {
if err != nil {
return fmt.Errorf("gitea %s/%s: %w", owner, repo, err)
}
for _, rel := range batch {
if rel.Draft {
continue
}
tag := rel.TagName
data, err := json.Marshal(rel)
if err != nil {
return fmt.Errorf("gitea marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if latest == "" && !rel.Prerelease {
latest = tag
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchChrome(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range chromedist.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("chromedist: %w", err)
}
for _, ver := range batch {
tag := ver.Version
data, err := json.Marshal(ver)
if err != nil {
return fmt.Errorf("chromedist marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
latest = tag
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchGPG(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range gpgdist.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("gpgdist: %w", err)
}
for _, entry := range batch {
tag := entry.Version
data, err := json.Marshal(entry)
if err != nil {
return fmt.Errorf("gpgdist marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
latest = tag
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
func fetchMariaDB(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error {
d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName))
if err != nil {
return err
}
var added, changed, skipped int
var latest string
for batch, err := range mariadbdist.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("mariadbdist: %w", err)
}
for _, rel := range batch {
tag := rel.ReleaseID
data, err := json.Marshal(rel)
if err != nil {
return fmt.Errorf("mariadbdist marshal %s: %w", tag, err)
}
action, err := d.Merge(tag, data)
if err != nil {
return err
}
switch action {
case "added":
added++
case "changed":
changed++
default:
skipped++
}
isStable := rel.MajorStatus == "Stable"
if isStable {
if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 {
latest = tag
}
}
}
}
if err := updateLatest(d, latest); err != nil {
return err
}
log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest())
return nil
}
cmd/inspect/main.go
+625
View File
@@ -0,0 +1,625 @@
// Command inspect downloads release archives, unpacks them, and reports
// their internal structure. This helps discover how packages are laid out
// and whether the layout changes across versions.
//
// Usage:
//
// go run ./cmd/inspect -csv distributables.csv -cache ./_cache/downloads ollama sd
package main
import (
"context"
"encoding/csv"
"flag"
"fmt"
"io"
"log"
"mime"
"net/http"
"os"
"os/exec"
"path/filepath"
"sort"
"strings"
"time"
"github.com/webinstall/webi-installers/internal/httpclient"
)
// Row is one CSV row from distributables.csv.
type Row struct {
Package string
Version string
Channel string
Date string
OS string
Arch string
Libc string
Format string
Download string
Filename string
Extra string
}
// archiveFormats are the formats we download and unpack.
var archiveFormats = map[string]bool{
".tar.gz": true,
".tar.xz": true,
".tar.bz2": true,
".tar.zst": true,
".zip": true,
".dmg": true,
".gz": true,
".xz": true,
}
// inspectOSes are the OSes we inspect.
var inspectOSes = map[string]bool{
"linux": true,
"darwin": true,
"windows": true,
"": true, // source-only packages
}
// preferredArch picks one arch per OS to download.
func preferredArch(os_ string) string {
switch os_ {
case "darwin":
return "aarch64"
default:
return "x86_64"
}
}
func main() {
csvFile := flag.String("csv", "distributables.csv", "path to distributables CSV")
cacheDir := flag.String("cache", "_cache/downloads", "download cache directory")
flag.Parse()
packages := flag.Args()
if len(packages) == 0 {
log.Fatal("usage: inspect [-csv FILE] [-cache DIR] PACKAGE [PACKAGE...]")
}
rows, err := readCSV(*csvFile)
if err != nil {
log.Fatalf("read csv: %v", err)
}
client := httpclient.New()
// Override timeout for large downloads.
client.Timeout = 10 * time.Minute
for _, pkg := range packages {
log.Printf("=== %s ===", pkg)
if err := inspectPackage(client, rows, pkg, *cacheDir); err != nil {
log.Printf("ERROR: %s: %v", pkg, err)
}
}
}
func readCSV(path string) ([]Row, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
r := csv.NewReader(f)
header, err := r.Read()
if err != nil {
return nil, err
}
// Build column index.
idx := make(map[string]int, len(header))
for i, col := range header {
idx[col] = i
}
var rows []Row
for {
record, err := r.Read()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
get := func(col string) string {
if i, ok := idx[col]; ok && i < len(record) {
return record[i]
}
return ""
}
rows = append(rows, Row{
Package: get("package"),
Version: get("version"),
Channel: get("channel"),
Date: get("date"),
OS: get("os"),
Arch: get("arch"),
Libc: get("libc"),
Format: get("format"),
Download: get("download"),
Filename: get("filename"),
Extra: get("extra"),
})
}
return rows, nil
}
func inspectPackage(client *http.Client, allRows []Row, pkg, cacheDir string) error {
// Filter rows for this package.
var pkgRows []Row
for _, r := range allRows {
if r.Package == pkg {
pkgRows = append(pkgRows, r)
}
}
if len(pkgRows) == 0 {
return fmt.Errorf("no rows found")
}
// Find latest stable version, fall back to any version.
versions := findVersionsByDate(pkgRows)
if len(versions) == 0 {
return fmt.Errorf("no versions found")
}
latestVer := versions[0]
log.Printf(" latest version: %s", latestVer)
// Check if latest has assets uploaded (more than just source tarballs).
latestRows := filterVersion(pkgRows, latestVer)
hasRealAssets := false
for _, r := range latestRows {
if r.Extra != "source" && archiveFormats[r.Format] {
hasRealAssets = true
break
}
}
// If latest looks empty, step back one version.
if !hasRealAssets && len(versions) > 1 {
latestVer = versions[1]
latestRows = filterVersion(pkgRows, latestVer)
log.Printf(" latest has no assets, using: %s", latestVer)
}
// Inspect the latest version.
if err := inspectVersion(client, pkg, latestVer, latestRows, cacheDir); err != nil {
return err
}
// Find versions roughly a year apart going back.
yearVersions := findYearlyVersions(pkgRows, latestVer)
for _, v := range yearVersions {
log.Printf(" --- checking %s ---", v)
vRows := filterVersion(pkgRows, v)
if err := inspectVersion(client, pkg, v, vRows, cacheDir); err != nil {
log.Printf(" ERROR: %v", err)
}
}
return nil
}
// findVersionsByDate returns versions sorted newest first, preferring stable.
func findVersionsByDate(rows []Row) []string {
type vInfo struct {
version string
date string
stable bool
}
seen := map[string]*vInfo{}
for _, r := range rows {
if _, ok := seen[r.Version]; !ok {
seen[r.Version] = &vInfo{
version: r.Version,
date: r.Date,
stable: r.Channel == "stable",
}
}
}
var vs []*vInfo
for _, v := range seen {
vs = append(vs, v)
}
// Sort: stable first, then by date descending, then version descending.
sort.Slice(vs, func(i, j int) bool {
if vs[i].stable != vs[j].stable {
return vs[i].stable
}
if vs[i].date != vs[j].date {
return vs[i].date > vs[j].date
}
return vs[i].version > vs[j].version
})
result := make([]string, len(vs))
for i, v := range vs {
result[i] = v.version
}
return result
}
// findYearlyVersions picks versions roughly a year apart before the given version.
func findYearlyVersions(rows []Row, latestVer string) []string {
// Find the date of latest.
var latestDate string
for _, r := range rows {
if r.Version == latestVer && r.Date != "" {
latestDate = r.Date
break
}
}
if latestDate == "" {
return nil
}
latestTime, err := time.Parse("2006-01-02", latestDate)
if err != nil {
return nil
}
// Collect all stable versions with dates.
type vd struct {
version string
date time.Time
}
seen := map[string]bool{}
var all []vd
for _, r := range rows {
if seen[r.Version] || r.Date == "" || r.Channel != "stable" {
continue
}
seen[r.Version] = true
t, err := time.Parse("2006-01-02", r.Date)
if err != nil {
continue
}
if t.Before(latestTime) {
all = append(all, vd{r.Version, t})
}
}
sort.Slice(all, func(i, j int) bool {
return all[i].date.After(all[j].date)
})
// Pick versions roughly a year apart.
var result []string
nextTarget := latestTime.AddDate(-1, 0, 0)
for _, v := range all {
if v.date.Before(nextTarget) || v.date.Equal(nextTarget) {
result = append(result, v.version)
nextTarget = v.date.AddDate(-1, 0, 0)
}
}
return result
}
func filterVersion(rows []Row, version string) []Row {
var result []Row
for _, r := range rows {
if r.Version == version {
result = append(result, r)
}
}
return result
}
// inspectVersion downloads and inspects archives for one version.
func inspectVersion(client *http.Client, pkg, version string, rows []Row, cacheDir string) error {
// Group by OS, pick one arch per OS, pick distinct formats.
type dlKey struct {
os_ string
format string
}
selected := map[dlKey]*Row{}
for i := range rows {
r := &rows[i]
if !inspectOSes[r.OS] {
continue
}
if !archiveFormats[r.Format] {
continue
}
key := dlKey{r.OS, r.Format}
existing := selected[key]
if existing == nil {
selected[key] = r
continue
}
// Prefer the preferred arch.
pref := preferredArch(r.OS)
if r.Arch == pref && existing.Arch != pref {
selected[key] = r
}
// Skip rocm/jetpack variants.
if strings.Contains(r.Filename, "rocm") || strings.Contains(r.Filename, "jetpack") {
if !strings.Contains(existing.Filename, "rocm") && !strings.Contains(existing.Filename, "jetpack") {
continue // keep existing non-special variant
}
}
}
if len(selected) == 0 {
log.Printf(" %s: no downloadable archives", version)
return nil
}
// Sort keys for deterministic output.
var keys []dlKey
for k := range selected {
keys = append(keys, k)
}
sort.Slice(keys, func(i, j int) bool {
if keys[i].os_ != keys[j].os_ {
return keys[i].os_ < keys[j].os_
}
return keys[i].format < keys[j].format
})
for _, key := range keys {
r := selected[key]
os_ := r.OS
if os_ == "" {
os_ = "any"
}
log.Printf(" [%s] %s %s → %s", version, os_, r.Format, r.Filename)
dlPath, err := download(client, r.Download, r.Filename, filepath.Join(cacheDir, pkg, version))
if err != nil {
log.Printf(" download error: %v", err)
continue
}
contents, err := unpackAndList(dlPath, r.Format)
if err != nil {
log.Printf(" unpack error: %v", err)
continue
}
printContents(contents)
}
return nil
}
// download fetches a URL to the cache dir. Returns the path to the cached file.
// Skips download if the file already exists.
func download(client *http.Client, url, hintFilename, dir string) (string, error) {
// Check if already cached by hint filename.
cached := filepath.Join(dir, hintFilename)
if _, err := os.Stat(cached); err == nil {
return cached, nil
}
if err := os.MkdirAll(dir, 0o755); err != nil {
return "", err
}
ctx := context.Background()
resp, err := httpclient.Get(ctx, client, url)
if err != nil {
return "", fmt.Errorf("GET %s: %w", url, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("GET %s: %s", url, resp.Status)
}
// Determine filename from Content-Disposition or hint.
filename := hintFilename
if cd := resp.Header.Get("Content-Disposition"); cd != "" {
_, params, err := mime.ParseMediaType(cd)
if err == nil {
if fn, ok := params["filename"]; ok && fn != "" {
filename = fn
}
}
}
outPath := filepath.Join(dir, filename)
// Atomic write: temp file + rename.
tmp := outPath + ".tmp"
f, err := os.Create(tmp)
if err != nil {
return "", err
}
n, err := io.Copy(f, resp.Body)
if closeErr := f.Close(); closeErr != nil && err == nil {
err = closeErr
}
if err != nil {
os.Remove(tmp)
return "", fmt.Errorf("download %s: %w", url, err)
}
if err := os.Rename(tmp, outPath); err != nil {
os.Remove(tmp)
return "", err
}
log.Printf(" downloaded %s (%d bytes)", filename, n)
return outPath, nil
}
// FileEntry describes one file inside an archive.
type FileEntry struct {
Path string
Size int64
Mode os.FileMode
IsDir bool
IsExec bool
IsSymlink bool
LinkTarget string
}
// unpackAndList extracts an archive to a temp dir and lists contents.
func unpackAndList(archivePath, format string) ([]FileEntry, error) {
tmpDir, err := os.MkdirTemp("", "webi-inspect-*")
if err != nil {
return nil, err
}
defer os.RemoveAll(tmpDir)
switch format {
case ".tar.gz":
err = run("tar", "xzf", archivePath, "-C", tmpDir)
case ".tar.xz":
err = run("tar", "xJf", archivePath, "-C", tmpDir)
case ".tar.bz2":
err = run("tar", "xjf", archivePath, "-C", tmpDir)
case ".tar.zst":
err = run("tar", "--zstd", "-xf", archivePath, "-C", tmpDir)
case ".zip":
err = run("unzip", "-q", "-o", archivePath, "-d", tmpDir)
case ".dmg":
err = extractDMG(archivePath, tmpDir)
case ".gz":
// Single file gzip.
base := filepath.Base(archivePath)
base = strings.TrimSuffix(base, ".gz")
outPath := filepath.Join(tmpDir, base)
err = run("sh", "-c", fmt.Sprintf("gunzip -c %q > %q", archivePath, outPath))
case ".xz":
base := filepath.Base(archivePath)
base = strings.TrimSuffix(base, ".xz")
outPath := filepath.Join(tmpDir, base)
err = run("sh", "-c", fmt.Sprintf("xz -dc %q > %q", archivePath, outPath))
default:
return nil, fmt.Errorf("unsupported format: %s", format)
}
if err != nil {
return nil, fmt.Errorf("extract %s: %w", format, err)
}
return listDir(tmpDir, "")
}
func extractDMG(dmgPath, outDir string) error {
// Try 7z first (doesn't require mounting).
if _, err := exec.LookPath("7z"); err == nil {
return run("7z", "x", "-o"+outDir, dmgPath)
}
// Fall back to hdiutil mount + copy + unmount.
mountPoint, err := os.MkdirTemp("", "webi-dmg-*")
if err != nil {
return err
}
defer os.RemoveAll(mountPoint)
if err := run("hdiutil", "attach", dmgPath, "-mountpoint", mountPoint, "-nobrowse", "-quiet"); err != nil {
return fmt.Errorf("mount dmg: %w", err)
}
defer run("hdiutil", "detach", mountPoint, "-quiet")
// Copy contents.
return run("cp", "-R", mountPoint+"/.", outDir)
}
func run(name string, args ...string) error {
cmd := exec.Command(name, args...)
cmd.Stderr = os.Stderr
return cmd.Run()
}
func listDir(root, prefix string) ([]FileEntry, error) {
entries, err := os.ReadDir(filepath.Join(root, prefix))
if err != nil {
return nil, err
}
var result []FileEntry
for _, e := range entries {
relPath := filepath.Join(prefix, e.Name())
fullPath := filepath.Join(root, relPath)
info, err := e.Info()
if err != nil {
continue
}
entry := FileEntry{
Path: relPath,
Size: info.Size(),
Mode: info.Mode(),
IsDir: e.IsDir(),
}
if info.Mode()&os.ModeSymlink != 0 {
entry.IsSymlink = true
target, _ := os.Readlink(fullPath)
entry.LinkTarget = target
}
if !e.IsDir() && info.Mode()&0o111 != 0 {
entry.IsExec = true
}
result = append(result, entry)
if e.IsDir() {
sub, err := listDir(root, relPath)
if err != nil {
continue
}
result = append(result, sub...)
}
}
return result, nil
}
func printContents(entries []FileEntry) {
for _, e := range entries {
marker := " "
if e.IsExec {
marker = "* "
}
if e.IsDir {
marker = "d "
}
if e.IsSymlink {
marker = "→ "
}
size := ""
if !e.IsDir {
size = formatSize(e.Size)
}
line := fmt.Sprintf(" %s%-50s %8s", marker, e.Path, size)
if e.IsSymlink {
line += " → " + e.LinkTarget
}
log.Print(line)
}
}
func formatSize(n int64) string {
switch {
case n >= 1<<30:
return fmt.Sprintf("%.1fG", float64(n)/float64(1<<30))
case n >= 1<<20:
return fmt.Sprintf("%.1fM", float64(n)/float64(1<<20))
case n >= 1<<10:
return fmt.Sprintf("%.1fK", float64(n)/float64(1<<10))
default:
return fmt.Sprintf("%dB", n)
}
}
cmd/uaparse/main.go
+356
View File
@@ -0,0 +1,356 @@
// Command uaparse analyzes User-Agent strings from webi.sh logs.
//
// It reads UA strings (one per line) from stdin or a file, parses each
// through uadetect, and produces summary output showing:
// - unique platform tuples (os, arch, libc) with counts
// - platform hints extracted from kernel version strings (cloud provider,
// container runtime, device info)
// - detection failures and malformed UAs
//
// Usage:
//
// uaparse < LIVE-UAS.txt
// uaparse LIVE-UAS.txt
// uaparse -json LIVE-UAS.txt
// uaparse -fixtures LIVE-UAS.txt # output Go test fixtures
package main
import (
"bufio"
"encoding/json"
"flag"
"fmt"
"os"
"regexp"
"sort"
"strings"
"github.com/webinstall/webi-installers/internal/uadetect"
)
// PlatformKey is the resolution-relevant tuple — everything else is noise
// for artifact selection.
type PlatformKey struct {
OS string `json:"os"`
Arch string `json:"arch"`
Libc string `json:"libc"`
}
func (k PlatformKey) String() string {
return fmt.Sprintf("%-10s %-10s %s", k.OS, k.Arch, k.Libc)
}
// PlatformEntry holds a unique platform and its metadata.
type PlatformEntry struct {
Key PlatformKey `json:"key"`
Count int `json:"count"`
Examples []string `json:"examples"` // up to 3 representative UAs
Hints []string `json:"hints"` // unique platform hints seen
}
// UAIssue records a malformed or undetectable UA.
type UAIssue struct {
Line int `json:"line"`
UA string `json:"ua"`
Reason string `json:"reason"`
}
// Hint is a platform detail extracted from the kernel version string.
type Hint struct {
Tag string // short label: "amzn", "azure", "gcp", "wsl", etc.
Count int
}
func main() {
jsonOut := flag.Bool("json", false, "output as JSON")
fixtures := flag.Bool("fixtures", false, "output Go test fixture table")
flag.Parse()
var scanner *bufio.Scanner
if flag.NArg() > 0 {
f, err := os.Open(flag.Arg(0))
if err != nil {
fmt.Fprintf(os.Stderr, "uaparse: %v\n", err)
os.Exit(1)
}
defer f.Close()
scanner = bufio.NewScanner(f)
} else {
scanner = bufio.NewScanner(os.Stdin)
}
platforms := make(map[PlatformKey]*PlatformEntry)
hints := make(map[string]int)
var issues []UAIssue
lineNum := 0
for scanner.Scan() {
lineNum++
ua := strings.TrimSpace(scanner.Text())
if ua == "" {
continue
}
// Detect corruption: truncated/double-pasted lines.
if isMalformed(ua) {
issues = append(issues, UAIssue{
Line: lineNum,
UA: ua,
Reason: "malformed (truncated or corrupted)",
})
continue
}
// Parse through uadetect.
result := uadetect.Parse(ua)
// Check for detection failures.
if result.OS == "" {
issues = append(issues, UAIssue{
Line: lineNum,
UA: ua,
Reason: "OS not detected",
})
}
if result.Arch == "" {
issues = append(issues, UAIssue{
Line: lineNum,
UA: ua,
Reason: "arch not detected",
})
}
key := PlatformKey{
OS: string(result.OS),
Arch: string(result.Arch),
Libc: string(result.Libc),
}
entry, ok := platforms[key]
if !ok {
entry = &PlatformEntry{Key: key}
platforms[key] = entry
}
entry.Count++
if len(entry.Examples) < 3 {
entry.Examples = append(entry.Examples, ua)
}
// Extract platform hints from kernel version.
for _, h := range extractHints(ua) {
if !containsStr(entry.Hints, h) {
entry.Hints = append(entry.Hints, h)
}
hints[h]++
}
}
if err := scanner.Err(); err != nil {
fmt.Fprintf(os.Stderr, "uaparse: read error: %v\n", err)
os.Exit(1)
}
// Sort platforms by count descending.
entries := make([]*PlatformEntry, 0, len(platforms))
for _, e := range platforms {
entries = append(entries, e)
}
sort.Slice(entries, func(i, j int) bool {
return entries[i].Count > entries[j].Count
})
if *jsonOut {
outputJSON(entries, issues, hints)
} else if *fixtures {
outputFixtures(entries)
} else {
outputTable(entries, issues, hints, lineNum)
}
}
func outputTable(entries []*PlatformEntry, issues []UAIssue, hints map[string]int, total int) {
fmt.Printf("=== UA Analysis: %d lines → %d unique platforms ===\n\n", total, len(entries))
fmt.Printf("%-10s %-10s %-6s %6s %s\n", "OS", "ARCH", "LIBC", "COUNT", "HINTS")
fmt.Println(strings.Repeat("-", 72))
for _, e := range entries {
hintStr := ""
if len(e.Hints) > 0 {
hintStr = strings.Join(e.Hints, ", ")
}
fmt.Printf("%-10s %-10s %-6s %6d %s\n",
displayOS(e.Key.OS), e.Key.Arch, displayLibc(e.Key.Libc),
e.Count, hintStr)
}
if len(hints) > 0 {
fmt.Printf("\n=== Platform Hints (environment signals from kernel strings) ===\n\n")
sortedHints := make([]Hint, 0, len(hints))
for tag, count := range hints {
sortedHints = append(sortedHints, Hint{tag, count})
}
sort.Slice(sortedHints, func(i, j int) bool {
return sortedHints[i].Count > sortedHints[j].Count
})
for _, h := range sortedHints {
fmt.Printf(" %-20s %d\n", h.Tag, h.Count)
}
}
if len(issues) > 0 {
fmt.Printf("\n=== Issues (%d) ===\n\n", len(issues))
for _, iss := range issues {
fmt.Printf(" line %d: %s\n %s\n", iss.Line, iss.Reason, iss.UA)
}
}
}
func outputJSON(entries []*PlatformEntry, issues []UAIssue, hints map[string]int) {
out := struct {
Platforms []*PlatformEntry `json:"platforms"`
Issues []UAIssue `json:"issues"`
Hints map[string]int `json:"hints"`
}{entries, issues, hints}
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
_ = enc.Encode(out)
}
func outputFixtures(entries []*PlatformEntry) {
fmt.Println("// Generated by cmd/uaparse from live UA data.")
fmt.Println("// Each entry represents a unique (os, arch, libc) platform seen in production.")
fmt.Println("var liveUAPlatforms = []struct {")
fmt.Println("\tua string")
fmt.Println("\tos buildmeta.OS")
fmt.Println("\tarch buildmeta.Arch")
fmt.Println("\tlibc buildmeta.Libc")
fmt.Println("}{")
for _, e := range entries {
if e.Key.OS == "" || e.Key.Arch == "" {
continue // skip undetectable
}
ua := e.Examples[0]
fmt.Printf("\t{%q, %s, %s, %s},\n",
ua, goConst("OS", e.Key.OS), goConst("Arch", e.Key.Arch), goConst("Libc", e.Key.Libc))
}
fmt.Println("}")
}
// isMalformed checks for genuinely corrupted UA strings (network truncation).
func isMalformed(ua string) bool {
// Extremely short (less than 10 chars) suggests truncation.
if len(ua) < 10 {
return true
}
return false
}
// extractHints finds environment signals in a UA string.
func extractHints(ua string) []string {
lower := strings.ToLower(ua)
var out []string
patterns := []struct {
substr string
tag string
}{
{"amzn", "amzn"}, // Amazon Linux
{"-azure", "azure"}, // Azure VM
{"-gcp", "gcp"}, // Google Cloud
{"-aws", "aws"}, // AWS kernel
{"-oracle", "oracle"}, // Oracle Cloud
{"el7", "rhel7"}, // RHEL/CentOS 7
{"el8", "rhel8"}, // RHEL/CentOS 8
{"el9", "rhel9"}, // RHEL/CentOS 9
{".fc", "fedora"}, // Fedora
{"+deb", "debian"}, // Debian
{"-generic", "ubuntu"}, // Ubuntu generic kernel
{"-pve", "proxmox"}, // Proxmox VE
{"linuxkit", "docker"}, // Docker Desktop / linuxkit
{"orbstack", "orbstack"},
{"microsoft-standard-wsl", "wsl"},
{"android", "android"},
{"+rpt-rpi", "rpi"}, // Raspberry Pi
{"cygwin", "cygwin"},
{"mingw", "mingw"},
{"msys", "msys"},
{"freebsd", "freebsd"},
{"-nvidia", "nvidia"},
{"gentoo", "gentoo"},
{"coreweave", "coreweave"},
}
for _, p := range patterns {
if strings.Contains(lower, p.substr) {
out = append(out, p.tag)
}
}
return out
}
// androidDeviceRe extracts device/build info from Android kernel strings.
var androidDeviceRe = regexp.MustCompile(`ab[A-Z0-9]+`)
func displayOS(os string) string {
if os == "" {
return "(none)"
}
return os
}
func displayLibc(libc string) string {
if libc == "" {
return "(none)"
}
return libc
}
func goConst(prefix, val string) string {
m := map[string]map[string]string{
"OS": {
"darwin": "buildmeta.OSDarwin",
"linux": "buildmeta.OSLinux",
"windows": "buildmeta.OSWindows",
"freebsd": "buildmeta.OSFreeBSD",
"android": "buildmeta.OSAndroid",
"": `""`,
},
"Arch": {
"aarch64": "buildmeta.ArchARM64",
"x86_64": "buildmeta.ArchAMD64",
"armv7": "buildmeta.ArchARMv7",
"armv6": "buildmeta.ArchARMv6",
"x86": "buildmeta.ArchX86",
"ppc64le": "buildmeta.ArchPPC64LE",
"ppc64": "buildmeta.ArchPPC64",
"s390x": "buildmeta.ArchS390X",
"riscv64": "buildmeta.ArchRISCV64",
"": `""`,
},
"Libc": {
"gnu": "buildmeta.LibcGNU",
"musl": "buildmeta.LibcMusl",
"msvc": "buildmeta.LibcMSVC",
"none": "buildmeta.LibcNone",
"": `""`,
},
}
if v, ok := m[prefix][val]; ok {
return v
}
return fmt.Sprintf("%q /* unmapped */", val)
}
func containsStr(ss []string, s string) bool {
for _, v := range ss {
if v == s {
return true
}
}
return false
}