From f5a8a0cd5a586d59749f4ccc79e3ec3cae64ce72 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Thu, 14 May 2026 17:39:57 -0600 Subject: [PATCH] feat(tools): add development and testing utilities (classify, comparecache, e2etest, fetchraw, inspect, uaparse) --- cmd/classify/main.go | 1293 ++++++++++++++++++++++++++++++++++++++ cmd/comparecache/main.go | 914 +++++++++++++++++++++++++++ cmd/e2etest/main.go | 846 +++++++++++++++++++++++++ cmd/fetchraw/main.go | 862 +++++++++++++++++++++++++ cmd/inspect/main.go | 625 ++++++++++++++++++ cmd/uaparse/main.go | 356 +++++++++++ 6 files changed, 4896 insertions(+) create mode 100644 cmd/classify/main.go create mode 100644 cmd/comparecache/main.go create mode 100644 cmd/e2etest/main.go create mode 100644 cmd/fetchraw/main.go create mode 100644 cmd/inspect/main.go create mode 100644 cmd/uaparse/main.go diff --git a/cmd/classify/main.go b/cmd/classify/main.go new file mode 100644 index 0000000..d91a0f8 --- /dev/null +++ b/cmd/classify/main.go @@ -0,0 +1,1293 @@ +// Command classify reads raw cached releases and produces a CSV of every +// distributable across all (or selected) packages. +// +// Usage: +// +// go run ./cmd/classify -cache ./_cache/raw -out distributables.csv +// go run ./cmd/classify -cache ./_cache/raw -out distributables.csv go node hugo +package main + +import ( + "encoding/csv" + "encoding/json" + "flag" + "fmt" + "log" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/webinstall/webi-installers/internal/classify" + "github.com/webinstall/webi-installers/internal/installerconf" + "github.com/webinstall/webi-installers/internal/lexver" + "github.com/webinstall/webi-installers/internal/rawcache" +) + +// Dist is one downloadable distributable — the core row in our CSV. +type Dist struct { + Package string // webi package name + Version string // version string (as-is from upstream) + Channel string // stable, beta, rc, dev, etc. + OS string // darwin, linux, windows, freebsd, etc. + Arch string // amd64, arm64, x86, armv7, etc. + Libc string // gnu, musl, msvc, or "" + Format string // tar.gz, zip, dmg, exe, msi, pkg, etc. + Download string // full download URL + Filename string // asset filename or archive path + SHA256 string // hash if available + Size int64 // file size in bytes if available + LTS bool // long-term support flag + Date string // release date if available + Extra string // source-specific notes (e.g. "extended", "installer") +} + +var csvHeader = []string{ + "package", "version", "ver_major", "ver_minor", "ver_patch", "ver_pre", + "channel", "date", "lts", + "os", "arch", "libc", "format", + "download", "filename", "sha256", "size", "extra", +} + +func main() { + cacheDir := flag.String("cache", "_cache/raw", "root directory for raw cache") + confDir := flag.String("conf", ".", "root directory containing {pkg}/releases.conf files") + outFile := flag.String("out", "distributables.csv", "output CSV file") + flag.Parse() + + packages := flag.Args() + if len(packages) == 0 { + // Discover all packages. + var err error + packages, err = discoverPackages(*confDir) + if err != nil { + log.Fatalf("discover: %v", err) + } + } + + log.Printf("classifying %d packages", len(packages)) + + var allDists []Dist + for _, pkg := range packages { + conf, err := installerconf.Read(filepath.Join(*confDir, pkg, "releases.conf")) + if err != nil { + log.Printf(" %s: skip (no conf: %v)", pkg, err) + continue + } + + // Skip aliases. + if alias := conf.Extra["alias_of"]; alias != "" { + continue + } + + source := conf.Source + d, err := rawcache.Open(filepath.Join(*cacheDir, pkg)) + if err != nil { + log.Printf(" %s: skip (no cache: %v)", pkg, err) + continue + } + + dists, err := classifyPackage(pkg, source, conf, d) + if err != nil { + log.Printf(" %s: ERROR: %v", pkg, err) + continue + } + + log.Printf(" %s: %d distributables", pkg, len(dists)) + allDists = append(allDists, dists...) + } + + // Write CSV. + f, err := os.Create(*outFile) + if err != nil { + log.Fatalf("create %s: %v", *outFile, err) + } + defer f.Close() + + w := csv.NewWriter(f) + w.Write(csvHeader) + for _, d := range allDists { + v := lexver.Parse(d.Version) + lts := "" + if d.LTS { + lts = "true" + } + size := "" + if d.Size > 0 { + size = fmt.Sprintf("%d", d.Size) + } + pre := v.Channel + if v.ChannelNum > 0 { + pre = fmt.Sprintf("%s%d", v.Channel, v.ChannelNum) + } + w.Write([]string{ + d.Package, d.Version, + fmt.Sprintf("%d", v.Major()), fmt.Sprintf("%d", v.Minor()), fmt.Sprintf("%d", v.Patch()), pre, + d.Channel, d.Date, lts, + d.OS, d.Arch, d.Libc, d.Format, + d.Download, d.Filename, d.SHA256, size, d.Extra, + }) + } + w.Flush() + if err := w.Error(); err != nil { + log.Fatalf("write csv: %v", err) + } + + log.Printf("wrote %d rows to %s", len(allDists), *outFile) +} + +func discoverPackages(confDir string) ([]string, error) { + matches, err := filepath.Glob(filepath.Join(confDir, "*", "releases.conf")) + if err != nil { + return nil, err + } + var pkgs []string + for _, path := range matches { + name := filepath.Base(filepath.Dir(path)) + if strings.HasPrefix(name, "_") { + continue + } + pkgs = append(pkgs, name) + } + sort.Strings(pkgs) + return pkgs, nil +} + +func classifyPackage(pkg, source string, conf *installerconf.Conf, d *rawcache.Dir) ([]Dist, error) { + switch source { + case "github": + return classifyGitHub(pkg, conf, d) + case "golang": + return classifyGolang(pkg, d) + case "nodedist": + return classifyNodeDist(pkg, conf, d) + case "zigdist": + return classifyZigDist(pkg, d) + case "flutterdist": + return classifyFlutterDist(pkg, d) + case "chromedist": + return classifyChromeDist(pkg, d) + case "hashicorp": + return classifyHashiCorp(pkg, conf, d) + case "juliadist": + return classifyJuliaDist(pkg, d) + case "iterm2dist": + return classifyITerm2Dist(pkg, d) + case "gpgdist": + return classifyGPGDist(pkg, d) + case "mariadbdist": + return classifyMariaDBDist(pkg, d) + case "gittag": + return classifyGitTag(pkg, d) + case "gitea": + return classifyGitea(pkg, conf, d) + default: + return nil, fmt.Errorf("unknown source %q", source) + } +} + +// readAllReleases reads every cached release file from the active slot. +func readAllReleases(d *rawcache.Dir) (map[string][]byte, error) { + active, err := d.ActivePath() + if err != nil { + return nil, err + } + entries, err := os.ReadDir(active) + if err != nil { + return nil, err + } + result := make(map[string][]byte, len(entries)) + for _, e := range entries { + if e.IsDir() || strings.HasPrefix(e.Name(), "_") { + continue + } + data, err := os.ReadFile(filepath.Join(active, e.Name())) + if err != nil { + return nil, err + } + result[e.Name()] = data + } + return result, nil +} + +// classifyFilename runs the generic classifier on a filename and returns +// os, arch, libc, format strings. +func classifyFilename(filename string) (os_, arch, libc, format string) { + r := classify.Filename(filename) + return string(r.OS), string(r.Arch), string(r.Libc), string(r.Format) +} + +// --- GitHub releases --- + +type ghRelease struct { + TagName string `json:"tag_name"` + Name string `json:"name"` + Prerelease bool `json:"prerelease"` + Draft bool `json:"draft"` + PublishedAt string `json:"published_at"` + Assets []ghAsset `json:"assets"` + TarballURL string `json:"tarball_url"` + ZipballURL string `json:"zipball_url"` +} + +type ghAsset struct { + Name string `json:"name"` + BrowserDownloadURL string `json:"browser_download_url"` + Size int64 `json:"size"` + ContentType string `json:"content_type"` +} + +func classifyGitHub(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]Dist, error) { + tagPrefix := conf.TagPrefix + assetFilter := strings.ToLower(conf.Extra["asset_filter"]) // asset must contain this + assetExclude := strings.ToLower(conf.Extra["asset_exclude"]) // asset must NOT contain this + releases, err := readAllReleases(d) + if err != nil { + return nil, err + } + + var dists []Dist + for _, data := range releases { + var rel ghRelease + if err := json.Unmarshal(data, &rel); err != nil { + continue + } + if rel.Draft { + continue + } + + version := rel.TagName + if tagPrefix != "" { + version = strings.TrimPrefix(version, tagPrefix) + } + + channel := "stable" + if rel.Prerelease { + channel = "beta" + } + + date := "" + if rel.PublishedAt != "" { + date = rel.PublishedAt[:10] // YYYY-MM-DD + } + + for _, asset := range rel.Assets { + name := asset.Name + lower := strings.ToLower(name) + + // Skip checksums, signatures, SBOMs, etc. + if isMetaAsset(name) { + continue + } + + // Per-package asset filters. + if assetFilter != "" && !strings.Contains(lower, assetFilter) { + continue + } + if assetExclude != "" && strings.Contains(lower, assetExclude) { + continue + } + + os_, arch, libc, format := classifyFilename(name) + + dists = append(dists, Dist{ + Package: pkg, + Version: version, + Channel: channel, + OS: os_, + Arch: arch, + Libc: libc, + Format: format, + Download: asset.BrowserDownloadURL, + Filename: name, + Size: asset.Size, + Date: date, + }) + } + + // Source-tarball packages: no binary assets, distributed via + // GitHub's auto-generated tarball/zipball URLs. + if len(rel.Assets) == 0 { + if rel.TarballURL != "" { + dists = append(dists, Dist{ + Package: pkg, + Version: version, + Channel: channel, + Format: ".tar.gz", + Download: rel.TarballURL, + Filename: rel.TagName + ".tar.gz", + Date: date, + Extra: "source", + }) + } + if rel.ZipballURL != "" { + dists = append(dists, Dist{ + Package: pkg, + Version: version, + Channel: channel, + Format: ".zip", + Download: rel.ZipballURL, + Filename: rel.TagName + ".zip", + Date: date, + Extra: "source", + }) + } + } + } + return dists, nil +} + +// --- Go releases --- + +type goRelease struct { + Version string `json:"version"` + Stable bool `json:"stable"` + Files []goFile `json:"files"` +} + +type goFile struct { + Filename string `json:"filename"` + OS string `json:"os"` + Arch string `json:"arch"` + Version string `json:"version"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` + Kind string `json:"kind"` +} + +func classifyGolang(pkg string, d *rawcache.Dir) ([]Dist, error) { + releases, err := readAllReleases(d) + if err != nil { + return nil, err + } + + var dists []Dist + for _, data := range releases { + var rel goRelease + if err := json.Unmarshal(data, &rel); err != nil { + continue + } + + channel := "stable" + if !rel.Stable { + channel = "beta" + } + + // Go versions come as "go1.23.6" — strip prefix for standard semver parsing. + version := strings.TrimPrefix(rel.Version, "go") + + for _, f := range rel.Files { + if f.Kind == "source" || f.OS == "" { + continue + } + + // Go API gives structured os/arch — use it directly. + os_ := normalizeGoOS(f.OS) + arch := normalizeGoArch(f.Arch) + format := detectFormat(f.Filename) + + extra := "" + if f.Kind == "installer" { + extra = "installer" + } + + dists = append(dists, Dist{ + Package: pkg, + Version: version, + Channel: channel, + OS: os_, + Arch: arch, + Format: format, + Download: fmt.Sprintf("https://dl.google.com/go/%s", f.Filename), + Filename: f.Filename, + SHA256: f.SHA256, + Size: f.Size, + Extra: extra, + }) + } + } + return dists, nil +} + +func normalizeGoOS(os_ string) string { + // Go's GOOS values already match buildmeta for the common cases. + return os_ +} + +func normalizeGoArch(arch string) string { + switch arch { + case "amd64": + return "x86_64" + case "arm64": + return "aarch64" + case "386": + return "x86" + case "armv6l": + return "armv6" + default: + return arch + } +} + +// --- Node.js dist --- + +type nodeEntry struct { + Version string `json:"version"` + Date string `json:"date"` + Files []string `json:"files"` + LTS json.RawMessage `json:"lts"` + Security bool `json:"security"` +} + +func classifyNodeDist(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]Dist, error) { + baseURL := conf.BaseURL + releases, err := readAllReleases(d) + if err != nil { + return nil, err + } + + var dists []Dist + for _, data := range releases { + var entry nodeEntry + if err := json.Unmarshal(data, &entry); err != nil { + continue + } + + lts := string(entry.LTS) != "false" && string(entry.LTS) != "" + channel := "stable" + // Odd major versions are "current" (beta/dev). + ver := strings.TrimPrefix(entry.Version, "v") + parts := strings.SplitN(ver, ".", 2) + if len(parts) > 0 { + major := 0 + fmt.Sscanf(parts[0], "%d", &major) + if major%2 != 0 { + channel = "beta" + } + } + + for _, file := range entry.Files { + if file == "src" || file == "headers" { + continue + } + + fileDists := expandNodeFile(pkg, entry.Version, channel, entry.Date, lts, baseURL, file) + dists = append(dists, fileDists...) + } + } + return dists, nil +} + +func expandNodeFile(pkg, version, channel, date string, lts bool, baseURL, file string) []Dist { + parts := strings.Split(file, "-") + if len(parts) < 2 { + return nil + } + + osMap := map[string]string{ + "osx": "darwin", "linux": "linux", "win": "windows", + "sunos": "sunos", "aix": "aix", + } + archMap := map[string]string{ + "x64": "x86_64", "x86": "x86", "arm64": "aarch64", + "armv7l": "armv7", "armv6l": "armv6", + "ppc64": "ppc64", "ppc64le": "ppc64le", "s390x": "s390x", + } + + os_ := osMap[parts[0]] + arch := archMap[parts[1]] + if os_ == "" || arch == "" { + return nil + } + + libc := "" + pkgType := "" + if len(parts) > 2 { + pkgType = parts[2] + } + + var formats []string + switch pkgType { + case "musl": + libc = "musl" + formats = []string{".tar.gz", ".tar.xz"} + case "tar": + formats = []string{".tar.gz", ".tar.xz"} + case "zip": + formats = []string{".zip"} + case "7z": + formats = []string{".7z"} + case "pkg": + formats = []string{".pkg"} + case "msi": + formats = []string{".msi"} + case "exe": + formats = []string{".exe"} + case "": + formats = []string{".tar.gz", ".tar.xz"} + default: + return nil + } + + if libc == "" && os_ == "linux" { + libc = "gnu" + } + + osPart := parts[0] + if osPart == "osx" { + osPart = "darwin" + } + archPart := parts[1] + muslExtra := "" + if libc == "musl" { + muslExtra = "-musl" + } + + var dists []Dist + for _, format := range formats { + ext := format // already has leading dot + var filename string + if format == ".msi" { + filename = fmt.Sprintf("node-%s-%s%s%s", version, archPart, muslExtra, ext) + } else { + filename = fmt.Sprintf("node-%s-%s-%s%s%s", version, osPart, archPart, muslExtra, ext) + } + + dists = append(dists, Dist{ + Package: pkg, + Version: version, + Channel: channel, + OS: os_, + Arch: arch, + Libc: libc, + Format: format, + Download: fmt.Sprintf("%s/%s/%s", baseURL, version, filename), + Filename: filename, + LTS: lts, + Date: date, + }) + } + return dists +} + +// --- Zig dist --- + +type zigRelease struct { + Version string `json:"version"` + Date string `json:"date"` + Platforms map[string]json.RawMessage `json:"-"` +} + +type zigPlatform struct { + Tarball string `json:"tarball"` + Shasum string `json:"shasum"` + Size json.Number `json:"size"` +} + +func classifyZigDist(pkg string, d *rawcache.Dir) ([]Dist, error) { + releases, err := readAllReleases(d) + if err != nil { + return nil, err + } + + var dists []Dist + for _, data := range releases { + // Parse the cached JSON which has version, date, platforms. + var rel struct { + Version string `json:"version"` + Date string `json:"date"` + Platforms map[string]zigPlatform `json:"platforms"` + } + if err := json.Unmarshal(data, &rel); err != nil { + continue + } + + channel := "stable" + if strings.Contains(rel.Version, "+") || strings.Contains(rel.Version, "-") || !strings.Contains(rel.Version, ".") { + channel = "beta" + } + + for key, plat := range rel.Platforms { + if plat.Tarball == "" { + continue + } + + // key is like "x86_64-linux", "aarch64-macos", "x86_64-windows" + parts := strings.SplitN(key, "-", 2) + if len(parts) != 2 { + continue + } + + arch := normalizeZigArch(parts[0]) + os_ := normalizeZigOS(parts[1]) + format := detectFormat(plat.Tarball) + + dists = append(dists, Dist{ + Package: pkg, + Version: rel.Version, + Channel: channel, + OS: os_, + Arch: arch, + Format: format, + Download: plat.Tarball, + Filename: filepath.Base(plat.Tarball), + SHA256: plat.Shasum, + Size: zigSize(plat.Size), + Date: rel.Date, + }) + } + } + return dists, nil +} + +func normalizeZigArch(a string) string { + switch a { + case "x86_64": + return "x86_64" + case "aarch64": + return "aarch64" + case "x86": + return "x86" + case "armv7a": + return "armv7" + case "riscv64": + return "riscv64" + case "powerpc64le": + return "ppc64le" + case "powerpc": + return "ppc" + case "loongarch64": + return "loong64" + case "s390x": + return "s390x" + default: + return a + } +} + +func zigSize(n json.Number) int64 { + v, _ := n.Int64() + return v +} + +func normalizeZigOS(o string) string { + switch o { + case "macos": + return "darwin" + case "linux": + return "linux" + case "windows": + return "windows" + default: + return o + } +} + +// --- Flutter dist --- + +type flutterRelease struct { + Hash string `json:"hash"` + Channel string `json:"channel"` + Version string `json:"version"` + ReleaseDate string `json:"release_date"` + Archive string `json:"archive"` + SHA256 string `json:"sha256"` + DownloadURL string `json:"download_url"` + OS string `json:"os"` +} + +func classifyFlutterDist(pkg string, d *rawcache.Dir) ([]Dist, error) { + releases, err := readAllReleases(d) + if err != nil { + return nil, err + } + + var dists []Dist + for _, data := range releases { + var rel flutterRelease + if err := json.Unmarshal(data, &rel); err != nil { + continue + } + + os_ := rel.OS + if os_ == "macos" { + os_ = "darwin" + } + + // Detect arch from the archive path. + arch := "" + archive := strings.ToLower(rel.Archive) + switch { + case strings.Contains(archive, "arm64"): + arch = "arm64" + case strings.Contains(archive, "x64") || strings.Contains(archive, "x86_64"): + arch = "amd64" + } + + format := detectFormat(rel.Archive) + date := "" + if len(rel.ReleaseDate) >= 10 { + date = rel.ReleaseDate[:10] + } + + dists = append(dists, Dist{ + Package: pkg, + Version: rel.Version, + Channel: rel.Channel, + OS: os_, + Arch: arch, + Format: format, + Download: rel.DownloadURL, + Filename: filepath.Base(rel.Archive), + SHA256: rel.SHA256, + Date: date, + }) + } + return dists, nil +} + +// --- Chrome for Testing --- + +type chromeVersion struct { + Version string `json:"version"` + Revision string `json:"revision"` + Downloads map[string][]chromeDownload `json:"downloads"` +} + +type chromeDownload struct { + Platform string `json:"platform"` + URL string `json:"url"` +} + +func classifyChromeDist(pkg string, d *rawcache.Dir) ([]Dist, error) { + releases, err := readAllReleases(d) + if err != nil { + return nil, err + } + + var dists []Dist + for _, data := range releases { + var ver chromeVersion + if err := json.Unmarshal(data, &ver); err != nil { + continue + } + + drivers, ok := ver.Downloads["chromedriver"] + if !ok { + continue + } + + for _, dl := range drivers { + os_, arch := normalizeChromeplatform(dl.Platform) + + dists = append(dists, Dist{ + Package: pkg, + Version: ver.Version, + Channel: "stable", + OS: os_, + Arch: arch, + Format: ".zip", + Download: dl.URL, + Filename: filepath.Base(dl.URL), + }) + } + } + return dists, nil +} + +func normalizeChromeplatform(p string) (os_, arch string) { + switch p { + case "linux64": + return "linux", "x86_64" + case "mac-arm64": + return "darwin", "aarch64" + case "mac-x64": + return "darwin", "x86_64" + case "win32": + return "windows", "x86" + case "win64": + return "windows", "x86_64" + default: + return p, "" + } +} + +// --- HashiCorp --- + +type hcVersion struct { + Version string `json:"version"` + Builds []hcBuild `json:"builds"` +} + +type hcBuild struct { + Version string `json:"version"` + OS string `json:"os"` + Arch string `json:"arch"` + Filename string `json:"filename"` + URL string `json:"url"` +} + +func classifyHashiCorp(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]Dist, error) { + releases, err := readAllReleases(d) + if err != nil { + return nil, err + } + + var dists []Dist + for _, data := range releases { + var ver hcVersion + if err := json.Unmarshal(data, &ver); err != nil { + continue + } + + channel := "stable" + if strings.Contains(ver.Version, "-") { + channel = "beta" + } + + for _, b := range ver.Builds { + arch := normalizeHCArch(b.Arch) + format := detectFormat(b.Filename) + + dists = append(dists, Dist{ + Package: pkg, + Version: ver.Version, + Channel: channel, + OS: b.OS, + Arch: arch, + Format: format, + Download: b.URL, + Filename: b.Filename, + }) + } + } + return dists, nil +} + +func normalizeHCArch(a string) string { + switch a { + case "amd64": + return "x86_64" + case "arm64": + return "aarch64" + case "386": + return "x86" + case "arm": + return "armv6" + default: + return a + } +} + +// --- Julia --- + +type juliaRelease struct { + Version string `json:"version"` + Stable bool `json:"stable"` + Files []juliaFile `json:"files"` +} + +type juliaFile struct { + URL string `json:"url"` + Triplet string `json:"triplet"` + Kind string `json:"kind"` + Arch string `json:"arch"` + OS string `json:"os"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` + Extension string `json:"extension"` +} + +func classifyJuliaDist(pkg string, d *rawcache.Dir) ([]Dist, error) { + releases, err := readAllReleases(d) + if err != nil { + return nil, err + } + + var dists []Dist + for _, data := range releases { + var rel juliaRelease + if err := json.Unmarshal(data, &rel); err != nil { + continue + } + + channel := "stable" + if !rel.Stable { + channel = "beta" + } + + for _, f := range rel.Files { + if f.Kind == "installer" { + continue + } + + os_ := normalizeJuliaOS(f.OS) + arch := normalizeJuliaArch(f.Arch) + libc := "" + if strings.Contains(f.URL, "musl") { + libc = "musl" + } else if os_ == "linux" { + libc = "gnu" + } + + format := f.Extension + if format != "" && format[0] != '.' { + format = "." + format + } + + dists = append(dists, Dist{ + Package: pkg, + Version: rel.Version, + Channel: channel, + OS: os_, + Arch: arch, + Libc: libc, + Format: format, + Download: f.URL, + Filename: filepath.Base(f.URL), + SHA256: f.SHA256, + Size: f.Size, + }) + } + } + return dists, nil +} + +func normalizeJuliaOS(o string) string { + switch o { + case "mac": + return "darwin" + case "winnt": + return "windows" + default: + return o + } +} + +func normalizeJuliaArch(a string) string { + switch a { + case "x86_64": + return "x86_64" + case "aarch64": + return "aarch64" + case "i686": + return "x86" + case "armv7l": + return "armv7" + case "powerpc64le": + return "ppc64le" + default: + return a + } +} + +// --- iTerm2 --- + +type iterm2Entry struct { + Version string `json:"version"` + Channel string `json:"channel"` + URL string `json:"url"` +} + +func classifyITerm2Dist(pkg string, d *rawcache.Dir) ([]Dist, error) { + releases, err := readAllReleases(d) + if err != nil { + return nil, err + } + + var dists []Dist + for _, data := range releases { + var entry iterm2Entry + if err := json.Unmarshal(data, &entry); err != nil { + continue + } + + dists = append(dists, Dist{ + Package: pkg, + Version: entry.Version, + Channel: entry.Channel, + OS: "darwin", + Format: ".zip", + Download: entry.URL, + Filename: filepath.Base(entry.URL), + }) + } + return dists, nil +} + +// --- GPG --- + +type gpgEntry struct { + Version string `json:"version"` + URL string `json:"url"` +} + +func classifyGPGDist(pkg string, d *rawcache.Dir) ([]Dist, error) { + releases, err := readAllReleases(d) + if err != nil { + return nil, err + } + + var dists []Dist + for _, data := range releases { + var entry gpgEntry + if err := json.Unmarshal(data, &entry); err != nil { + continue + } + + dists = append(dists, Dist{ + Package: pkg, + Version: entry.Version, + Channel: "stable", + OS: "darwin", + Arch: "x86_64", + Format: ".dmg", + Download: entry.URL, + Filename: filepath.Base(entry.URL), + }) + } + return dists, nil +} + +// --- MariaDB --- + +type mariadbRelease struct { + ReleaseID string `json:"release_id"` + DateOfRelease string `json:"date_of_release"` + Files []mariadbFile `json:"files"` + MajorStatus string `json:"major_status"` +} + +type mariadbFile struct { + FileName string `json:"file_name"` + OS string `json:"os"` + CPU string `json:"cpu"` + Checksum mariadbChecksum `json:"checksum"` + FileDownloadURL string `json:"file_download_url"` +} + +type mariadbChecksum struct { + SHA256 string `json:"sha256sum"` +} + +func classifyMariaDBDist(pkg string, d *rawcache.Dir) ([]Dist, error) { + releases, err := readAllReleases(d) + if err != nil { + return nil, err + } + + var dists []Dist + for _, data := range releases { + var rel mariadbRelease + if err := json.Unmarshal(data, &rel); err != nil { + continue + } + + channel := "stable" + if rel.MajorStatus != "Stable" { + channel = strings.ToLower(rel.MajorStatus) + } + + for _, f := range rel.Files { + if f.OS == "" || f.CPU == "" { + continue // source or docs + } + if strings.Contains(f.FileName, "debug") { + continue + } + + os_ := strings.ToLower(f.OS) + arch := normalizeMariaDBArch(f.CPU) + format := detectFormat(f.FileName) + + dists = append(dists, Dist{ + Package: pkg, + Version: rel.ReleaseID, + Channel: channel, + OS: os_, + Arch: arch, + Format: format, + Download: f.FileDownloadURL, + Filename: f.FileName, + SHA256: f.Checksum.SHA256, + Date: rel.DateOfRelease, + }) + } + } + return dists, nil +} + +func normalizeMariaDBArch(a string) string { + a = strings.TrimSpace(a) + switch a { + case "x86_64": + return "x86_64" + case "aarch64": + return "aarch64" + default: + return a + } +} + +// --- Git tag --- + +type gitTagEntry struct { + Version string `json:"Version"` + GitTag string `json:"GitTag"` + CommitHash string `json:"CommitHash"` + Date string `json:"Date"` +} + +func classifyGitTag(pkg string, d *rawcache.Dir) ([]Dist, error) { + releases, err := readAllReleases(d) + if err != nil { + return nil, err + } + + var dists []Dist + for _, data := range releases { + var entry gitTagEntry + if err := json.Unmarshal(data, &entry); err != nil { + continue + } + + date := "" + if len(entry.Date) >= 10 { + date = entry.Date[:10] + } + + dists = append(dists, Dist{ + Package: pkg, + Version: entry.Version, + Channel: "stable", + Format: ".git", + Download: entry.GitTag, + Date: date, + Extra: "commit:" + entry.CommitHash, + }) + } + return dists, nil +} + +// --- Gitea --- + +type giteaRelease struct { + TagName string `json:"tag_name"` + Prerelease bool `json:"prerelease"` + Draft bool `json:"draft"` + PublishedAt string `json:"published_at"` + Assets []giteaAsset `json:"assets"` +} + +type giteaAsset struct { + Name string `json:"name"` + BrowserDownloadURL string `json:"browser_download_url"` + Size int64 `json:"size"` +} + +func classifyGitea(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]Dist, error) { + releases, err := readAllReleases(d) + if err != nil { + return nil, err + } + + var dists []Dist + for _, data := range releases { + var rel giteaRelease + if err := json.Unmarshal(data, &rel); err != nil { + continue + } + if rel.Draft { + continue + } + + channel := "stable" + if rel.Prerelease { + channel = "beta" + } + + date := "" + if len(rel.PublishedAt) >= 10 { + date = rel.PublishedAt[:10] + } + + for _, asset := range rel.Assets { + if isMetaAsset(asset.Name) { + continue + } + + os_, arch, libc, format := classifyFilename(asset.Name) + + dists = append(dists, Dist{ + Package: pkg, + Version: rel.TagName, + Channel: channel, + OS: os_, + Arch: arch, + Libc: libc, + Format: format, + Download: asset.BrowserDownloadURL, + Filename: asset.Name, + Size: asset.Size, + Date: date, + }) + } + } + return dists, nil +} + +// --- Helpers --- + +// isMetaAsset returns true for checksums, signatures, SBOMs, and other +// non-distributable assets. +func isMetaAsset(name string) bool { + lower := strings.ToLower(name) + for _, suffix := range []string{ + ".sha256", ".sha256sum", ".sha512", ".sha512sum", + ".md5", ".md5sum", ".sig", ".asc", ".pem", + "checksums.txt", "sha256sums", "sha512sums", + ".sbom", ".spdx", ".json.sig", ".sigstore", + "_src.tar.gz", "_src.tar.xz", "_src.zip", + ".d.ts", // TypeScript definitions + ".pub", // cosign/SSH public keys + } { + if strings.HasSuffix(lower, suffix) { + return true + } + } + for _, contains := range []string{ + "checksums", "sha256sum", "sha512sum", + "buildable-artifact", + } { + if strings.Contains(lower, contains) { + return true + } + } + // Exact name matches for known non-distributable files. + for _, exact := range []string{ + "install.sh", "install.ps1", "compat.json", + } { + if lower == exact { + return true + } + } + return false +} + +// detectFormat extracts the file format from a filename. +// Returns with leading dot to match buildmeta.Format constants. +func detectFormat(name string) string { + lower := strings.ToLower(name) + // Check compound extensions first. + for _, ext := range []string{".tar.gz", ".tar.xz", ".tar.bz2", ".tar.zst", ".exe.xz", ".app.zip"} { + if strings.HasSuffix(lower, ext) { + return ext + } + } + // .tgz is a common alias for .tar.gz (used by ollama, npm, etc.) + if strings.HasSuffix(lower, ".tgz") { + return ".tar.gz" + } + ext := filepath.Ext(lower) + return ext // includes leading dot, or "" if none +} diff --git a/cmd/comparecache/main.go b/cmd/comparecache/main.go new file mode 100644 index 0000000..c3b6fd4 --- /dev/null +++ b/cmd/comparecache/main.go @@ -0,0 +1,914 @@ +// Command comparecache compares Go-generated cache output against the +// Node.js LIVE_cache. It identifies categorical differences in asset +// selection — which filenames appear in one cache but not the other. +// +// The comparison is done at the filename level (not OS/arch/ext fields) +// because the Node.js cache leaves those empty (normalize.js fills them +// at serve time), while the Go pipeline classifies at write time. +// +// Usage: +// +// go run ./cmd/comparecache -live ./LIVE_cache -go ./_cache +// go run ./cmd/comparecache -live ./LIVE_cache -go ./_cache bat jq +// go run ./cmd/comparecache -live ./LIVE_cache -go ./_cache -summary +package main + +import ( + "encoding/json" + "flag" + "fmt" + "log" + "math/rand/v2" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "time" + + "github.com/webinstall/webi-installers/internal/classify" + "github.com/webinstall/webi-installers/internal/lexver" +) + +type cacheEntry struct { + Releases []cacheRelease `json:"releases"` +} + +type cacheRelease struct { + Name string `json:"name"` + Filename string `json:"_filename"` // Node.js uses _filename for some sources + Version string `json:"version"` + Download string `json:"download"` + Channel string `json:"channel"` + OS string `json:"os"` + Arch string `json:"arch"` + Libc string `json:"libc"` + Ext string `json:"ext"` +} + +// fieldDiff records a field-level difference for an asset that exists +// in both caches (same filename) but has different classification. +type fieldDiff struct { + Filename string + Field string // "os", "arch", "libc", "ext", "channel" + Live string + Go string + BothSet bool // true when both live and go have non-empty values +} + +type packageDiff struct { + Name string + LiveCount int + GoCount int + OnlyInLive []string // filenames only in Node.js cache + OnlyInGo []string // filenames only in Go cache + FieldDiffs []fieldDiff // classification differences on shared assets + VersionsLive []string // unique versions in live + VersionsGo []string // unique versions in go + GoMissing bool // true if Go didn't produce output for this package + LiveMissing bool // true if no live cache for this package + Categories []string // categorical difference labels +} + +func main() { + liveDir := flag.String("live", "./LIVE_cache", "path to Node.js LIVE_cache directory") + goDir := flag.String("go", "./_cache", "path to Go cache directory") + summary := flag.Bool("summary", false, "only print summary, not per-package details") + diffsOnly := flag.Bool("diffs", false, "only show packages with asset differences (skip matches)") + latest := flag.Bool("latest", false, "only compare latest version in each cache") + windowed := flag.Bool("windowed", false, "limit Go versions to the Node.js version range (2nd to 2nd-to-last)") + sample := flag.Int("sample", 0, "for each package diff, show N randomly sampled assets (implies -windowed -diffs)") + flag.Parse() + filterPkgs := flag.Args() + + // -sample implies -windowed and -diffs so we focus on real classification + // differences, not version-depth noise. + if *sample > 0 { + *windowed = true + *diffsOnly = true + } + + totalStart := time.Now() + + // Find the most recent month directory in each cache. + liveMonth := findLatestMonth(*liveDir) + goMonth := findLatestMonth(*goDir) + if liveMonth == "" { + log.Fatalf("no month directories found in %s", *liveDir) + } + + livePath := filepath.Join(*liveDir, liveMonth) + goPath := "" + if goMonth != "" { + goPath = filepath.Join(*goDir, goMonth) + } + + // Discover all packages across both caches. + discoverStart := time.Now() + allPkgs := discoverPackages(livePath, goPath) + if len(filterPkgs) > 0 { + nameSet := make(map[string]bool, len(filterPkgs)) + for _, n := range filterPkgs { + nameSet[n] = true + } + var filtered []string + for _, p := range allPkgs { + if nameSet[p] { + filtered = append(filtered, p) + } + } + allPkgs = filtered + } + log.Printf("discovered %d packages in %s", len(allPkgs), time.Since(discoverStart)) + + compareStart := time.Now() + var diffs []packageDiff + for _, pkg := range allPkgs { + d := compare(livePath, goPath, pkg, *latest, *windowed) + categorize(&d) + diffs = append(diffs, d) + } + log.Printf("compared %d packages in %s", len(diffs), time.Since(compareStart)) + + if *summary { + printSummary(diffs) + } else { + printDetails(diffs, *diffsOnly, *sample) + } + + log.Printf("total: %s", time.Since(totalStart)) +} + +func findLatestMonth(dir string) string { + entries, err := os.ReadDir(dir) + if err != nil { + return "" + } + var months []string + for _, e := range entries { + if e.IsDir() && len(e.Name()) == 7 && e.Name()[4] == '-' { + months = append(months, e.Name()) + } + } + if len(months) == 0 { + return "" + } + sort.Strings(months) + return months[len(months)-1] +} + +func discoverPackages(livePath, goPath string) []string { + seen := make(map[string]bool) + for _, dir := range []string{livePath, goPath} { + if dir == "" { + continue + } + entries, err := os.ReadDir(dir) + if err != nil { + continue + } + for _, e := range entries { + name := e.Name() + if strings.HasSuffix(name, ".json") && !strings.HasSuffix(name, ".updated.txt") { + pkg := strings.TrimSuffix(name, ".json") + seen[pkg] = true + } + } + } + var pkgs []string + for p := range seen { + pkgs = append(pkgs, p) + } + sort.Strings(pkgs) + return pkgs +} + +func loadCache(dir, pkg string) *cacheEntry { + if dir == "" { + return nil + } + data, err := os.ReadFile(filepath.Join(dir, pkg+".json")) + if err != nil { + return nil + } + var entry cacheEntry + if err := json.Unmarshal(data, &entry); err != nil { + return nil + } + return &entry +} + +// effectiveName returns the best available filename for a release entry. +// Node.js sometimes uses _filename (a path) instead of name. +func effectiveName(name, filename, download string) string { + if name != "" { + return name + } + if filename != "" { + // _filename may be a path like "stable/macos/flutter_macos_3.41.4.zip" + if i := strings.LastIndex(filename, "/"); i >= 0 { + return filename[i+1:] + } + return filename + } + // Last resort: basename of download URL. + if download != "" { + if i := strings.LastIndex(download, "/"); i >= 0 { + return download[i+1:] + } + } + return "" +} + +// versionWindow returns the 2nd and 2nd-to-last versions from a sorted +// version list. This trims the edges where Node.js may have a newer fetch +// or Go may have deeper history, focusing on the overlapping middle. +func versionWindow(versions []string) (low, high string) { + if len(versions) <= 2 { + // Too few versions to window — use all. + if len(versions) > 0 { + return versions[0], versions[len(versions)-1] + } + return "", "" + } + // 2nd version (skip oldest) and 2nd-to-last (skip newest). + return versions[1], versions[len(versions)-2] +} + +// filterVersionRange returns only the versions in sorted order that fall +// within [low, high] inclusive (by lexver comparison). +func filterVersionRange(vf map[string]map[string]bool, versions []string, low, high string) (map[string]bool, []string) { + lowV := lexver.Parse(low) + highV := lexver.Parse(high) + + files := make(map[string]bool) + var kept []string + for _, v := range versions { + pv := lexver.Parse(v) + if lexver.Compare(pv, lowV) >= 0 && lexver.Compare(pv, highV) <= 0 { + kept = append(kept, v) + for f := range vf[v] { + files[f] = true + } + } + } + return files, kept +} + +func compare(livePath, goPath, pkg string, latestOnly, windowed bool) packageDiff { + live := loadCache(livePath, pkg) + goCache := loadCache(goPath, pkg) + + d := packageDiff{Name: pkg} + + if live == nil { + d.LiveMissing = true + } + if goCache == nil { + d.GoMissing = true + } + if d.LiveMissing && d.GoMissing { + return d + } + + normVersion := normalizeVersionFunc(pkg) + + // Collect filenames by version. If filter is non-nil, skip filenames it rejects. + extractVersionFiles := func(ce *cacheEntry, filter func(string) bool) (map[string]map[string]bool, []string) { + vf := make(map[string]map[string]bool) + for _, r := range ce.Releases { + name := effectiveName(r.Name, r.Filename, r.Download) + if filter != nil && !filter(name) { + continue + } + ver := normVersion(r.Version) + if vf[ver] == nil { + vf[ver] = make(map[string]bool) + } + vf[ver][name] = true + } + var versions []string + for v := range vf { + versions = append(versions, v) + } + slices.SortFunc(versions, func(a, b string) int { + return lexver.Compare(lexver.Parse(a), lexver.Parse(b)) + }) + return vf, versions + } + notNoise := func(name string) bool { return !isLiveNoise(name) } + + var liveFiles, goFiles map[string]bool + + // Parse live cache. + var liveVF map[string]map[string]bool + var liveVersions []string + if live != nil { + liveVF, liveVersions = extractVersionFiles(live, notNoise) + d.VersionsLive = liveVersions + d.LiveCount = len(live.Releases) + } + + // Parse Go cache. + var goVF map[string]map[string]bool + var goVersions []string + if goCache != nil { + goVF, goVersions = extractVersionFiles(goCache, notNoise) + d.VersionsGo = goVersions + d.GoCount = len(goCache.Releases) + } + + // Determine which files to compare based on mode. + if latestOnly { + // Compare only the latest version from each cache. + if live != nil && len(liveVersions) > 0 { + liveFiles = liveVF[liveVersions[len(liveVersions)-1]] + } + if goCache != nil && len(goVersions) > 0 { + goFiles = goVF[goVersions[len(goVersions)-1]] + } + } else if windowed && live != nil && len(liveVersions) > 0 { + // Use the Node.js version range (2nd to 2nd-to-last) to establish + // the window. Include ALL Node.js versions in the window (so missing + // Go versions are visible), but exclude Go-only versions (those are + // just deeper history, not real gaps). + low, high := versionWindow(liveVersions) + lowV := lexver.Parse(low) + highV := lexver.Parse(high) + + // Collect all live files in the window. + liveFiles = make(map[string]bool) + liveInWindow := make(map[string]bool) + for _, v := range liveVersions { + pv := lexver.Parse(v) + if lexver.Compare(pv, lowV) >= 0 && lexver.Compare(pv, highV) <= 0 { + liveInWindow[v] = true + for f := range liveVF[v] { + liveFiles[f] = true + } + } + } + + // For Go, only include versions that Node.js also has in the window. + // Go-only versions are hidden (deeper history, not gaps). + goFiles = make(map[string]bool) + for _, v := range goVersions { + if !liveInWindow[v] { + continue + } + for f := range goVF[v] { + goFiles[f] = true + } + } + } else { + // Compare all versions — use pre-filtered version maps. + if live != nil { + liveFiles = make(map[string]bool) + for _, files := range liveVF { + for f := range files { + liveFiles[f] = true + } + } + } + if goCache != nil { + goFiles = make(map[string]bool) + for _, files := range goVF { + for f := range files { + goFiles[f] = true + } + } + } + } + + if liveFiles == nil { + liveFiles = make(map[string]bool) + } + if goFiles == nil { + goFiles = make(map[string]bool) + } + + for f := range liveFiles { + if !goFiles[f] { + d.OnlyInLive = append(d.OnlyInLive, f) + } + } + for f := range goFiles { + if !liveFiles[f] { + d.OnlyInGo = append(d.OnlyInGo, f) + } + } + sort.Strings(d.OnlyInLive) + sort.Strings(d.OnlyInGo) + + // Field-level comparison on assets that exist in both caches. + // Build version+filename → fields maps from each cache. + if live != nil && goCache != nil { + type assetKey struct { + version string + filename string + } + liveByKey := make(map[assetKey]cacheRelease) + for _, r := range live.Releases { + name := effectiveName(r.Name, r.Filename, r.Download) + ver := normVersion(r.Version) + liveByKey[assetKey{ver, name}] = r + } + + for _, r := range goCache.Releases { + name := effectiveName(r.Name, r.Filename, r.Download) + ver := normVersion(r.Version) + lr, ok := liveByKey[assetKey{ver, name}] + if !ok { + continue + } + + // Compare classification fields. + // Use equivalence checks for os/arch/ext so naming + // convention differences don't mask real classification bugs. + for _, cmp := range []struct { + field string + live string + go_ string + equiv bool + }{ + {"os", lr.OS, r.OS, equivOS(lr.OS, r.OS)}, + {"arch", lr.Arch, r.Arch, equivArch(lr.Arch, r.Arch)}, + {"libc", lr.Libc, r.Libc, lr.Libc == r.Libc}, + {"ext", lr.Ext, r.Ext, equivExt(lr.Ext, r.Ext)}, + {"channel", lr.Channel, r.Channel, lr.Channel == r.Channel}, + } { + if cmp.equiv { + continue + } + d.FieldDiffs = append(d.FieldDiffs, fieldDiff{ + Filename: name, + Field: cmp.field, + Live: cmp.live, + Go: cmp.go_, + BothSet: cmp.live != "" && cmp.go_ != "", + }) + } + } + sort.Slice(d.FieldDiffs, func(i, j int) bool { + if d.FieldDiffs[i].Field != d.FieldDiffs[j].Field { + return d.FieldDiffs[i].Field < d.FieldDiffs[j].Field + } + return d.FieldDiffs[i].Filename < d.FieldDiffs[j].Filename + }) + } + + return d +} + +// equivOS returns true if two OS values are equivalent across naming conventions. +func equivOS(a, b string) bool { + return a == b || canonicalOS(a) == canonicalOS(b) +} + +func canonicalOS(s string) string { + switch strings.ToLower(s) { + case "darwin", "macos", "mac", "osx": + return "darwin" + case "win", "windows": + return "windows" + default: + return strings.ToLower(s) + } +} + +// equivArch returns true if two arch values are equivalent. +func equivArch(a, b string) bool { + return a == b || canonicalArch(a) == canonicalArch(b) +} + +func canonicalArch(s string) string { + switch strings.ToLower(s) { + case "x86_64", "amd64", "x64": + return "x86_64" + case "aarch64", "arm64": + return "aarch64" + case "armv7", "armv7l": + return "armv7" + case "armv6", "armv6l": + return "armv6" + case "x86", "i386", "i686", "386": + return "x86" + default: + return strings.ToLower(s) + } +} + +// equivExt returns true if two extension values are equivalent. +func equivExt(a, b string) bool { + // Normalize: strip leading dot, handle common aliases. + return a == b || canonicalExt(a) == canonicalExt(b) +} + +func canonicalExt(s string) string { + s = strings.TrimPrefix(s, ".") + switch s { + case "tgz": + return "tar.gz" + default: + return s + } +} + +func categorize(d *packageDiff) { + if d.GoMissing { + d.Categories = append(d.Categories, "go-missing") + return + } + if d.LiveMissing { + d.Categories = append(d.Categories, "live-missing") + return + } + + if len(d.OnlyInLive) == 0 && len(d.OnlyInGo) == 0 && len(d.FieldDiffs) == 0 { + d.Categories = append(d.Categories, "match") + return + } + if len(d.OnlyInLive) == 0 && len(d.OnlyInGo) == 0 && len(d.FieldDiffs) > 0 { + d.Categories = append(d.Categories, "fields-only") + } + + // Check if differences are only version depth (Go has more history). + liveVersionSet := make(map[string]bool, len(d.VersionsLive)) + for _, v := range d.VersionsLive { + liveVersionSet[v] = true + } + goVersionSet := make(map[string]bool, len(d.VersionsGo)) + for _, v := range d.VersionsGo { + goVersionSet[v] = true + } + + goExtraVersions := 0 + for _, v := range d.VersionsGo { + if !liveVersionSet[v] { + goExtraVersions++ + } + } + liveExtraVersions := 0 + for _, v := range d.VersionsLive { + if !goVersionSet[v] { + liveExtraVersions++ + } + } + + if goExtraVersions > 0 { + d.Categories = append(d.Categories, fmt.Sprintf("go-extra-versions(%d)", goExtraVersions)) + } + if liveExtraVersions > 0 { + d.Categories = append(d.Categories, fmt.Sprintf("live-extra-versions(%d)", liveExtraVersions)) + } + + // Check for meta-asset filtering differences. + metaOnlyInLive := 0 + nonMetaOnlyInLive := 0 + for _, f := range d.OnlyInLive { + if classify.IsMetaAsset(f) { + metaOnlyInLive++ + } else { + nonMetaOnlyInLive++ + } + } + metaOnlyInGo := 0 + nonMetaOnlyInGo := 0 + for _, f := range d.OnlyInGo { + if classify.IsMetaAsset(f) { + metaOnlyInGo++ + } else { + nonMetaOnlyInGo++ + } + } + + if metaOnlyInLive > 0 { + d.Categories = append(d.Categories, fmt.Sprintf("live-has-meta(%d)", metaOnlyInLive)) + } + if metaOnlyInGo > 0 { + d.Categories = append(d.Categories, fmt.Sprintf("go-has-meta(%d)", metaOnlyInGo)) + } + + // Check for source tarball differences. + srcOnlyInGo := 0 + for _, f := range d.OnlyInGo { + if strings.HasSuffix(f, ".tar.gz") || strings.HasSuffix(f, ".zip") { + if strings.HasPrefix(f, "v") || strings.HasPrefix(f, "refs/") { + srcOnlyInGo++ + } + } + } + if srcOnlyInGo > 0 { + d.Categories = append(d.Categories, fmt.Sprintf("go-has-source-tarballs(%d)", srcOnlyInGo)) + } + + if nonMetaOnlyInLive > 0 { + d.Categories = append(d.Categories, fmt.Sprintf("live-extra-assets(%d)", nonMetaOnlyInLive)) + } + if nonMetaOnlyInGo > 0 { + d.Categories = append(d.Categories, fmt.Sprintf("go-extra-assets(%d)", nonMetaOnlyInGo)) + } + + // Count field diffs by field name, separating real disagreements + // from expected "live empty, Go classified" differences. + type fieldCount struct { + bothSet int // both caches have a value but they disagree + oneEmpty int // one side is empty (typically live — normalize.js fills at serve time) + } + fieldCounts := make(map[string]fieldCount) + for _, fd := range d.FieldDiffs { + fc := fieldCounts[fd.Field] + if fd.BothSet { + fc.bothSet++ + } else { + fc.oneEmpty++ + } + fieldCounts[fd.Field] = fc + } + for _, field := range []string{"os", "arch", "libc", "ext", "channel"} { + fc := fieldCounts[field] + if fc.bothSet > 0 { + d.Categories = append(d.Categories, fmt.Sprintf("diff-%s(%d)", field, fc.bothSet)) + } + if fc.oneEmpty > 0 { + d.Categories = append(d.Categories, fmt.Sprintf("fill-%s(%d)", field, fc.oneEmpty)) + } + } +} + +// isLiveNoise returns true for filenames that the Node.js cache keeps +// but Go intentionally filters out. Pre-filtering these from the live +// side prevents them from appearing as live-extra-assets noise. +// +// This includes everything classify.IsMetaAsset catches plus formats +// that Go's legacy export strips (.deb, .rpm, etc.). +func isLiveNoise(name string) bool { + if classify.IsMetaAsset(name) { + return true + } + + lower := strings.ToLower(name) + + // Formats Go filters from legacy export but Node.js keeps. + for _, suffix := range []string{ + ".deb", ".rpm", ".gpg", + } { + if strings.HasSuffix(lower, suffix) { + return true + } + } + + // Source tarballs (e.g. gitea-src-1.25.4.tar.gz, caddy_2.10.0_src.tar.gz, go1.26.1.src.tar.gz). + if strings.Contains(lower, "-src-") || strings.Contains(lower, "_src.") || strings.Contains(lower, ".src.") || strings.HasPrefix(lower, "src-") { + return true + } + + // Docs tarballs (e.g. gitea-docs-1.22.3.tar.gz). + if strings.Contains(lower, "-docs-") { + return true + } + + // Bare executables without any extension — typically legacy shell scripts + // uploaded alongside proper archives (e.g. kubectx, kubens). + if !strings.Contains(name, ".") { + return true + } + + // GPU accelerator / hardware variants that Go tags as variants + // but Node.js keeps with special arch names. + for _, v := range []string{"-rocm", "-jetpack"} { + if strings.Contains(lower, v) { + return true + } + } + + // Linux binaries for packages where Node.js only kept macOS .app.zip. + // Go correctly includes these as installable on Linux. + if strings.HasPrefix(lower, "fish-") && strings.Contains(lower, "-linux-") { + return true + } + + return false +} + +// normalizeVersionFunc returns a version normalizer for a given package. +// Most packages return the identity function. Some (like git) need +// version string normalization to match across Go and Node.js caches. +func normalizeVersionFunc(pkg string) func(string) string { + switch pkg { + case "git": + return func(v string) string { + // Git for Windows: v2.53.0.windows.1 → v2.53.0 + // v2.53.0.windows.2 → v2.53.0.2 + idx := strings.Index(v, ".windows.") + if idx < 0 { + return v + } + suffix := v[idx+len(".windows."):] + base := v[:idx] + if suffix == "1" { + return base + } + return base + "." + suffix + } + case "lf": + return func(v string) string { + // lf: r21 → 0.21.0 + if strings.HasPrefix(v, "r") { + return "0." + v[1:] + ".0" + } + return v + } + case "bun": + return func(v string) string { + // bun: bun-v1.3.9 → v1.3.9 + return strings.TrimPrefix(v, "bun-") + } + case "watchexec": + return func(v string) string { + // watchexec monorepo: cli-v1.20.5 → v1.20.5 + return strings.TrimPrefix(v, "cli-") + } + case "go": + return func(v string) string { + // Go: go1.10 → 1.10.0 (pad to 3 parts) + v = strings.TrimPrefix(v, "go") + parts := strings.SplitN(v, ".", 3) + for len(parts) < 3 { + parts = append(parts, "0") + } + return strings.Join(parts, ".") + } + default: + return func(v string) string { return v } + } +} + +func printSummary(diffs []packageDiff) { + // Count by category. + categoryCounts := make(map[string]int) + for _, d := range diffs { + for _, c := range d.Categories { + // Strip the count suffix for grouping. + base := c + if idx := strings.Index(c, "("); idx != -1 { + base = c[:idx] + } + categoryCounts[base]++ + } + } + + fmt.Println("=== COMPARISON SUMMARY ===") + fmt.Printf("Total packages: %d\n\n", len(diffs)) + + var cats []string + for c := range categoryCounts { + cats = append(cats, c) + } + sort.Strings(cats) + for _, c := range cats { + fmt.Printf(" %-30s %d\n", c, categoryCounts[c]) + } + + fmt.Println("\n=== PER-PACKAGE CATEGORIES ===") + for _, d := range diffs { + fmt.Printf("%-25s %s\n", d.Name, strings.Join(d.Categories, ", ")) + } +} + +func printDetails(diffs []packageDiff, diffsOnly bool, sampleN int) { + for _, d := range diffs { + if diffsOnly && len(d.OnlyInLive) == 0 && len(d.OnlyInGo) == 0 && len(d.FieldDiffs) == 0 { + continue + } + + fmt.Printf("=== %s ===\n", d.Name) + fmt.Printf(" Categories: %s\n", strings.Join(d.Categories, ", ")) + fmt.Printf(" Live: %d assets, %d versions | Go: %d assets, %d versions\n", + d.LiveCount, len(d.VersionsLive), d.GoCount, len(d.VersionsGo)) + + printAssetList("Only in LIVE", d.OnlyInLive, sampleN) + printAssetList("Only in Go", d.OnlyInGo, sampleN) + printFieldDiffs(d.FieldDiffs, sampleN) + + fmt.Println() + } +} + +// printFieldDiffs shows classification differences on shared assets. +// Shows "real" diffs (both sides non-empty) first, then "fill" diffs +// (one side empty) as a summary count only. +func printFieldDiffs(diffs []fieldDiff, sampleN int) { + if len(diffs) == 0 { + return + } + + // Separate real disagreements from fill diffs. + var real, fill []fieldDiff + for _, fd := range diffs { + if fd.BothSet { + real = append(real, fd) + } else { + fill = append(fill, fd) + } + } + + // Show real disagreements in detail. + if len(real) > 0 { + byField := make(map[string][]fieldDiff) + for _, fd := range real { + byField[fd.Field] = append(byField[fd.Field], fd) + } + + for _, field := range []string{"os", "arch", "libc", "ext", "channel"} { + fds := byField[field] + if len(fds) == 0 { + continue + } + + fmt.Printf(" DISAGREE %s (%d):\n", field, len(fds)) + printFieldDiffItems(fds, sampleN) + } + } + + // Summarize fill diffs (live empty, Go classified) as counts. + if len(fill) > 0 { + byField := make(map[string]int) + for _, fd := range fill { + byField[fd.Field]++ + } + var parts []string + for _, field := range []string{"os", "arch", "libc", "ext", "channel"} { + if n := byField[field]; n > 0 { + parts = append(parts, fmt.Sprintf("%s(%d)", field, n)) + } + } + if len(parts) > 0 { + fmt.Printf(" Go fills empty: %s\n", strings.Join(parts, ", ")) + } + } +} + +func printFieldDiffItems(fds []fieldDiff, sampleN int) { + items := fds + if sampleN > 0 && len(items) > sampleN { + sampled := make([]fieldDiff, len(items)) + copy(sampled, items) + rand.Shuffle(len(sampled), func(i, j int) { + sampled[i], sampled[j] = sampled[j], sampled[i] + }) + items = sampled[:sampleN] + sort.Slice(items, func(i, j int) bool { + return items[i].Filename < items[j].Filename + }) + } + + limit := 20 + for i, fd := range items { + if sampleN == 0 && i >= limit { + fmt.Printf(" ... and %d more\n", len(fds)-limit) + break + } + fmt.Printf(" - %s: live=%q go=%q\n", fd.Filename, fd.Live, fd.Go) + } + if sampleN > 0 && len(fds) > sampleN { + fmt.Printf(" ... sampled %d of %d\n", sampleN, len(fds)) + } +} + +// printAssetList prints a list of asset filenames, optionally sampling N at +// random. When sampleN > 0 and the list is longer, it picks N random items +// so you can spot classification bugs across the full range instead of only +// seeing the first alphabetical entries. +func printAssetList(label string, items []string, sampleN int) { + if len(items) == 0 { + return + } + + fmt.Printf(" %s (%d):\n", label, len(items)) + + if sampleN > 0 && len(items) > sampleN { + // Shuffle a copy, take first N, then sort for readable output. + sampled := make([]string, len(items)) + copy(sampled, items) + rand.Shuffle(len(sampled), func(i, j int) { + sampled[i], sampled[j] = sampled[j], sampled[i] + }) + picked := sampled[:sampleN] + sort.Strings(picked) + for _, f := range picked { + fmt.Printf(" - %s\n", f) + } + fmt.Printf(" ... sampled %d of %d (run again for different sample)\n", sampleN, len(items)) + return + } + + limit := 20 + for i, f := range items { + if i >= limit { + fmt.Printf(" ... and %d more\n", len(items)-limit) + break + } + fmt.Printf(" - %s\n", f) + } +} diff --git a/cmd/e2etest/main.go b/cmd/e2etest/main.go new file mode 100644 index 0000000..6d39430 --- /dev/null +++ b/cmd/e2etest/main.go @@ -0,0 +1,846 @@ +// Command e2etest runs the full release pipeline for selected packages +// and compares results against the live webi.sh API. +// +// It fetches from upstream, classifies assets, resolves the best match +// for a set of test queries, then fetches the same queries from the live +// API and reports any differences. +// +// Usage: +// +// go run ./cmd/e2etest +// go run ./cmd/e2etest -packages goreleaser,ollama,node +// go run ./cmd/e2etest -cache ./_cache/raw # reuse existing cache +package main + +import ( + "bufio" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/webinstall/webi-installers/internal/buildmeta" + "github.com/webinstall/webi-installers/internal/installerconf" + "github.com/webinstall/webi-installers/internal/lexver" + "github.com/webinstall/webi-installers/internal/rawcache" + "github.com/webinstall/webi-installers/internal/releases/github" + "github.com/webinstall/webi-installers/internal/releases/githubish" + "github.com/webinstall/webi-installers/internal/releases/nodedist" + "github.com/webinstall/webi-installers/internal/resolve" +) + +// testCase is one query to resolve and compare against the live API. +type testCase struct { + Name string + Package string + OS buildmeta.OS + Arch buildmeta.Arch + Libc buildmeta.Libc + Formats []string + UA string // User-Agent for live API query +} + +// liveResult holds parsed fields from the live webi API response. +type liveResult struct { + Version string + OS string + Arch string + Libc string + Ext string + PkgURL string + PkgFile string + Channel string + Stable string + Latest string + Oses string + Arches string + Libcs string + Formats string +} + +// UA format from webi.sh bootstrap: "curl {uname -s}/{uname -r} {uname -m}/unknown {libc}" +// libc is "gnu", "musl", or "libc" (for darwin/other) +var cases = []testCase{ + { + Name: "goreleaser/linux/x86_64", Package: "goreleaser", + OS: buildmeta.OSLinux, Arch: buildmeta.ArchAMD64, Libc: buildmeta.LibcGNU, + Formats: []string{".tar.gz", ".tar.xz", ".zip"}, + UA: "curl Linux/6.6.123 x86_64/unknown gnu", + }, + { + Name: "goreleaser/darwin/arm64", Package: "goreleaser", + OS: buildmeta.OSDarwin, Arch: buildmeta.ArchARM64, Libc: "", + Formats: []string{".tar.gz", ".tar.xz", ".zip"}, + UA: "curl Darwin/25.2.0 arm64/unknown libc", + }, + { + Name: "goreleaser/windows/x86_64", Package: "goreleaser", + OS: buildmeta.OSWindows, Arch: buildmeta.ArchAMD64, Libc: "", + Formats: []string{".zip", ".exe"}, + UA: "PowerShell/7.0 Windows/10.0 x86_64/unknown msvc", + }, + { + Name: "ollama/linux/x86_64", Package: "ollama", + OS: buildmeta.OSLinux, Arch: buildmeta.ArchAMD64, Libc: buildmeta.LibcGNU, + Formats: []string{".tar.gz", ".tar.xz", ".tar.zst", ".zip"}, + UA: "curl Linux/6.6.123 x86_64/unknown gnu", + }, + { + Name: "ollama/darwin/arm64", Package: "ollama", + OS: buildmeta.OSDarwin, Arch: buildmeta.ArchARM64, Libc: "", + Formats: []string{".tar.gz", ".tar.xz", ".tar.zst", ".zip", ".dmg"}, + UA: "curl Darwin/25.2.0 arm64/unknown libc", + }, + { + Name: "ollama/linux/arm64", Package: "ollama", + OS: buildmeta.OSLinux, Arch: buildmeta.ArchARM64, Libc: buildmeta.LibcGNU, + Formats: []string{".tar.gz", ".tar.xz", ".tar.zst", ".zip"}, + UA: "curl Linux/6.6.123 aarch64/unknown gnu", + }, + { + Name: "node/linux/x86_64", Package: "node", + OS: buildmeta.OSLinux, Arch: buildmeta.ArchAMD64, Libc: buildmeta.LibcGNU, + Formats: []string{".tar.xz", ".tar.gz", ".zip"}, + UA: "curl Linux/6.6.123 x86_64/unknown gnu", + }, + { + Name: "node/darwin/arm64", Package: "node", + OS: buildmeta.OSDarwin, Arch: buildmeta.ArchARM64, Libc: "", + Formats: []string{".tar.xz", ".tar.gz", ".zip"}, + UA: "curl Darwin/25.2.0 arm64/unknown libc", + }, + { + Name: "node/linux/arm64", Package: "node", + OS: buildmeta.OSLinux, Arch: buildmeta.ArchARM64, Libc: buildmeta.LibcGNU, + Formats: []string{".tar.xz", ".tar.gz", ".zip"}, + UA: "curl Linux/6.6.123 aarch64/unknown gnu", + }, +} + +func main() { + cacheDir := flag.String("cache", "_cache/raw", "root directory for raw cache") + confDir := flag.String("conf", ".", "root directory containing {pkg}/releases.conf files") + token := flag.String("token", os.Getenv("GITHUB_TOKEN"), "GitHub API token") + skipFetch := flag.Bool("skip-fetch", false, "skip fetching, use existing cache") + skipLive := flag.Bool("skip-live", false, "skip live API comparison") + packages := flag.String("packages", "goreleaser,ollama,node", "comma-separated packages to test") + flag.Parse() + + pkgList := strings.Split(*packages, ",") + pkgSet := make(map[string]bool, len(pkgList)) + for _, p := range pkgList { + pkgSet[strings.TrimSpace(p)] = true + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + client := &http.Client{Timeout: 30 * time.Second} + var auth *githubish.Auth + if *token != "" { + auth = &githubish.Auth{Token: *token} + } + + // Step 1: Fetch raw releases. + if !*skipFetch { + log.Println("=== Step 1: Fetching releases ===") + for _, pkg := range pkgList { + if err := fetchPackage(ctx, client, *cacheDir, *confDir, pkg, auth); err != nil { + log.Fatalf("fetch %s: %v", pkg, err) + } + } + } else { + log.Println("=== Step 1: Skipping fetch (using cache) ===") + } + + // Step 2: Classify releases. + log.Println("=== Step 2: Classifying releases ===") + allDists := make(map[string][]resolve.Dist) + for _, pkg := range pkgList { + conf, err := installerconf.Read(filepath.Join(*confDir, pkg, "releases.conf")) + if err != nil { + log.Fatalf("read conf %s: %v", pkg, err) + } + d, err := rawcache.Open(filepath.Join(*cacheDir, pkg)) + if err != nil { + log.Fatalf("open cache %s: %v", pkg, err) + } + dists, err := classifyFromCache(pkg, conf, d) + if err != nil { + log.Fatalf("classify %s: %v", pkg, err) + } + allDists[pkg] = dists + log.Printf(" %s: %d distributables", pkg, len(dists)) + + // Show catalog. + cat := resolve.Survey(dists) + log.Printf(" oses=%v arches=%v libcs=%v formats=%v", cat.OSes, cat.Arches, cat.Libcs, cat.Formats) + log.Printf(" latest=%s stable=%s", cat.Latest, cat.Stable) + } + + // Step 3: Resolve best match for each test case. + log.Println("=== Step 3: Resolving best matches ===") + type result struct { + tc testCase + match *resolve.Match + live *liveResult + } + var results []result + for _, tc := range cases { + if !pkgSet[tc.Package] { + continue + } + dists := allDists[tc.Package] + q := resolve.Query{ + OS: tc.OS, + Arch: tc.Arch, + Libc: tc.Libc, + Formats: tc.Formats, + Channel: "stable", + } + m := resolve.Best(dists, q) + results = append(results, result{tc: tc, match: m}) + } + + // Step 4: Compare with live API. + if !*skipLive { + log.Println("=== Step 4: Comparing with live API ===") + for i := range results { + tc := results[i].tc + live, err := queryLiveAPI(client, tc) + if err != nil { + log.Printf(" %s: live API error: %v", tc.Name, err) + continue + } + results[i].live = live + } + } + + // Step 5: Report. + log.Println("") + log.Println("=== Results ===") + log.Println("") + + pass, fail, warn := 0, 0, 0 + for _, r := range results { + tc := r.tc + m := r.match + live := r.live + + if m == nil { + log.Printf("FAIL %s: no match found", tc.Name) + fail++ + continue + } + + log.Printf("--- %s ---", tc.Name) + log.Printf(" Go: version=%s file=%s ext=%s url=%s", m.Version, m.Filename, m.Format, m.Download) + + if live != nil { + log.Printf(" Live: version=%s file=%s ext=%s url=%s", live.Version, live.PkgFile, live.Ext, live.PkgURL) + + if live.Version == "0.0.0" { + log.Printf(" WARN: live API returned error (no match)") + warn++ + } else if m.Version == live.Version && m.Filename == live.PkgFile { + log.Printf(" PASS: exact match") + pass++ + } else if m.Version == live.Version && m.Download == live.PkgURL { + log.Printf(" PASS: same URL (filename display differs: go=%s live=%s)", m.Filename, live.PkgFile) + pass++ + } else if m.Version == live.Version { + log.Printf(" WARN: same version, different file (go=%s live=%s)", m.Filename, live.PkgFile) + warn++ + } else { + log.Printf(" DIFF: version mismatch (go=%s live=%s)", m.Version, live.Version) + fail++ + } + } else { + log.Printf(" (no live comparison)") + pass++ + } + } + + log.Println("") + log.Printf("Summary: %d pass, %d fail, %d warn (live API errors)", pass, fail, warn) + + if fail > 0 { + os.Exit(1) + } +} + +// fetchPackage fetches raw releases for one package. +func fetchPackage(ctx context.Context, client *http.Client, cacheRoot, confDir, pkg string, auth *githubish.Auth) error { + conf, err := installerconf.Read(filepath.Join(confDir, pkg, "releases.conf")) + if err != nil { + return fmt.Errorf("read conf: %w", err) + } + + source := conf.Source + log.Printf(" %s: source=%s", pkg, source) + + switch source { + case "github": + return fetchGitHub(ctx, client, cacheRoot, pkg, conf, auth) + case "nodedist": + return fetchNodeDist(ctx, client, cacheRoot, pkg, conf) + default: + return fmt.Errorf("unsupported source %q (only github and nodedist for e2e test)", source) + } +} + +func fetchGitHub(ctx context.Context, client *http.Client, cacheRoot, pkg string, conf *installerconf.Conf, auth *githubish.Auth) error { + owner := conf.Owner + repo := conf.Repo + tagPrefix := conf.TagPrefix + + d, err := rawcache.Open(filepath.Join(cacheRoot, pkg)) + if err != nil { + return err + } + + var added, changed, skipped int + var latest string + for batch, err := range github.Fetch(ctx, client, owner, repo, auth) { + if err != nil { + return fmt.Errorf("github %s/%s: %w", owner, repo, err) + } + for _, rel := range batch { + if rel.Draft { + continue + } + + tag := rel.TagName + if tagPrefix != "" { + if !strings.HasPrefix(tag, tagPrefix) { + continue + } + tag = strings.TrimPrefix(tag, tagPrefix) + } + + data, err := json.Marshal(rel) + if err != nil { + return fmt.Errorf("marshal %s: %w", tag, err) + } + + action, err := d.Merge(tag, data) + if err != nil { + return err + } + switch action { + case "added": + added++ + case "changed": + changed++ + default: + skipped++ + } + + if latest == "" && !rel.Prerelease { + latest = tag + } + } + } + + if latest != "" { + current := d.Latest() + if current == "" || lexver.Compare(lexver.Parse(latest), lexver.Parse(current)) > 0 { + d.SetLatest(latest) + } + } + + log.Printf(" +%d ~%d =%d latest=%s", added, changed, skipped, d.Latest()) + return nil +} + +func fetchNodeDist(ctx context.Context, client *http.Client, cacheRoot, pkg string, conf *installerconf.Conf) error { + baseURL := conf.BaseURL + d, err := rawcache.Open(filepath.Join(cacheRoot, pkg)) + if err != nil { + return err + } + + var added, changed, skipped int + var latest string + for batch, err := range nodedist.Fetch(ctx, client, baseURL) { + if err != nil { + return fmt.Errorf("nodedist: %w", err) + } + for _, entry := range batch { + tag := entry.Version + data, err := json.Marshal(entry) + if err != nil { + return fmt.Errorf("marshal %s: %w", tag, err) + } + + action, err := d.Merge(tag, data) + if err != nil { + return err + } + switch action { + case "added": + added++ + case "changed": + changed++ + default: + skipped++ + } + + if latest == "" { + latest = tag + } + } + } + + if latest != "" { + current := d.Latest() + if current == "" || lexver.Compare(lexver.Parse(latest), lexver.Parse(current)) > 0 { + d.SetLatest(latest) + } + } + + log.Printf(" +%d ~%d =%d latest=%s", added, changed, skipped, d.Latest()) + return nil +} + +// classifyFromCache reads the raw cache and produces classified dists. +func classifyFromCache(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]resolve.Dist, error) { + source := conf.Source + switch source { + case "github": + return classifyGitHub(pkg, conf, d) + case "nodedist": + return classifyNodeDist(pkg, conf, d) + default: + return nil, fmt.Errorf("unsupported source %q", source) + } +} + +func classifyGitHub(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]resolve.Dist, error) { + tagPrefix := conf.TagPrefix + releases, err := readAllReleases(d) + if err != nil { + return nil, err + } + + var dists []resolve.Dist + for _, data := range releases { + var rel struct { + TagName string `json:"tag_name"` + Prerelease bool `json:"prerelease"` + Draft bool `json:"draft"` + PublishedAt string `json:"published_at"` + Assets []struct { + Name string `json:"name"` + BrowserDownloadURL string `json:"browser_download_url"` + Size int64 `json:"size"` + } `json:"assets"` + } + if err := json.Unmarshal(data, &rel); err != nil { + continue + } + if rel.Draft { + continue + } + + version := rel.TagName + if tagPrefix != "" { + version = strings.TrimPrefix(version, tagPrefix) + } + // Strip leading "v" for version normalization. + version = strings.TrimPrefix(version, "v") + + channel := "stable" + if rel.Prerelease { + channel = "beta" + } + + date := "" + if len(rel.PublishedAt) >= 10 { + date = rel.PublishedAt[:10] + } + + for _, asset := range rel.Assets { + if isMetaAsset(asset.Name) { + continue + } + + r := classifyFilename(asset.Name) + extra := detectExtra(asset.Name) + dists = append(dists, resolve.Dist{ + Package: pkg, + Version: version, + Channel: channel, + OS: r.os, + Arch: r.arch, + Libc: r.libc, + Format: r.format, + Download: asset.BrowserDownloadURL, + Filename: asset.Name, + Size: asset.Size, + Date: date, + Extra: extra, + }) + } + } + return dists, nil +} + +func classifyNodeDist(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]resolve.Dist, error) { + baseURL := conf.BaseURL + releases, err := readAllReleases(d) + if err != nil { + return nil, err + } + + var dists []resolve.Dist + for _, data := range releases { + var entry struct { + Version string `json:"version"` + Date string `json:"date"` + Files []string `json:"files"` + LTS json.RawMessage `json:"lts"` + Security bool `json:"security"` + } + if err := json.Unmarshal(data, &entry); err != nil { + continue + } + + lts := string(entry.LTS) != "false" && string(entry.LTS) != "" + version := strings.TrimPrefix(entry.Version, "v") + + // Webi treats even major versions as "stable" (LTS-eligible). + channel := "stable" + parts := strings.SplitN(version, ".", 2) + if len(parts) > 0 { + var major int + fmt.Sscanf(parts[0], "%d", &major) + if major%2 != 0 { + channel = "beta" + } + } + + for _, file := range entry.Files { + if file == "src" || file == "headers" { + continue + } + fileDists := expandNodeFile(pkg, entry.Version, version, channel, entry.Date, lts, baseURL, file) + dists = append(dists, fileDists...) + } + } + return dists, nil +} + +func expandNodeFile(pkg, rawVersion, version, channel, date string, lts bool, baseURL, file string) []resolve.Dist { + parts := strings.Split(file, "-") + if len(parts) < 2 { + return nil + } + + osMap := map[string]string{ + "osx": "darwin", "linux": "linux", "win": "windows", + "sunos": "sunos", "aix": "aix", + } + archMap := map[string]string{ + "x64": "x86_64", "x86": "x86", "arm64": "aarch64", + "armv7l": "armv7", "armv6l": "armv6", + "ppc64": "ppc64", "ppc64le": "ppc64le", "s390x": "s390x", + "loong64": "loong64", "riscv64": "riscv64", + } + + os_ := osMap[parts[0]] + arch := archMap[parts[1]] + if os_ == "" || arch == "" { + return nil + } + + libc := "" + pkgType := "" + if len(parts) > 2 { + pkgType = parts[2] + } + + var formats []string + switch pkgType { + case "musl": + libc = "musl" + formats = []string{".tar.gz", ".tar.xz"} + case "tar": + formats = []string{".tar.gz", ".tar.xz"} + case "zip": + formats = []string{".zip"} + case "7z": + formats = []string{".7z"} + case "pkg": + formats = []string{".pkg"} + case "msi": + formats = []string{".msi"} + case "exe": + formats = []string{".exe"} + case "": + formats = []string{".tar.gz", ".tar.xz"} + default: + return nil + } + + if libc == "" && os_ == "linux" { + libc = "gnu" + } + + osPart := parts[0] + if osPart == "osx" { + osPart = "darwin" + } + archPart := parts[1] + muslExtra := "" + if libc == "musl" { + muslExtra = "-musl" + } + + var dists []resolve.Dist + for _, format := range formats { + var filename string + if format == ".msi" { + filename = fmt.Sprintf("node-%s-%s%s%s", rawVersion, archPart, muslExtra, format) + } else { + filename = fmt.Sprintf("node-%s-%s-%s%s%s", rawVersion, osPart, archPart, muslExtra, format) + } + + dists = append(dists, resolve.Dist{ + Package: pkg, + Version: version, + Channel: channel, + OS: os_, + Arch: arch, + Libc: libc, + Format: format, + Download: fmt.Sprintf("%s/%s/%s", baseURL, rawVersion, filename), + Filename: filename, + LTS: lts, + Date: date, + }) + } + return dists +} + +// queryLiveAPI queries the live webi.sh API and parses the response header. +func queryLiveAPI(client *http.Client, tc testCase) (*liveResult, error) { + // Build format string matching what the webi.sh bootstrap sends. + // Order: tar,exe,zip,xz,dmg,git (least to most favorable in bootstrap, + // but the API doesn't care about order). + fmtParam := "tar,exe,zip,xz,dmg" + + url := fmt.Sprintf("https://webi.sh/api/installers/%s@stable.sh?formats=%s", tc.Package, fmtParam) + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", tc.UA) + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return parseLiveResponse(string(body)), nil +} + +// parseLiveResponse extracts WEBI_* and PKG_* variables from the shell script. +func parseLiveResponse(body string) *liveResult { + vars := make(map[string]string) + scanner := bufio.NewScanner(strings.NewReader(body)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + for _, prefix := range []string{"WEBI_", "PKG_"} { + if strings.HasPrefix(line, prefix) { + if eq := strings.IndexByte(line, '='); eq > 0 { + key := line[:eq] + val := line[eq+1:] + val = strings.Trim(val, "'\"") + vars[key] = val + } + } + } + } + + return &liveResult{ + Version: vars["WEBI_VERSION"], + OS: vars["WEBI_OS"], + Arch: vars["WEBI_ARCH"], + Libc: vars["WEBI_LIBC"], + Ext: vars["WEBI_EXT"], + PkgURL: vars["WEBI_PKG_URL"], + PkgFile: vars["WEBI_PKG_FILE"], + Channel: vars["WEBI_CHANNEL"], + Stable: vars["PKG_STABLE"], + Latest: vars["PKG_LATEST"], + Oses: vars["PKG_OSES"], + Arches: vars["PKG_ARCHES"], + Libcs: vars["PKG_LIBCS"], + Formats: vars["PKG_FORMATS"], + } +} + +// readAllReleases reads all cached release files. +func readAllReleases(d *rawcache.Dir) (map[string][]byte, error) { + active, err := d.ActivePath() + if err != nil { + return nil, err + } + entries, err := os.ReadDir(active) + if err != nil { + return nil, err + } + result := make(map[string][]byte, len(entries)) + for _, e := range entries { + if e.IsDir() || strings.HasPrefix(e.Name(), "_") { + continue + } + data, err := os.ReadFile(filepath.Join(active, e.Name())) + if err != nil { + return nil, err + } + result[e.Name()] = data + } + return result, nil +} + +type classResult struct { + os, arch, libc, format string +} + +func classifyFilename(name string) classResult { + // Use the classify package. + // Import it indirectly to avoid circular deps — inline the logic + // we need for the e2e test. + lower := strings.ToLower(name) + + var r classResult + r.format = detectFormat(name) + + // OS detection + switch { + case strings.Contains(lower, "linux"): + r.os = "linux" + case strings.Contains(lower, "darwin") || strings.Contains(lower, "macos") || strings.Contains(lower, "apple"): + r.os = "darwin" + case strings.Contains(lower, "windows") || strings.Contains(lower, "win64") || strings.Contains(lower, "win32"): + r.os = "windows" + case strings.HasSuffix(lower, ".dmg") || strings.HasSuffix(lower, ".app.zip"): + r.os = "darwin" + case strings.HasSuffix(lower, ".exe") || strings.HasSuffix(lower, ".msi"): + r.os = "windows" + case strings.Contains(lower, "freebsd"): + r.os = "freebsd" + } + + // Arch detection + switch { + case strings.Contains(lower, "x86_64") || strings.Contains(lower, "amd64") || strings.Contains(lower, "x64"): + r.arch = "x86_64" + case strings.Contains(lower, "aarch64") || strings.Contains(lower, "arm64"): + r.arch = "aarch64" + case strings.Contains(lower, "armv7") || strings.Contains(lower, "armhf"): + r.arch = "armv7" + case strings.Contains(lower, "armv6"): + r.arch = "armv6" + case strings.Contains(lower, "i686") || strings.Contains(lower, "i386") || strings.Contains(lower, "x86") || strings.Contains(lower, "386"): + r.arch = "x86" + case strings.Contains(lower, "ppc64le") || strings.Contains(lower, "powerpc64le"): + r.arch = "ppc64le" + case strings.Contains(lower, "ppc64") || strings.Contains(lower, "powerpc64"): + r.arch = "ppc64" + case strings.Contains(lower, "riscv64"): + r.arch = "riscv64" + case strings.Contains(lower, "s390x"): + r.arch = "s390x" + case strings.Contains(lower, "loong64"): + r.arch = "loong64" + } + + // Libc detection + switch { + case strings.Contains(lower, "musl"): + r.libc = "musl" + case strings.Contains(lower, "gnu"): + r.libc = "gnu" + case strings.Contains(lower, "msvc"): + r.libc = "msvc" + } + + return r +} + +func detectFormat(name string) string { + lower := strings.ToLower(name) + for _, ext := range []string{".tar.gz", ".tar.xz", ".tar.bz2", ".tar.zst", ".exe.xz", ".app.zip"} { + if strings.HasSuffix(lower, ext) { + return ext + } + } + // .tgz is a common alias for .tar.gz + if strings.HasSuffix(lower, ".tgz") { + return ".tar.gz" + } + return filepath.Ext(lower) +} + +// detectExtra identifies GPU/vendor-specific variant suffixes in filenames +// like "ollama-linux-amd64-rocm.tar.zst" or "ollama-linux-arm64-jetpack5.tar.zst". +func detectExtra(name string) string { + lower := strings.ToLower(name) + for _, variant := range []string{ + "-rocm", "-jetpack", "-cuda", "-vulkan", "-metal", + "-extended", "-static", "-debug", "-nightly", + } { + if strings.Contains(lower, variant) { + return strings.TrimPrefix(variant, "-") + } + } + return "" +} + +func isMetaAsset(name string) bool { + lower := strings.ToLower(name) + for _, suffix := range []string{ + ".sha256", ".sha256sum", ".sha512", ".sha512sum", + ".md5", ".md5sum", ".sig", ".asc", ".pem", + "checksums.txt", "sha256sums", "sha512sums", + ".sbom", ".spdx", ".json.sig", ".sigstore", + ".d.ts", ".pub", + } { + if strings.HasSuffix(lower, suffix) { + return true + } + } + for _, contains := range []string{ + "checksums", "sha256sum", "sha512sum", + "buildable-artifact", + } { + if strings.Contains(lower, contains) { + return true + } + } + for _, exact := range []string{ + "install.sh", "install.ps1", "compat.json", + } { + if lower == exact { + return true + } + } + return false +} diff --git a/cmd/fetchraw/main.go b/cmd/fetchraw/main.go new file mode 100644 index 0000000..37c6b56 --- /dev/null +++ b/cmd/fetchraw/main.go @@ -0,0 +1,862 @@ +// Command fetchraw fetches release histories from upstream APIs and +// merges them into rawcache. Safe to run repeatedly — unchanged releases +// are skipped, new/changed ones are recorded in the audit log. +// +// Reads releases.conf files from package directories to discover what +// to fetch. Adding a new package is just creating a conf file. +// +// Usage: +// +// go run ./cmd/fetchraw -cache ./_cache/raw +// go run ./cmd/fetchraw -cache ./_cache/raw hugo caddy +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "log" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/webinstall/webi-installers/internal/installerconf" + "github.com/webinstall/webi-installers/internal/lexver" + "github.com/webinstall/webi-installers/internal/rawcache" + "github.com/webinstall/webi-installers/internal/releases/chromedist" + "github.com/webinstall/webi-installers/internal/releases/flutterdist" + "github.com/webinstall/webi-installers/internal/releases/gitea" + "github.com/webinstall/webi-installers/internal/releases/github" + "github.com/webinstall/webi-installers/internal/releases/githubish" + "github.com/webinstall/webi-installers/internal/releases/gittag" + "github.com/webinstall/webi-installers/internal/releases/golang" + "github.com/webinstall/webi-installers/internal/releases/gpgdist" + "github.com/webinstall/webi-installers/internal/releases/hashicorp" + "github.com/webinstall/webi-installers/internal/releases/iterm2dist" + "github.com/webinstall/webi-installers/internal/releases/juliadist" + "github.com/webinstall/webi-installers/internal/releases/mariadbdist" + "github.com/webinstall/webi-installers/internal/releases/nodedist" + "github.com/webinstall/webi-installers/internal/releases/zigdist" +) + +func main() { + cacheDir := flag.String("cache", "_cache/raw", "root directory for raw cache") + confDir := flag.String("conf", ".", "root directory containing {pkg}/releases.conf files") + token := flag.String("token", os.Getenv("GITHUB_TOKEN"), "GitHub API token") + flag.Parse() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + + client := &http.Client{Timeout: 30 * time.Second} + var auth *githubish.Auth + if *token != "" { + auth = &githubish.Auth{Token: *token} + } + + // Discover packages from releases.conf files. + packages, err := discover(*confDir) + if err != nil { + log.Fatalf("discover: %v", err) + } + + // Filter to requested packages if args given. + args := flag.Args() + if len(args) > 0 { + nameSet := make(map[string]bool, len(args)) + for _, a := range args { + nameSet[a] = true + } + var filtered []pkgConf + for _, p := range packages { + if nameSet[p.name] { + filtered = append(filtered, p) + } + } + packages = filtered + } + + log.Printf("found %d packages", len(packages)) + + for _, pkg := range packages { + // Aliases share cache with their target — skip fetching. + if alias := pkg.conf.Extra["alias_of"]; alias != "" { + log.Printf(" %s: alias of %s, skipping", pkg.name, alias) + continue + } + + log.Printf("fetching %s...", pkg.name) + var err error + switch pkg.conf.Source { + case "github": + err = fetchGitHub(ctx, client, *cacheDir, pkg.name, pkg.conf, auth) + case "nodedist": + err = fetchNodeDist(ctx, client, *cacheDir, pkg.name, pkg.conf) + case "golang": + err = fetchGolang(ctx, client, *cacheDir, pkg.name) + case "zigdist": + err = fetchZig(ctx, client, *cacheDir, pkg.name) + case "flutterdist": + err = fetchFlutter(ctx, client, *cacheDir, pkg.name) + case "iterm2dist": + err = fetchITerm2(ctx, client, *cacheDir, pkg.name) + case "hashicorp": + err = fetchHashiCorp(ctx, client, *cacheDir, pkg.name, pkg.conf) + case "juliadist": + err = fetchJulia(ctx, client, *cacheDir, pkg.name) + case "gittag": + err = fetchGitTag(ctx, *cacheDir, pkg.name, pkg.conf) + case "gitea": + err = fetchGitea(ctx, client, *cacheDir, pkg.name, pkg.conf) + case "chromedist": + err = fetchChrome(ctx, client, *cacheDir, pkg.name) + case "gpgdist": + err = fetchGPG(ctx, client, *cacheDir, pkg.name) + case "mariadbdist": + err = fetchMariaDB(ctx, client, *cacheDir, pkg.name) + default: + log.Printf(" %s: unknown source %q, skipping", pkg.name, pkg.conf.Source) + continue + } + if err != nil { + log.Printf(" ERROR: %s: %v", pkg.name, err) + } + } +} + +type pkgConf struct { + name string + conf *installerconf.Conf +} + +// discover finds all {dir}/*/releases.conf files and returns them sorted. +func discover(dir string) ([]pkgConf, error) { + pattern := filepath.Join(dir, "*", "releases.conf") + matches, err := filepath.Glob(pattern) + if err != nil { + return nil, err + } + + var packages []pkgConf + for _, path := range matches { + name := filepath.Base(filepath.Dir(path)) + // Skip infrastructure dirs (_example, _webi, _common, etc.) + if strings.HasPrefix(name, "_") { + continue + } + conf, err := installerconf.Read(path) + if err != nil { + log.Printf("warning: %s: %v", path, err) + continue + } + packages = append(packages, pkgConf{name: name, conf: conf}) + } + + sort.Slice(packages, func(i, j int) bool { + return packages[i].name < packages[j].name + }) + + return packages, nil +} + +func fetchNodeDist(ctx context.Context, client *http.Client, cacheRoot, pkgName string, conf *installerconf.Conf) error { + baseURL := conf.BaseURL + if baseURL == "" { + return fmt.Errorf("missing url in releases.conf") + } + + d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName)) + if err != nil { + return err + } + + var added, changed, skipped int + var latest string + for batch, err := range nodedist.Fetch(ctx, client, baseURL) { + if err != nil { + return fmt.Errorf("%s fetch: %w", pkgName, err) + } + for _, entry := range batch { + tag := entry.Version + data, err := json.Marshal(entry) + if err != nil { + return fmt.Errorf("%s marshal %s: %w", pkgName, tag, err) + } + + action, err := d.Merge(tag, data) + if err != nil { + return err + } + switch action { + case "added": + added++ + case "changed": + changed++ + default: + skipped++ + } + + if latest == "" { + latest = tag + } + } + } + + if err := updateLatest(d, latest); err != nil { + return err + } + + log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest()) + return nil +} + +func fetchGitHub(ctx context.Context, client *http.Client, cacheRoot, pkgName string, conf *installerconf.Conf, auth *githubish.Auth) error { + owner := conf.Owner + repo := conf.Repo + tagPrefix := conf.TagPrefix + + if owner == "" || repo == "" { + return fmt.Errorf("missing owner or repo in releases.conf") + } + + d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName)) + if err != nil { + return err + } + + var added, changed, skipped int + var latest string + for batch, err := range github.Fetch(ctx, client, owner, repo, auth) { + if err != nil { + return fmt.Errorf("github %s/%s: %w", owner, repo, err) + } + for _, rel := range batch { + if rel.Draft { + continue + } + + tag := rel.TagName + + if tagPrefix != "" { + if !strings.HasPrefix(tag, tagPrefix) { + continue + } + tag = strings.TrimPrefix(tag, tagPrefix) + } + + data, err := json.Marshal(rel) + if err != nil { + return fmt.Errorf("marshal %s: %w", tag, err) + } + + action, err := d.Merge(tag, data) + if err != nil { + return err + } + switch action { + case "added": + added++ + case "changed": + changed++ + default: + skipped++ + } + + if latest == "" && !rel.Prerelease { + latest = tag + } + } + } + + if err := updateLatest(d, latest); err != nil { + return err + } + + log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest()) + return nil +} + +func updateLatest(d *rawcache.Dir, candidate string) error { + if candidate == "" { + return nil + } + current := d.Latest() + if current == "" || lexver.Compare(lexver.Parse(candidate), lexver.Parse(current)) > 0 { + return d.SetLatest(candidate) + } + return nil +} + +func fetchGolang(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error { + d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName)) + if err != nil { + return err + } + + var added, changed, skipped int + var latest string + for batch, err := range golang.Fetch(ctx, client) { + if err != nil { + return fmt.Errorf("golang: %w", err) + } + for _, rel := range batch { + tag := rel.Version // "go1.24.1" + data, err := json.Marshal(rel) + if err != nil { + return fmt.Errorf("golang marshal %s: %w", tag, err) + } + + action, err := d.Merge(tag, data) + if err != nil { + return err + } + switch action { + case "added": + added++ + case "changed": + changed++ + default: + skipped++ + } + + if latest == "" && rel.Stable { + latest = tag + } + } + } + + if err := updateLatest(d, latest); err != nil { + return err + } + + log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest()) + return nil +} + +func fetchZig(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error { + d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName)) + if err != nil { + return err + } + + var added, changed, skipped int + var latest string + for batch, err := range zigdist.Fetch(ctx, client) { + if err != nil { + return fmt.Errorf("zigdist: %w", err) + } + for _, rel := range batch { + tag := rel.Version + data, err := json.Marshal(rel) + if err != nil { + return fmt.Errorf("zigdist marshal %s: %w", tag, err) + } + + action, err := d.Merge(tag, data) + if err != nil { + return err + } + switch action { + case "added": + added++ + case "changed": + changed++ + default: + skipped++ + } + + // Stable versions have dots and no dev/pre markers. + isStable := strings.Contains(tag, ".") && !strings.ContainsAny(tag, "+-") + if isStable { + if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 { + latest = tag + } + } + } + } + + if err := updateLatest(d, latest); err != nil { + return err + } + + log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest()) + return nil +} + +func fetchFlutter(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error { + d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName)) + if err != nil { + return err + } + + var added, changed, skipped int + var latest string + for batch, err := range flutterdist.Fetch(ctx, client) { + if err != nil { + return fmt.Errorf("flutterdist: %w", err) + } + for _, rel := range batch { + // Use version+channel+os+arch as the tag. The arch is embedded + // in the archive path (e.g. flutter_macos_arm64_3.0.0-stable.zip + // vs flutter_macos_3.0.0-stable.zip for universal/x64). + arch := "" + base := filepath.Base(rel.Archive) + prefix := "flutter_" + rel.OS + "_" + if after, ok := strings.CutPrefix(base, prefix); ok { + if !strings.HasPrefix(after, rel.Version) { + // There's an arch segment between OS and version. + if idx := strings.Index(after, "_"); idx > 0 { + arch = after[:idx] + } + } + } + tag := fmt.Sprintf("%s-%s-%s", rel.Version, rel.Channel, rel.OS) + if arch != "" { + tag += "-" + arch + } + data, err := json.Marshal(rel) + if err != nil { + return fmt.Errorf("flutterdist marshal %s: %w", tag, err) + } + + action, err := d.Merge(tag, data) + if err != nil { + return err + } + switch action { + case "added": + added++ + case "changed": + changed++ + default: + skipped++ + } + + if latest == "" && rel.Channel == "stable" { + latest = tag + } + } + } + + if err := updateLatest(d, latest); err != nil { + return err + } + + log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest()) + return nil +} + +func fetchITerm2(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error { + d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName)) + if err != nil { + return err + } + + var added, changed, skipped int + var latest string + for batch, err := range iterm2dist.Fetch(ctx, client) { + if err != nil { + return fmt.Errorf("iterm2dist: %w", err) + } + for _, entry := range batch { + tag := entry.Version + if tag == "" { + continue + } + data, err := json.Marshal(entry) + if err != nil { + return fmt.Errorf("iterm2dist marshal %s: %w", tag, err) + } + + action, err := d.Merge(tag, data) + if err != nil { + return err + } + switch action { + case "added": + added++ + case "changed": + changed++ + default: + skipped++ + } + + if latest == "" && entry.Channel == "stable" { + latest = tag + } + } + } + + if err := updateLatest(d, latest); err != nil { + return err + } + + log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest()) + return nil +} + +func fetchHashiCorp(ctx context.Context, client *http.Client, cacheRoot, pkgName string, conf *installerconf.Conf) error { + product := conf.Extra["product"] + if product == "" { + return fmt.Errorf("missing product in releases.conf") + } + + d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName)) + if err != nil { + return err + } + + var added, changed, skipped int + var latest string + for idx, err := range hashicorp.Fetch(ctx, client, product) { + if err != nil { + return fmt.Errorf("hashicorp %s: %w", product, err) + } + for tag, ver := range idx.Versions { + data, err := json.Marshal(ver) + if err != nil { + return fmt.Errorf("hashicorp marshal %s: %w", tag, err) + } + + action, err := d.Merge(tag, data) + if err != nil { + return err + } + switch action { + case "added": + added++ + case "changed": + changed++ + default: + skipped++ + } + + // Stable = no prerelease markers. Compare all to find highest. + isStable := !strings.ContainsAny(tag, "-+") + if isStable { + if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 { + latest = tag + } + } + } + } + + if err := updateLatest(d, latest); err != nil { + return err + } + + log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest()) + return nil +} + +func fetchJulia(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error { + d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName)) + if err != nil { + return err + } + + var added, changed, skipped int + var latest string + for batch, err := range juliadist.Fetch(ctx, client) { + if err != nil { + return fmt.Errorf("juliadist: %w", err) + } + for _, rel := range batch { + tag := rel.Version + data, err := json.Marshal(rel) + if err != nil { + return fmt.Errorf("juliadist marshal %s: %w", tag, err) + } + + action, err := d.Merge(tag, data) + if err != nil { + return err + } + switch action { + case "added": + added++ + case "changed": + changed++ + default: + skipped++ + } + + if rel.Stable { + if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 { + latest = tag + } + } + } + } + + if err := updateLatest(d, latest); err != nil { + return err + } + + log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest()) + return nil +} + +func fetchGitTag(ctx context.Context, cacheRoot, pkgName string, conf *installerconf.Conf) error { + gitURL := conf.BaseURL + if gitURL == "" { + return fmt.Errorf("missing url in releases.conf") + } + + d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName)) + if err != nil { + return err + } + + repoDir := filepath.Join(cacheRoot, "_repos") + if err := os.MkdirAll(repoDir, 0o755); err != nil { + return err + } + + var added, changed, skipped int + var latest string + for batch, err := range gittag.Fetch(ctx, gitURL, repoDir) { + if err != nil { + return fmt.Errorf("gittag %s: %w", pkgName, err) + } + for _, entry := range batch { + tag := entry.Version + if tag == "" { + tag = "HEAD-" + entry.CommitHash + } + data, err := json.Marshal(entry) + if err != nil { + return fmt.Errorf("gittag marshal %s: %w", tag, err) + } + + action, err := d.Merge(tag, data) + if err != nil { + return err + } + switch action { + case "added": + added++ + case "changed": + changed++ + default: + skipped++ + } + + if entry.GitTag != "" && entry.GitTag != "HEAD" { + if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 { + latest = tag + } + } + } + } + + if err := updateLatest(d, latest); err != nil { + return err + } + + log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest()) + return nil +} + +func fetchGitea(ctx context.Context, client *http.Client, cacheRoot, pkgName string, conf *installerconf.Conf) error { + baseURL := conf.BaseURL + owner := conf.Owner + repo := conf.Repo + + if baseURL == "" || owner == "" || repo == "" { + return fmt.Errorf("missing base_url, owner, or repo in releases.conf") + } + + d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName)) + if err != nil { + return err + } + + var added, changed, skipped int + var latest string + for batch, err := range gitea.Fetch(ctx, client, baseURL, owner, repo, nil) { + if err != nil { + return fmt.Errorf("gitea %s/%s: %w", owner, repo, err) + } + for _, rel := range batch { + if rel.Draft { + continue + } + + tag := rel.TagName + data, err := json.Marshal(rel) + if err != nil { + return fmt.Errorf("gitea marshal %s: %w", tag, err) + } + + action, err := d.Merge(tag, data) + if err != nil { + return err + } + switch action { + case "added": + added++ + case "changed": + changed++ + default: + skipped++ + } + + if latest == "" && !rel.Prerelease { + latest = tag + } + } + } + + if err := updateLatest(d, latest); err != nil { + return err + } + + log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest()) + return nil +} + +func fetchChrome(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error { + d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName)) + if err != nil { + return err + } + + var added, changed, skipped int + var latest string + for batch, err := range chromedist.Fetch(ctx, client) { + if err != nil { + return fmt.Errorf("chromedist: %w", err) + } + for _, ver := range batch { + tag := ver.Version + data, err := json.Marshal(ver) + if err != nil { + return fmt.Errorf("chromedist marshal %s: %w", tag, err) + } + + action, err := d.Merge(tag, data) + if err != nil { + return err + } + switch action { + case "added": + added++ + case "changed": + changed++ + default: + skipped++ + } + + if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 { + latest = tag + } + } + } + + if err := updateLatest(d, latest); err != nil { + return err + } + + log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest()) + return nil +} + +func fetchGPG(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error { + d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName)) + if err != nil { + return err + } + + var added, changed, skipped int + var latest string + for batch, err := range gpgdist.Fetch(ctx, client) { + if err != nil { + return fmt.Errorf("gpgdist: %w", err) + } + for _, entry := range batch { + tag := entry.Version + data, err := json.Marshal(entry) + if err != nil { + return fmt.Errorf("gpgdist marshal %s: %w", tag, err) + } + + action, err := d.Merge(tag, data) + if err != nil { + return err + } + switch action { + case "added": + added++ + case "changed": + changed++ + default: + skipped++ + } + + if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 { + latest = tag + } + } + } + + if err := updateLatest(d, latest); err != nil { + return err + } + + log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest()) + return nil +} + +func fetchMariaDB(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error { + d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName)) + if err != nil { + return err + } + + var added, changed, skipped int + var latest string + for batch, err := range mariadbdist.Fetch(ctx, client) { + if err != nil { + return fmt.Errorf("mariadbdist: %w", err) + } + for _, rel := range batch { + tag := rel.ReleaseID + data, err := json.Marshal(rel) + if err != nil { + return fmt.Errorf("mariadbdist marshal %s: %w", tag, err) + } + + action, err := d.Merge(tag, data) + if err != nil { + return err + } + switch action { + case "added": + added++ + case "changed": + changed++ + default: + skipped++ + } + + isStable := rel.MajorStatus == "Stable" + if isStable { + if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 { + latest = tag + } + } + } + } + + if err := updateLatest(d, latest); err != nil { + return err + } + + log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest()) + return nil +} diff --git a/cmd/inspect/main.go b/cmd/inspect/main.go new file mode 100644 index 0000000..252d211 --- /dev/null +++ b/cmd/inspect/main.go @@ -0,0 +1,625 @@ +// Command inspect downloads release archives, unpacks them, and reports +// their internal structure. This helps discover how packages are laid out +// and whether the layout changes across versions. +// +// Usage: +// +// go run ./cmd/inspect -csv distributables.csv -cache ./_cache/downloads ollama sd +package main + +import ( + "context" + "encoding/csv" + "flag" + "fmt" + "io" + "log" + "mime" + "net/http" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/webinstall/webi-installers/internal/httpclient" +) + +// Row is one CSV row from distributables.csv. +type Row struct { + Package string + Version string + Channel string + Date string + OS string + Arch string + Libc string + Format string + Download string + Filename string + Extra string +} + +// archiveFormats are the formats we download and unpack. +var archiveFormats = map[string]bool{ + ".tar.gz": true, + ".tar.xz": true, + ".tar.bz2": true, + ".tar.zst": true, + ".zip": true, + ".dmg": true, + ".gz": true, + ".xz": true, +} + +// inspectOSes are the OSes we inspect. +var inspectOSes = map[string]bool{ + "linux": true, + "darwin": true, + "windows": true, + "": true, // source-only packages +} + +// preferredArch picks one arch per OS to download. +func preferredArch(os_ string) string { + switch os_ { + case "darwin": + return "aarch64" + default: + return "x86_64" + } +} + +func main() { + csvFile := flag.String("csv", "distributables.csv", "path to distributables CSV") + cacheDir := flag.String("cache", "_cache/downloads", "download cache directory") + flag.Parse() + + packages := flag.Args() + if len(packages) == 0 { + log.Fatal("usage: inspect [-csv FILE] [-cache DIR] PACKAGE [PACKAGE...]") + } + + rows, err := readCSV(*csvFile) + if err != nil { + log.Fatalf("read csv: %v", err) + } + + client := httpclient.New() + // Override timeout for large downloads. + client.Timeout = 10 * time.Minute + + for _, pkg := range packages { + log.Printf("=== %s ===", pkg) + if err := inspectPackage(client, rows, pkg, *cacheDir); err != nil { + log.Printf("ERROR: %s: %v", pkg, err) + } + } +} + +func readCSV(path string) ([]Row, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + r := csv.NewReader(f) + header, err := r.Read() + if err != nil { + return nil, err + } + + // Build column index. + idx := make(map[string]int, len(header)) + for i, col := range header { + idx[col] = i + } + + var rows []Row + for { + record, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + get := func(col string) string { + if i, ok := idx[col]; ok && i < len(record) { + return record[i] + } + return "" + } + + rows = append(rows, Row{ + Package: get("package"), + Version: get("version"), + Channel: get("channel"), + Date: get("date"), + OS: get("os"), + Arch: get("arch"), + Libc: get("libc"), + Format: get("format"), + Download: get("download"), + Filename: get("filename"), + Extra: get("extra"), + }) + } + return rows, nil +} + +func inspectPackage(client *http.Client, allRows []Row, pkg, cacheDir string) error { + // Filter rows for this package. + var pkgRows []Row + for _, r := range allRows { + if r.Package == pkg { + pkgRows = append(pkgRows, r) + } + } + if len(pkgRows) == 0 { + return fmt.Errorf("no rows found") + } + + // Find latest stable version, fall back to any version. + versions := findVersionsByDate(pkgRows) + if len(versions) == 0 { + return fmt.Errorf("no versions found") + } + + latestVer := versions[0] + log.Printf(" latest version: %s", latestVer) + + // Check if latest has assets uploaded (more than just source tarballs). + latestRows := filterVersion(pkgRows, latestVer) + hasRealAssets := false + for _, r := range latestRows { + if r.Extra != "source" && archiveFormats[r.Format] { + hasRealAssets = true + break + } + } + + // If latest looks empty, step back one version. + if !hasRealAssets && len(versions) > 1 { + latestVer = versions[1] + latestRows = filterVersion(pkgRows, latestVer) + log.Printf(" latest has no assets, using: %s", latestVer) + } + + // Inspect the latest version. + if err := inspectVersion(client, pkg, latestVer, latestRows, cacheDir); err != nil { + return err + } + + // Find versions roughly a year apart going back. + yearVersions := findYearlyVersions(pkgRows, latestVer) + for _, v := range yearVersions { + log.Printf(" --- checking %s ---", v) + vRows := filterVersion(pkgRows, v) + if err := inspectVersion(client, pkg, v, vRows, cacheDir); err != nil { + log.Printf(" ERROR: %v", err) + } + } + + return nil +} + +// findVersionsByDate returns versions sorted newest first, preferring stable. +func findVersionsByDate(rows []Row) []string { + type vInfo struct { + version string + date string + stable bool + } + seen := map[string]*vInfo{} + for _, r := range rows { + if _, ok := seen[r.Version]; !ok { + seen[r.Version] = &vInfo{ + version: r.Version, + date: r.Date, + stable: r.Channel == "stable", + } + } + } + + var vs []*vInfo + for _, v := range seen { + vs = append(vs, v) + } + + // Sort: stable first, then by date descending, then version descending. + sort.Slice(vs, func(i, j int) bool { + if vs[i].stable != vs[j].stable { + return vs[i].stable + } + if vs[i].date != vs[j].date { + return vs[i].date > vs[j].date + } + return vs[i].version > vs[j].version + }) + + result := make([]string, len(vs)) + for i, v := range vs { + result[i] = v.version + } + return result +} + +// findYearlyVersions picks versions roughly a year apart before the given version. +func findYearlyVersions(rows []Row, latestVer string) []string { + // Find the date of latest. + var latestDate string + for _, r := range rows { + if r.Version == latestVer && r.Date != "" { + latestDate = r.Date + break + } + } + if latestDate == "" { + return nil + } + + latestTime, err := time.Parse("2006-01-02", latestDate) + if err != nil { + return nil + } + + // Collect all stable versions with dates. + type vd struct { + version string + date time.Time + } + seen := map[string]bool{} + var all []vd + for _, r := range rows { + if seen[r.Version] || r.Date == "" || r.Channel != "stable" { + continue + } + seen[r.Version] = true + t, err := time.Parse("2006-01-02", r.Date) + if err != nil { + continue + } + if t.Before(latestTime) { + all = append(all, vd{r.Version, t}) + } + } + + sort.Slice(all, func(i, j int) bool { + return all[i].date.After(all[j].date) + }) + + // Pick versions roughly a year apart. + var result []string + nextTarget := latestTime.AddDate(-1, 0, 0) + for _, v := range all { + if v.date.Before(nextTarget) || v.date.Equal(nextTarget) { + result = append(result, v.version) + nextTarget = v.date.AddDate(-1, 0, 0) + } + } + + return result +} + +func filterVersion(rows []Row, version string) []Row { + var result []Row + for _, r := range rows { + if r.Version == version { + result = append(result, r) + } + } + return result +} + +// inspectVersion downloads and inspects archives for one version. +func inspectVersion(client *http.Client, pkg, version string, rows []Row, cacheDir string) error { + // Group by OS, pick one arch per OS, pick distinct formats. + type dlKey struct { + os_ string + format string + } + selected := map[dlKey]*Row{} + + for i := range rows { + r := &rows[i] + if !inspectOSes[r.OS] { + continue + } + if !archiveFormats[r.Format] { + continue + } + + key := dlKey{r.OS, r.Format} + existing := selected[key] + if existing == nil { + selected[key] = r + continue + } + + // Prefer the preferred arch. + pref := preferredArch(r.OS) + if r.Arch == pref && existing.Arch != pref { + selected[key] = r + } + // Skip rocm/jetpack variants. + if strings.Contains(r.Filename, "rocm") || strings.Contains(r.Filename, "jetpack") { + if !strings.Contains(existing.Filename, "rocm") && !strings.Contains(existing.Filename, "jetpack") { + continue // keep existing non-special variant + } + } + } + + if len(selected) == 0 { + log.Printf(" %s: no downloadable archives", version) + return nil + } + + // Sort keys for deterministic output. + var keys []dlKey + for k := range selected { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].os_ != keys[j].os_ { + return keys[i].os_ < keys[j].os_ + } + return keys[i].format < keys[j].format + }) + + for _, key := range keys { + r := selected[key] + os_ := r.OS + if os_ == "" { + os_ = "any" + } + log.Printf(" [%s] %s %s → %s", version, os_, r.Format, r.Filename) + + dlPath, err := download(client, r.Download, r.Filename, filepath.Join(cacheDir, pkg, version)) + if err != nil { + log.Printf(" download error: %v", err) + continue + } + + contents, err := unpackAndList(dlPath, r.Format) + if err != nil { + log.Printf(" unpack error: %v", err) + continue + } + + printContents(contents) + } + + return nil +} + +// download fetches a URL to the cache dir. Returns the path to the cached file. +// Skips download if the file already exists. +func download(client *http.Client, url, hintFilename, dir string) (string, error) { + // Check if already cached by hint filename. + cached := filepath.Join(dir, hintFilename) + if _, err := os.Stat(cached); err == nil { + return cached, nil + } + + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", err + } + + ctx := context.Background() + resp, err := httpclient.Get(ctx, client, url) + if err != nil { + return "", fmt.Errorf("GET %s: %w", url, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("GET %s: %s", url, resp.Status) + } + + // Determine filename from Content-Disposition or hint. + filename := hintFilename + if cd := resp.Header.Get("Content-Disposition"); cd != "" { + _, params, err := mime.ParseMediaType(cd) + if err == nil { + if fn, ok := params["filename"]; ok && fn != "" { + filename = fn + } + } + } + + outPath := filepath.Join(dir, filename) + + // Atomic write: temp file + rename. + tmp := outPath + ".tmp" + f, err := os.Create(tmp) + if err != nil { + return "", err + } + + n, err := io.Copy(f, resp.Body) + if closeErr := f.Close(); closeErr != nil && err == nil { + err = closeErr + } + if err != nil { + os.Remove(tmp) + return "", fmt.Errorf("download %s: %w", url, err) + } + + if err := os.Rename(tmp, outPath); err != nil { + os.Remove(tmp) + return "", err + } + + log.Printf(" downloaded %s (%d bytes)", filename, n) + return outPath, nil +} + +// FileEntry describes one file inside an archive. +type FileEntry struct { + Path string + Size int64 + Mode os.FileMode + IsDir bool + IsExec bool + IsSymlink bool + LinkTarget string +} + +// unpackAndList extracts an archive to a temp dir and lists contents. +func unpackAndList(archivePath, format string) ([]FileEntry, error) { + tmpDir, err := os.MkdirTemp("", "webi-inspect-*") + if err != nil { + return nil, err + } + defer os.RemoveAll(tmpDir) + + switch format { + case ".tar.gz": + err = run("tar", "xzf", archivePath, "-C", tmpDir) + case ".tar.xz": + err = run("tar", "xJf", archivePath, "-C", tmpDir) + case ".tar.bz2": + err = run("tar", "xjf", archivePath, "-C", tmpDir) + case ".tar.zst": + err = run("tar", "--zstd", "-xf", archivePath, "-C", tmpDir) + case ".zip": + err = run("unzip", "-q", "-o", archivePath, "-d", tmpDir) + case ".dmg": + err = extractDMG(archivePath, tmpDir) + case ".gz": + // Single file gzip. + base := filepath.Base(archivePath) + base = strings.TrimSuffix(base, ".gz") + outPath := filepath.Join(tmpDir, base) + err = run("sh", "-c", fmt.Sprintf("gunzip -c %q > %q", archivePath, outPath)) + case ".xz": + base := filepath.Base(archivePath) + base = strings.TrimSuffix(base, ".xz") + outPath := filepath.Join(tmpDir, base) + err = run("sh", "-c", fmt.Sprintf("xz -dc %q > %q", archivePath, outPath)) + default: + return nil, fmt.Errorf("unsupported format: %s", format) + } + if err != nil { + return nil, fmt.Errorf("extract %s: %w", format, err) + } + + return listDir(tmpDir, "") +} + +func extractDMG(dmgPath, outDir string) error { + // Try 7z first (doesn't require mounting). + if _, err := exec.LookPath("7z"); err == nil { + return run("7z", "x", "-o"+outDir, dmgPath) + } + + // Fall back to hdiutil mount + copy + unmount. + mountPoint, err := os.MkdirTemp("", "webi-dmg-*") + if err != nil { + return err + } + defer os.RemoveAll(mountPoint) + + if err := run("hdiutil", "attach", dmgPath, "-mountpoint", mountPoint, "-nobrowse", "-quiet"); err != nil { + return fmt.Errorf("mount dmg: %w", err) + } + defer run("hdiutil", "detach", mountPoint, "-quiet") + + // Copy contents. + return run("cp", "-R", mountPoint+"/.", outDir) +} + +func run(name string, args ...string) error { + cmd := exec.Command(name, args...) + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func listDir(root, prefix string) ([]FileEntry, error) { + entries, err := os.ReadDir(filepath.Join(root, prefix)) + if err != nil { + return nil, err + } + + var result []FileEntry + for _, e := range entries { + relPath := filepath.Join(prefix, e.Name()) + fullPath := filepath.Join(root, relPath) + + info, err := e.Info() + if err != nil { + continue + } + + entry := FileEntry{ + Path: relPath, + Size: info.Size(), + Mode: info.Mode(), + IsDir: e.IsDir(), + } + + if info.Mode()&os.ModeSymlink != 0 { + entry.IsSymlink = true + target, _ := os.Readlink(fullPath) + entry.LinkTarget = target + } + + if !e.IsDir() && info.Mode()&0o111 != 0 { + entry.IsExec = true + } + + result = append(result, entry) + + if e.IsDir() { + sub, err := listDir(root, relPath) + if err != nil { + continue + } + result = append(result, sub...) + } + } + return result, nil +} + +func printContents(entries []FileEntry) { + for _, e := range entries { + marker := " " + if e.IsExec { + marker = "* " + } + if e.IsDir { + marker = "d " + } + if e.IsSymlink { + marker = "→ " + } + + size := "" + if !e.IsDir { + size = formatSize(e.Size) + } + + line := fmt.Sprintf(" %s%-50s %8s", marker, e.Path, size) + if e.IsSymlink { + line += " → " + e.LinkTarget + } + log.Print(line) + } +} + +func formatSize(n int64) string { + switch { + case n >= 1<<30: + return fmt.Sprintf("%.1fG", float64(n)/float64(1<<30)) + case n >= 1<<20: + return fmt.Sprintf("%.1fM", float64(n)/float64(1<<20)) + case n >= 1<<10: + return fmt.Sprintf("%.1fK", float64(n)/float64(1<<10)) + default: + return fmt.Sprintf("%dB", n) + } +} diff --git a/cmd/uaparse/main.go b/cmd/uaparse/main.go new file mode 100644 index 0000000..d19124d --- /dev/null +++ b/cmd/uaparse/main.go @@ -0,0 +1,356 @@ +// Command uaparse analyzes User-Agent strings from webi.sh logs. +// +// It reads UA strings (one per line) from stdin or a file, parses each +// through uadetect, and produces summary output showing: +// - unique platform tuples (os, arch, libc) with counts +// - platform hints extracted from kernel version strings (cloud provider, +// container runtime, device info) +// - detection failures and malformed UAs +// +// Usage: +// +// uaparse < LIVE-UAS.txt +// uaparse LIVE-UAS.txt +// uaparse -json LIVE-UAS.txt +// uaparse -fixtures LIVE-UAS.txt # output Go test fixtures +package main + +import ( + "bufio" + "encoding/json" + "flag" + "fmt" + "os" + "regexp" + "sort" + "strings" + + "github.com/webinstall/webi-installers/internal/uadetect" +) + +// PlatformKey is the resolution-relevant tuple — everything else is noise +// for artifact selection. +type PlatformKey struct { + OS string `json:"os"` + Arch string `json:"arch"` + Libc string `json:"libc"` +} + +func (k PlatformKey) String() string { + return fmt.Sprintf("%-10s %-10s %s", k.OS, k.Arch, k.Libc) +} + +// PlatformEntry holds a unique platform and its metadata. +type PlatformEntry struct { + Key PlatformKey `json:"key"` + Count int `json:"count"` + Examples []string `json:"examples"` // up to 3 representative UAs + Hints []string `json:"hints"` // unique platform hints seen +} + +// UAIssue records a malformed or undetectable UA. +type UAIssue struct { + Line int `json:"line"` + UA string `json:"ua"` + Reason string `json:"reason"` +} + +// Hint is a platform detail extracted from the kernel version string. +type Hint struct { + Tag string // short label: "amzn", "azure", "gcp", "wsl", etc. + Count int +} + +func main() { + jsonOut := flag.Bool("json", false, "output as JSON") + fixtures := flag.Bool("fixtures", false, "output Go test fixture table") + flag.Parse() + + var scanner *bufio.Scanner + if flag.NArg() > 0 { + f, err := os.Open(flag.Arg(0)) + if err != nil { + fmt.Fprintf(os.Stderr, "uaparse: %v\n", err) + os.Exit(1) + } + defer f.Close() + scanner = bufio.NewScanner(f) + } else { + scanner = bufio.NewScanner(os.Stdin) + } + + platforms := make(map[PlatformKey]*PlatformEntry) + hints := make(map[string]int) + var issues []UAIssue + lineNum := 0 + + for scanner.Scan() { + lineNum++ + ua := strings.TrimSpace(scanner.Text()) + if ua == "" { + continue + } + + // Detect corruption: truncated/double-pasted lines. + if isMalformed(ua) { + issues = append(issues, UAIssue{ + Line: lineNum, + UA: ua, + Reason: "malformed (truncated or corrupted)", + }) + continue + } + + // Parse through uadetect. + result := uadetect.Parse(ua) + + // Check for detection failures. + if result.OS == "" { + issues = append(issues, UAIssue{ + Line: lineNum, + UA: ua, + Reason: "OS not detected", + }) + } + if result.Arch == "" { + issues = append(issues, UAIssue{ + Line: lineNum, + UA: ua, + Reason: "arch not detected", + }) + } + + key := PlatformKey{ + OS: string(result.OS), + Arch: string(result.Arch), + Libc: string(result.Libc), + } + + entry, ok := platforms[key] + if !ok { + entry = &PlatformEntry{Key: key} + platforms[key] = entry + } + entry.Count++ + if len(entry.Examples) < 3 { + entry.Examples = append(entry.Examples, ua) + } + + // Extract platform hints from kernel version. + for _, h := range extractHints(ua) { + if !containsStr(entry.Hints, h) { + entry.Hints = append(entry.Hints, h) + } + hints[h]++ + } + } + + if err := scanner.Err(); err != nil { + fmt.Fprintf(os.Stderr, "uaparse: read error: %v\n", err) + os.Exit(1) + } + + // Sort platforms by count descending. + entries := make([]*PlatformEntry, 0, len(platforms)) + for _, e := range platforms { + entries = append(entries, e) + } + sort.Slice(entries, func(i, j int) bool { + return entries[i].Count > entries[j].Count + }) + + if *jsonOut { + outputJSON(entries, issues, hints) + } else if *fixtures { + outputFixtures(entries) + } else { + outputTable(entries, issues, hints, lineNum) + } +} + +func outputTable(entries []*PlatformEntry, issues []UAIssue, hints map[string]int, total int) { + fmt.Printf("=== UA Analysis: %d lines → %d unique platforms ===\n\n", total, len(entries)) + + fmt.Printf("%-10s %-10s %-6s %6s %s\n", "OS", "ARCH", "LIBC", "COUNT", "HINTS") + fmt.Println(strings.Repeat("-", 72)) + for _, e := range entries { + hintStr := "" + if len(e.Hints) > 0 { + hintStr = strings.Join(e.Hints, ", ") + } + fmt.Printf("%-10s %-10s %-6s %6d %s\n", + displayOS(e.Key.OS), e.Key.Arch, displayLibc(e.Key.Libc), + e.Count, hintStr) + } + + if len(hints) > 0 { + fmt.Printf("\n=== Platform Hints (environment signals from kernel strings) ===\n\n") + sortedHints := make([]Hint, 0, len(hints)) + for tag, count := range hints { + sortedHints = append(sortedHints, Hint{tag, count}) + } + sort.Slice(sortedHints, func(i, j int) bool { + return sortedHints[i].Count > sortedHints[j].Count + }) + for _, h := range sortedHints { + fmt.Printf(" %-20s %d\n", h.Tag, h.Count) + } + } + + if len(issues) > 0 { + fmt.Printf("\n=== Issues (%d) ===\n\n", len(issues)) + for _, iss := range issues { + fmt.Printf(" line %d: %s\n %s\n", iss.Line, iss.Reason, iss.UA) + } + } +} + +func outputJSON(entries []*PlatformEntry, issues []UAIssue, hints map[string]int) { + out := struct { + Platforms []*PlatformEntry `json:"platforms"` + Issues []UAIssue `json:"issues"` + Hints map[string]int `json:"hints"` + }{entries, issues, hints} + + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + _ = enc.Encode(out) +} + +func outputFixtures(entries []*PlatformEntry) { + fmt.Println("// Generated by cmd/uaparse from live UA data.") + fmt.Println("// Each entry represents a unique (os, arch, libc) platform seen in production.") + fmt.Println("var liveUAPlatforms = []struct {") + fmt.Println("\tua string") + fmt.Println("\tos buildmeta.OS") + fmt.Println("\tarch buildmeta.Arch") + fmt.Println("\tlibc buildmeta.Libc") + fmt.Println("}{") + + for _, e := range entries { + if e.Key.OS == "" || e.Key.Arch == "" { + continue // skip undetectable + } + ua := e.Examples[0] + fmt.Printf("\t{%q, %s, %s, %s},\n", + ua, goConst("OS", e.Key.OS), goConst("Arch", e.Key.Arch), goConst("Libc", e.Key.Libc)) + } + + fmt.Println("}") +} + +// isMalformed checks for genuinely corrupted UA strings (network truncation). +func isMalformed(ua string) bool { + // Extremely short (less than 10 chars) suggests truncation. + if len(ua) < 10 { + return true + } + + return false +} + +// extractHints finds environment signals in a UA string. +func extractHints(ua string) []string { + lower := strings.ToLower(ua) + var out []string + + patterns := []struct { + substr string + tag string + }{ + {"amzn", "amzn"}, // Amazon Linux + {"-azure", "azure"}, // Azure VM + {"-gcp", "gcp"}, // Google Cloud + {"-aws", "aws"}, // AWS kernel + {"-oracle", "oracle"}, // Oracle Cloud + {"el7", "rhel7"}, // RHEL/CentOS 7 + {"el8", "rhel8"}, // RHEL/CentOS 8 + {"el9", "rhel9"}, // RHEL/CentOS 9 + {".fc", "fedora"}, // Fedora + {"+deb", "debian"}, // Debian + {"-generic", "ubuntu"}, // Ubuntu generic kernel + {"-pve", "proxmox"}, // Proxmox VE + {"linuxkit", "docker"}, // Docker Desktop / linuxkit + {"orbstack", "orbstack"}, + {"microsoft-standard-wsl", "wsl"}, + {"android", "android"}, + {"+rpt-rpi", "rpi"}, // Raspberry Pi + {"cygwin", "cygwin"}, + {"mingw", "mingw"}, + {"msys", "msys"}, + {"freebsd", "freebsd"}, + {"-nvidia", "nvidia"}, + {"gentoo", "gentoo"}, + {"coreweave", "coreweave"}, + } + + for _, p := range patterns { + if strings.Contains(lower, p.substr) { + out = append(out, p.tag) + } + } + + return out +} + +// androidDeviceRe extracts device/build info from Android kernel strings. +var androidDeviceRe = regexp.MustCompile(`ab[A-Z0-9]+`) + +func displayOS(os string) string { + if os == "" { + return "(none)" + } + return os +} + +func displayLibc(libc string) string { + if libc == "" { + return "(none)" + } + return libc +} + +func goConst(prefix, val string) string { + m := map[string]map[string]string{ + "OS": { + "darwin": "buildmeta.OSDarwin", + "linux": "buildmeta.OSLinux", + "windows": "buildmeta.OSWindows", + "freebsd": "buildmeta.OSFreeBSD", + "android": "buildmeta.OSAndroid", + "": `""`, + }, + "Arch": { + "aarch64": "buildmeta.ArchARM64", + "x86_64": "buildmeta.ArchAMD64", + "armv7": "buildmeta.ArchARMv7", + "armv6": "buildmeta.ArchARMv6", + "x86": "buildmeta.ArchX86", + "ppc64le": "buildmeta.ArchPPC64LE", + "ppc64": "buildmeta.ArchPPC64", + "s390x": "buildmeta.ArchS390X", + "riscv64": "buildmeta.ArchRISCV64", + "": `""`, + }, + "Libc": { + "gnu": "buildmeta.LibcGNU", + "musl": "buildmeta.LibcMusl", + "msvc": "buildmeta.LibcMSVC", + "none": "buildmeta.LibcNone", + "": `""`, + }, + } + if v, ok := m[prefix][val]; ok { + return v + } + return fmt.Sprintf("%q /* unmapped */", val) +} + +func containsStr(ss []string, s string) bool { + for _, v := range ss { + if v == s { + return true + } + } + return false +}