From 2e052fa5537151facf11c1b9e50f274dbaa4f9f6 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Tue, 10 Mar 2026 11:23:41 -0600 Subject: [PATCH] wire 9 custom source fetchers into webicached MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add fetch + classify functions for all custom source types: - chromedist (chromedriver): Chrome for Testing JSON index - flutterdist (flutter): Google Storage per-OS release indexes - golang (go): golang.org/dl JSON API - gpgdist (gpg): SourceForge RSS scraping - hashicorp (terraform): releases.hashicorp.com product index - iterm2dist (iterm2): HTML scraping of downloads page - juliadist (julia): S3 versions.json with platform files - mariadbdist (mariadb): two-step REST API (majors → releases) - zigdist (zig): mixed-schema JSON with platform keys All 9 fetcher packages already existed in internal/releases/ but were not wired into webicached's fetchRaw/classifyPackage switches. Now all 103 packages produce classified cache output. --- cmd/webicached/main.go | 611 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 610 insertions(+), 1 deletion(-) diff --git a/cmd/webicached/main.go b/cmd/webicached/main.go index 87fccc9..4d39f72 100644 --- a/cmd/webicached/main.go +++ b/cmd/webicached/main.go @@ -30,11 +30,20 @@ import ( "github.com/webinstall/webi-installers/internal/classify" "github.com/webinstall/webi-installers/internal/installerconf" "github.com/webinstall/webi-installers/internal/rawcache" + "github.com/webinstall/webi-installers/internal/releases/chromedist" + "github.com/webinstall/webi-installers/internal/releases/flutterdist" "github.com/webinstall/webi-installers/internal/releases/gitea" "github.com/webinstall/webi-installers/internal/releases/github" "github.com/webinstall/webi-installers/internal/releases/githubish" "github.com/webinstall/webi-installers/internal/releases/gittag" + "github.com/webinstall/webi-installers/internal/releases/golang" + "github.com/webinstall/webi-installers/internal/releases/gpgdist" + "github.com/webinstall/webi-installers/internal/releases/hashicorp" + "github.com/webinstall/webi-installers/internal/releases/iterm2dist" + "github.com/webinstall/webi-installers/internal/releases/juliadist" + "github.com/webinstall/webi-installers/internal/releases/mariadbdist" "github.com/webinstall/webi-installers/internal/releases/nodedist" + "github.com/webinstall/webi-installers/internal/releases/zigdist" "github.com/webinstall/webi-installers/internal/storage" "github.com/webinstall/webi-installers/internal/storage/fsstore" ) @@ -233,8 +242,25 @@ func fetchRaw(ctx context.Context, client *http.Client, rawDir string, pkg pkgCo return fetchGitTag(ctx, rawDir, pkg.name, pkg.conf) case "gitea": return fetchGitea(ctx, client, rawDir, pkg.name, pkg.conf) + case "chromedist": + return fetchChromeDist(ctx, client, rawDir, pkg.name) + case "flutterdist": + return fetchFlutterDist(ctx, client, rawDir, pkg.name) + case "golang": + return fetchGolang(ctx, client, rawDir, pkg.name) + case "gpgdist": + return fetchGPGDist(ctx, client, rawDir, pkg.name) + case "hashicorp": + return fetchHashiCorp(ctx, client, rawDir, pkg.name, pkg.conf) + case "iterm2dist": + return fetchITerm2Dist(ctx, client, rawDir, pkg.name) + case "juliadist": + return fetchJuliaDist(ctx, client, rawDir, pkg.name) + case "mariadbdist": + return fetchMariaDBDist(ctx, client, rawDir, pkg.name) + case "zigdist": + return fetchZigDist(ctx, client, rawDir, pkg.name) default: - // Sources not yet ported — skip silently for now. log.Printf(" %s: source %q not yet supported, skipping", pkg.name, pkg.conf.Source) return nil } @@ -362,6 +388,24 @@ func classifyPackage(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]s return classifyGitTag(pkg, d) case "gitea": return classifyGitea(pkg, conf, d) + case "chromedist": + return classifyChromeDist(d) + case "flutterdist": + return classifyFlutterDist(d) + case "golang": + return classifyGolang(d) + case "gpgdist": + return classifyGPGDist(d) + case "hashicorp": + return classifyHashiCorp(d) + case "iterm2dist": + return classifyITerm2Dist(d) + case "juliadist": + return classifyJuliaDist(d) + case "mariadbdist": + return classifyMariaDBDist(d) + case "zigdist": + return classifyZigDist(d) default: return nil, nil } @@ -726,6 +770,571 @@ func classifyGitea(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]sto return assets, nil } +// --- Chrome for Testing --- + +func fetchChromeDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error { + d, err := rawcache.Open(filepath.Join(rawDir, pkgName)) + if err != nil { + return err + } + + for batch, err := range chromedist.Fetch(ctx, client) { + if err != nil { + return fmt.Errorf("chromedist: %w", err) + } + for _, ver := range batch { + data, _ := json.Marshal(ver) + d.Merge(ver.Version, data) + } + } + return nil +} + +func classifyChromeDist(d *rawcache.Dir) ([]storage.Asset, error) { + releases, err := readAllRaw(d) + if err != nil { + return nil, err + } + + var assets []storage.Asset + for _, data := range releases { + var ver chromedist.Version + if err := json.Unmarshal(data, &ver); err != nil { + continue + } + + downloads := ver.Downloads["chromedriver"] + if len(downloads) == 0 { + continue + } + + for _, dl := range downloads { + r := classify.Filename(dl.URL) + assets = append(assets, storage.Asset{ + Filename: "chromedriver-" + dl.Platform + ".zip", + Version: ver.Version, + Channel: "stable", + OS: string(r.OS), + Arch: string(r.Arch), + Format: ".zip", + Download: dl.URL, + }) + } + } + return assets, nil +} + +// --- Flutter --- + +func fetchFlutterDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error { + d, err := rawcache.Open(filepath.Join(rawDir, pkgName)) + if err != nil { + return err + } + + for batch, err := range flutterdist.Fetch(ctx, client) { + if err != nil { + return fmt.Errorf("flutterdist: %w", err) + } + for _, rel := range batch { + // Key by version+channel+os for uniqueness. + key := rel.Version + "-" + rel.Channel + "-" + rel.OS + data, _ := json.Marshal(rel) + d.Merge(key, data) + } + } + return nil +} + +func classifyFlutterDist(d *rawcache.Dir) ([]storage.Asset, error) { + releases, err := readAllRaw(d) + if err != nil { + return nil, err + } + + var assets []storage.Asset + for _, data := range releases { + var rel flutterdist.Release + if err := json.Unmarshal(data, &rel); err != nil { + continue + } + + date := "" + if len(rel.ReleaseDate) >= 10 { + date = rel.ReleaseDate[:10] + } + + filename := filepath.Base(rel.Archive) + r := classify.Filename(filename) + + assets = append(assets, storage.Asset{ + Filename: filename, + Version: rel.Version, + Channel: rel.Channel, + OS: string(r.OS), + Arch: string(r.Arch), + Format: string(r.Format), + Download: rel.DownloadURL, + Date: date, + }) + } + return assets, nil +} + +// --- Go (golang.org) --- + +func fetchGolang(ctx context.Context, client *http.Client, rawDir, pkgName string) error { + d, err := rawcache.Open(filepath.Join(rawDir, pkgName)) + if err != nil { + return err + } + + for batch, err := range golang.Fetch(ctx, client) { + if err != nil { + return fmt.Errorf("golang: %w", err) + } + for _, rel := range batch { + data, _ := json.Marshal(rel) + d.Merge(rel.Version, data) + } + } + return nil +} + +func classifyGolang(d *rawcache.Dir) ([]storage.Asset, error) { + releases, err := readAllRaw(d) + if err != nil { + return nil, err + } + + var assets []storage.Asset + for _, data := range releases { + var rel golang.Release + if err := json.Unmarshal(data, &rel); err != nil { + continue + } + + // Strip "go" prefix from version: "go1.24.1" → "1.24.1" + version := strings.TrimPrefix(rel.Version, "go") + channel := "stable" + if !rel.Stable { + channel = "beta" + } + + for _, f := range rel.Files { + if f.Kind == "source" { + continue + } + // Skip bootstrap and odd builds. + if strings.Contains(f.Filename, "bootstrap") { + continue + } + + r := classify.Filename(f.Filename) + + assets = append(assets, storage.Asset{ + Filename: f.Filename, + Version: version, + Channel: channel, + OS: string(r.OS), + Arch: string(r.Arch), + Libc: string(r.Libc), + Format: string(r.Format), + Download: "https://dl.google.com/go/" + f.Filename, + }) + } + } + return assets, nil +} + +// --- GPG (SourceForge) --- + +func fetchGPGDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error { + d, err := rawcache.Open(filepath.Join(rawDir, pkgName)) + if err != nil { + return err + } + + for batch, err := range gpgdist.Fetch(ctx, client) { + if err != nil { + return fmt.Errorf("gpgdist: %w", err) + } + for _, entry := range batch { + data, _ := json.Marshal(entry) + d.Merge(entry.Version, data) + } + } + return nil +} + +func classifyGPGDist(d *rawcache.Dir) ([]storage.Asset, error) { + releases, err := readAllRaw(d) + if err != nil { + return nil, err + } + + var assets []storage.Asset + for _, data := range releases { + var entry gpgdist.Entry + if err := json.Unmarshal(data, &entry); err != nil { + continue + } + + assets = append(assets, storage.Asset{ + Filename: fmt.Sprintf("GnuPG-%s.dmg", entry.Version), + Version: entry.Version, + Channel: "stable", + OS: "darwin", + Arch: "amd64", + Format: ".dmg", + Download: entry.URL, + }) + } + return assets, nil +} + +// --- HashiCorp --- + +func fetchHashiCorp(ctx context.Context, client *http.Client, rawDir, pkgName string, conf *installerconf.Conf) error { + product := conf.Extra["product"] + if product == "" { + product = pkgName + } + + d, err := rawcache.Open(filepath.Join(rawDir, pkgName)) + if err != nil { + return err + } + + for idx, err := range hashicorp.Fetch(ctx, client, product) { + if err != nil { + return fmt.Errorf("hashicorp %s: %w", product, err) + } + for ver, vdata := range idx.Versions { + data, _ := json.Marshal(vdata) + d.Merge(ver, data) + } + } + return nil +} + +func classifyHashiCorp(d *rawcache.Dir) ([]storage.Asset, error) { + releases, err := readAllRaw(d) + if err != nil { + return nil, err + } + + var assets []storage.Asset + for _, data := range releases { + var ver hashicorp.Version + if err := json.Unmarshal(data, &ver); err != nil { + continue + } + + channel := "stable" + v := ver.Version + if strings.Contains(v, "-rc") { + channel = "rc" + } else if strings.Contains(v, "-beta") { + channel = "beta" + } else if strings.Contains(v, "-alpha") { + channel = "alpha" + } + + for _, b := range ver.Builds { + r := classify.Filename(b.Filename) + + assets = append(assets, storage.Asset{ + Filename: b.Filename, + Version: ver.Version, + Channel: channel, + OS: string(r.OS), + Arch: string(r.Arch), + Format: string(r.Format), + Download: b.URL, + }) + } + } + return assets, nil +} + +// --- iTerm2 --- + +func fetchITerm2Dist(ctx context.Context, client *http.Client, rawDir, pkgName string) error { + d, err := rawcache.Open(filepath.Join(rawDir, pkgName)) + if err != nil { + return err + } + + for batch, err := range iterm2dist.Fetch(ctx, client) { + if err != nil { + return fmt.Errorf("iterm2dist: %w", err) + } + for _, entry := range batch { + key := entry.Version + if entry.Channel == "beta" { + key += "-beta" + } + data, _ := json.Marshal(entry) + d.Merge(key, data) + } + } + return nil +} + +func classifyITerm2Dist(d *rawcache.Dir) ([]storage.Asset, error) { + releases, err := readAllRaw(d) + if err != nil { + return nil, err + } + + var assets []storage.Asset + for _, data := range releases { + var entry iterm2dist.Entry + if err := json.Unmarshal(data, &entry); err != nil { + continue + } + + filename := filepath.Base(entry.URL) + + assets = append(assets, storage.Asset{ + Filename: filename, + Version: entry.Version, + Channel: entry.Channel, + OS: "darwin", + Format: ".zip", + Download: entry.URL, + }) + } + return assets, nil +} + +// --- Julia --- + +func fetchJuliaDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error { + d, err := rawcache.Open(filepath.Join(rawDir, pkgName)) + if err != nil { + return err + } + + for batch, err := range juliadist.Fetch(ctx, client) { + if err != nil { + return fmt.Errorf("juliadist: %w", err) + } + for _, rel := range batch { + data, _ := json.Marshal(rel) + d.Merge(rel.Version, data) + } + } + return nil +} + +func classifyJuliaDist(d *rawcache.Dir) ([]storage.Asset, error) { + releases, err := readAllRaw(d) + if err != nil { + return nil, err + } + + osMap := map[string]string{ + "mac": "darwin", "linux": "linux", "winnt": "windows", + "freebsd": "freebsd", + } + archMap := map[string]string{ + "x86_64": "x86_64", "i686": "x86", "aarch64": "aarch64", + "armv7l": "armv7", "powerpc64le": "ppc64le", + } + + var assets []storage.Asset + for _, data := range releases { + var rel juliadist.Release + if err := json.Unmarshal(data, &rel); err != nil { + continue + } + + channel := "stable" + if !rel.Stable { + channel = "beta" + } + + for _, f := range rel.Files { + if f.Kind == "installer" { + continue + } + + os_ := osMap[f.OS] + arch := archMap[f.Arch] + libc := "" + if os_ == "linux" { + if strings.Contains(f.URL, "musl") { + libc = "musl" + } else { + libc = "gnu" + } + } + + filename := filepath.Base(f.URL) + + assets = append(assets, storage.Asset{ + Filename: filename, + Version: rel.Version, + Channel: channel, + OS: os_, + Arch: arch, + Libc: libc, + Format: "." + f.Extension, + Download: f.URL, + }) + } + } + return assets, nil +} + +// --- MariaDB --- + +func fetchMariaDBDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error { + d, err := rawcache.Open(filepath.Join(rawDir, pkgName)) + if err != nil { + return err + } + + for batch, err := range mariadbdist.Fetch(ctx, client) { + if err != nil { + return fmt.Errorf("mariadbdist: %w", err) + } + for _, rel := range batch { + data, _ := json.Marshal(rel) + d.Merge(rel.ReleaseID, data) + } + } + return nil +} + +func classifyMariaDBDist(d *rawcache.Dir) ([]storage.Asset, error) { + releases, err := readAllRaw(d) + if err != nil { + return nil, err + } + + channelMap := map[string]string{ + "Stable": "stable", "RC": "rc", "Alpha": "preview", + } + + var assets []storage.Asset + for _, data := range releases { + var rel mariadbdist.Release + if err := json.Unmarshal(data, &rel); err != nil { + continue + } + + channel := channelMap[rel.MajorStatus] + if channel == "" { + channel = "preview" + } + + lts := rel.MajorStatus == "Stable" + + for _, f := range rel.Files { + // Skip source packages (no OS or CPU). + if f.OS == "" || f.CPU == "" { + continue + } + // Skip debug builds. + if strings.Contains(strings.ToLower(f.FileName), "debug") { + continue + } + + r := classify.Filename(f.FileName) + + assets = append(assets, storage.Asset{ + Filename: f.FileName, + Version: rel.ReleaseID, + Channel: channel, + LTS: lts, + OS: string(r.OS), + Arch: string(r.Arch), + Format: string(r.Format), + Download: f.FileDownloadURL, + Date: rel.DateOfRelease, + }) + } + } + return assets, nil +} + +// --- Zig --- + +func fetchZigDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error { + d, err := rawcache.Open(filepath.Join(rawDir, pkgName)) + if err != nil { + return err + } + + for batch, err := range zigdist.Fetch(ctx, client) { + if err != nil { + return fmt.Errorf("zigdist: %w", err) + } + for _, rel := range batch { + data, _ := json.Marshal(rel) + d.Merge(rel.Version, data) + } + } + return nil +} + +func classifyZigDist(d *rawcache.Dir) ([]storage.Asset, error) { + releases, err := readAllRaw(d) + if err != nil { + return nil, err + } + + var assets []storage.Asset + for _, data := range releases { + var rel zigdist.Release + if err := json.Unmarshal(data, &rel); err != nil { + continue + } + + channel := "stable" + if !strings.Contains(rel.Version, ".") { + // Branch names like "master" have no dots. + channel = "beta" + } else if strings.ContainsAny(rel.Version, "+-") { + channel = "beta" + } + + for platform, p := range rel.Platforms { + // Skip source and odd entries. + if strings.Contains(platform, "bootstrap") || platform == "src" { + continue + } + if strings.Contains(platform, "armv6kz") { + continue + } + + // Platform is "arch-os", e.g. "x86_64-linux", "aarch64-macos". + parts := strings.SplitN(platform, "-", 2) + if len(parts) != 2 { + continue + } + + filename := filepath.Base(p.Tarball) + r := classify.Filename(filename) + + assets = append(assets, storage.Asset{ + Filename: filename, + Version: rel.Version, + Channel: channel, + OS: string(r.OS), + Arch: string(r.Arch), + Format: string(r.Format), + Download: p.Tarball, + Date: rel.Date, + }) + } + } + return assets, nil +} + // --- Helpers --- func isMetaAsset(name string) bool {