diff --git a/aliasman/releases.conf b/aliasman/releases.conf new file mode 100644 index 0000000..db928f1 --- /dev/null +++ b/aliasman/releases.conf @@ -0,0 +1,3 @@ +source = github +owner = BeyondCodeBootcamp +repo = aliasman diff --git a/awless/releases.conf b/awless/releases.conf new file mode 100644 index 0000000..b203f77 --- /dev/null +++ b/awless/releases.conf @@ -0,0 +1,3 @@ +source = github +owner = wallix +repo = awless diff --git a/chromedriver/releases.conf b/chromedriver/releases.conf new file mode 100644 index 0000000..39911e7 --- /dev/null +++ b/chromedriver/releases.conf @@ -0,0 +1 @@ +source = chromedist diff --git a/cmd/fetchraw/main.go b/cmd/fetchraw/main.go index c6c6513..b40f603 100644 --- a/cmd/fetchraw/main.go +++ b/cmd/fetchraw/main.go @@ -27,13 +27,18 @@ import ( "github.com/webinstall/webi-installers/internal/installerconf" "github.com/webinstall/webi-installers/internal/lexver" "github.com/webinstall/webi-installers/internal/rawcache" + "github.com/webinstall/webi-installers/internal/releases/chromedist" "github.com/webinstall/webi-installers/internal/releases/flutterdist" + "github.com/webinstall/webi-installers/internal/releases/gitea" "github.com/webinstall/webi-installers/internal/releases/github" "github.com/webinstall/webi-installers/internal/releases/githubish" + "github.com/webinstall/webi-installers/internal/releases/gittag" "github.com/webinstall/webi-installers/internal/releases/golang" + "github.com/webinstall/webi-installers/internal/releases/gpgdist" "github.com/webinstall/webi-installers/internal/releases/hashicorp" "github.com/webinstall/webi-installers/internal/releases/iterm2dist" "github.com/webinstall/webi-installers/internal/releases/juliadist" + "github.com/webinstall/webi-installers/internal/releases/mariadbdist" "github.com/webinstall/webi-installers/internal/releases/nodedist" "github.com/webinstall/webi-installers/internal/releases/zigdist" ) @@ -78,6 +83,12 @@ func main() { log.Printf("found %d packages", len(packages)) for _, pkg := range packages { + // Aliases share cache with their target — skip fetching. + if alias := pkg.conf.Get("alias_of"); alias != "" { + log.Printf(" %s: alias of %s, skipping", pkg.name, alias) + continue + } + log.Printf("fetching %s...", pkg.name) var err error switch pkg.conf.Source() { @@ -97,6 +108,16 @@ func main() { err = fetchHashiCorp(ctx, client, *cacheDir, pkg.name, pkg.conf) case "juliadist": err = fetchJulia(ctx, client, *cacheDir, pkg.name) + case "gittag": + err = fetchGitTag(ctx, *cacheDir, pkg.name, pkg.conf) + case "gitea": + err = fetchGitea(ctx, client, *cacheDir, pkg.name, pkg.conf) + case "chromedist": + err = fetchChrome(ctx, client, *cacheDir, pkg.name) + case "gpgdist": + err = fetchGPG(ctx, client, *cacheDir, pkg.name) + case "mariadbdist": + err = fetchMariaDB(ctx, client, *cacheDir, pkg.name) default: log.Printf(" %s: unknown source %q, skipping", pkg.name, pkg.conf.Source()) continue @@ -563,3 +584,263 @@ func fetchJulia(ctx context.Context, client *http.Client, cacheRoot, pkgName str log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest()) return nil } + +func fetchGitTag(ctx context.Context, cacheRoot, pkgName string, conf *installerconf.Conf) error { + gitURL := conf.Get("url") + if gitURL == "" { + return fmt.Errorf("missing url in releases.conf") + } + + d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName)) + if err != nil { + return err + } + + repoDir := filepath.Join(cacheRoot, "_repos") + if err := os.MkdirAll(repoDir, 0o755); err != nil { + return err + } + + var added, changed, skipped int + var latest string + for batch, err := range gittag.Fetch(ctx, gitURL, repoDir) { + if err != nil { + return fmt.Errorf("gittag %s: %w", pkgName, err) + } + for _, entry := range batch { + tag := entry.Version + if tag == "" { + tag = "HEAD-" + entry.CommitHash + } + data, err := json.Marshal(entry) + if err != nil { + return fmt.Errorf("gittag marshal %s: %w", tag, err) + } + + action, err := d.Merge(tag, data) + if err != nil { + return err + } + switch action { + case "added": + added++ + case "changed": + changed++ + default: + skipped++ + } + + if entry.GitTag != "" && entry.GitTag != "HEAD" { + if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 { + latest = tag + } + } + } + } + + if err := updateLatest(d, latest); err != nil { + return err + } + + log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest()) + return nil +} + +func fetchGitea(ctx context.Context, client *http.Client, cacheRoot, pkgName string, conf *installerconf.Conf) error { + baseURL := conf.Get("base_url") + owner := conf.Get("owner") + repo := conf.Get("repo") + + if baseURL == "" || owner == "" || repo == "" { + return fmt.Errorf("missing base_url, owner, or repo in releases.conf") + } + + d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName)) + if err != nil { + return err + } + + var added, changed, skipped int + var latest string + for batch, err := range gitea.Fetch(ctx, client, baseURL, owner, repo, nil) { + if err != nil { + return fmt.Errorf("gitea %s/%s: %w", owner, repo, err) + } + for _, rel := range batch { + if rel.Draft { + continue + } + + tag := rel.TagName + data, err := json.Marshal(rel) + if err != nil { + return fmt.Errorf("gitea marshal %s: %w", tag, err) + } + + action, err := d.Merge(tag, data) + if err != nil { + return err + } + switch action { + case "added": + added++ + case "changed": + changed++ + default: + skipped++ + } + + if latest == "" && !rel.Prerelease { + latest = tag + } + } + } + + if err := updateLatest(d, latest); err != nil { + return err + } + + log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest()) + return nil +} + +func fetchChrome(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error { + d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName)) + if err != nil { + return err + } + + var added, changed, skipped int + var latest string + for batch, err := range chromedist.Fetch(ctx, client) { + if err != nil { + return fmt.Errorf("chromedist: %w", err) + } + for _, ver := range batch { + tag := ver.Version + data, err := json.Marshal(ver) + if err != nil { + return fmt.Errorf("chromedist marshal %s: %w", tag, err) + } + + action, err := d.Merge(tag, data) + if err != nil { + return err + } + switch action { + case "added": + added++ + case "changed": + changed++ + default: + skipped++ + } + + if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 { + latest = tag + } + } + } + + if err := updateLatest(d, latest); err != nil { + return err + } + + log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest()) + return nil +} + +func fetchGPG(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error { + d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName)) + if err != nil { + return err + } + + var added, changed, skipped int + var latest string + for batch, err := range gpgdist.Fetch(ctx, client) { + if err != nil { + return fmt.Errorf("gpgdist: %w", err) + } + for _, entry := range batch { + tag := entry.Version + data, err := json.Marshal(entry) + if err != nil { + return fmt.Errorf("gpgdist marshal %s: %w", tag, err) + } + + action, err := d.Merge(tag, data) + if err != nil { + return err + } + switch action { + case "added": + added++ + case "changed": + changed++ + default: + skipped++ + } + + if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 { + latest = tag + } + } + } + + if err := updateLatest(d, latest); err != nil { + return err + } + + log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest()) + return nil +} + +func fetchMariaDB(ctx context.Context, client *http.Client, cacheRoot, pkgName string) error { + d, err := rawcache.Open(filepath.Join(cacheRoot, pkgName)) + if err != nil { + return err + } + + var added, changed, skipped int + var latest string + for batch, err := range mariadbdist.Fetch(ctx, client) { + if err != nil { + return fmt.Errorf("mariadbdist: %w", err) + } + for _, rel := range batch { + tag := rel.ReleaseID + data, err := json.Marshal(rel) + if err != nil { + return fmt.Errorf("mariadbdist marshal %s: %w", tag, err) + } + + action, err := d.Merge(tag, data) + if err != nil { + return err + } + switch action { + case "added": + added++ + case "changed": + changed++ + default: + skipped++ + } + + isStable := rel.MajorStatus == "Stable" + if isStable { + if latest == "" || lexver.Compare(lexver.Parse(tag), lexver.Parse(latest)) > 0 { + latest = tag + } + } + } + } + + if err := updateLatest(d, latest); err != nil { + return err + } + + log.Printf(" %s: +%d ~%d =%d latest=%s", pkgName, added, changed, skipped, d.Latest()) + return nil +} diff --git a/dashd/releases.conf b/dashd/releases.conf new file mode 100644 index 0000000..3ae0242 --- /dev/null +++ b/dashd/releases.conf @@ -0,0 +1 @@ +alias_of = dashcore diff --git a/duckdns.sh/releases.conf b/duckdns.sh/releases.conf new file mode 100644 index 0000000..ce619e1 --- /dev/null +++ b/duckdns.sh/releases.conf @@ -0,0 +1,3 @@ +source = github +owner = BeyondCodeBootcamp +repo = DuckDNS.sh diff --git a/golang/releases.conf b/golang/releases.conf new file mode 100644 index 0000000..eab2136 --- /dev/null +++ b/golang/releases.conf @@ -0,0 +1 @@ +alias_of = go diff --git a/gpg/releases.conf b/gpg/releases.conf new file mode 100644 index 0000000..d822427 --- /dev/null +++ b/gpg/releases.conf @@ -0,0 +1 @@ +source = gpgdist diff --git a/hugo-extended/releases.conf b/hugo-extended/releases.conf new file mode 100644 index 0000000..34b9804 --- /dev/null +++ b/hugo-extended/releases.conf @@ -0,0 +1,3 @@ +source = github +owner = gohugoio +repo = hugo diff --git a/internal/releases/chromedist/chromedist.go b/internal/releases/chromedist/chromedist.go new file mode 100644 index 0000000..b5e2acf --- /dev/null +++ b/internal/releases/chromedist/chromedist.go @@ -0,0 +1,72 @@ +// Package chromedist fetches Chrome for Testing release data. +// +// Google publishes a JSON index of known-good Chrome/ChromeDriver versions at: +// +// https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json +// +// Each version entry has per-platform download URLs for chrome, chromedriver, +// and chrome-headless-shell. +package chromedist + +import ( + "context" + "encoding/json" + "fmt" + "iter" + "net/http" +) + +// Index is the top-level response. +type Index struct { + Timestamp string `json:"timestamp"` + Versions []Version `json:"versions"` +} + +// Version is one Chrome for Testing version with its downloads. +type Version struct { + Version string `json:"version"` // "121.0.6120.0" + Revision string `json:"revision"` // "1222902" + Downloads map[string][]Download `json:"downloads"` // "chromedriver" → []Download +} + +// Download is one platform-specific download URL. +type Download struct { + Platform string `json:"platform"` // "linux64", "mac-arm64", "mac-x64", "win32", "win64" + URL string `json:"url"` +} + +// Fetch retrieves the Chrome for Testing release index. +// +// Yields one batch containing all versions. +func Fetch(ctx context.Context, client *http.Client) iter.Seq2[[]Version, error] { + return func(yield func([]Version, error) bool) { + url := "https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json" + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + yield(nil, fmt.Errorf("chromedist: %w", err)) + return + } + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + yield(nil, fmt.Errorf("chromedist: fetch: %w", err)) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + yield(nil, fmt.Errorf("chromedist: fetch: %s", resp.Status)) + return + } + + var idx Index + if err := json.NewDecoder(resp.Body).Decode(&idx); err != nil { + yield(nil, fmt.Errorf("chromedist: decode: %w", err)) + return + } + + yield(idx.Versions, nil) + } +} diff --git a/internal/releases/gpgdist/gpgdist.go b/internal/releases/gpgdist/gpgdist.go new file mode 100644 index 0000000..ee2ff98 --- /dev/null +++ b/internal/releases/gpgdist/gpgdist.go @@ -0,0 +1,70 @@ +// Package gpgdist fetches GPG for macOS release data from SourceForge RSS. +// +// The gpgosx project publishes DMG installers on SourceForge. The RSS feed +// at https://sourceforge.net/projects/gpgosx/rss?path=/ lists download links +// for each version. +package gpgdist + +import ( + "context" + "fmt" + "io" + "iter" + "net/http" + "regexp" +) + +// Entry is one GPG macOS release. +type Entry struct { + Version string `json:"version"` // "2.4.7" + URL string `json:"url"` // full SourceForge download URL +} + +var linkRe = regexp.MustCompile( + `(https://sourceforge\.net/projects/gpgosx/files/GnuPG-([\d.]+)\.dmg/download)`, +) + +// Fetch retrieves GPG macOS releases from the SourceForge RSS feed. +// +// Yields one batch containing all releases. +func Fetch(ctx context.Context, client *http.Client) iter.Seq2[[]Entry, error] { + return func(yield func([]Entry, error) bool) { + url := "https://sourceforge.net/projects/gpgosx/rss?path=/" + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + yield(nil, fmt.Errorf("gpgdist: %w", err)) + return + } + req.Header.Set("Accept", "application/rss+xml") + + resp, err := client.Do(req) + if err != nil { + yield(nil, fmt.Errorf("gpgdist: fetch: %w", err)) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + yield(nil, fmt.Errorf("gpgdist: fetch: %s", resp.Status)) + return + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + yield(nil, fmt.Errorf("gpgdist: read: %w", err)) + return + } + + matches := linkRe.FindAllStringSubmatch(string(body), -1) + var entries []Entry + for _, m := range matches { + entries = append(entries, Entry{ + URL: m[1], + Version: m[2], + }) + } + + yield(entries, nil) + } +} diff --git a/internal/releases/mariadbdist/mariadbdist.go b/internal/releases/mariadbdist/mariadbdist.go new file mode 100644 index 0000000..2f2fb96 --- /dev/null +++ b/internal/releases/mariadbdist/mariadbdist.go @@ -0,0 +1,159 @@ +// Package mariadbdist fetches MariaDB release data from the downloads API. +// +// MariaDB publishes release information via a REST API: +// +// https://downloads.mariadb.org/rest-api/mariadb/ +// https://downloads.mariadb.org/rest-api/mariadb/{major.minor}/ +// +// The first endpoint lists major release series; the second lists all point +// releases within a series, including download URLs per platform. +package mariadbdist + +import ( + "context" + "encoding/json" + "fmt" + "iter" + "net/http" + "regexp" +) + +// MajorRelease describes one release series (e.g. "11.4"). +type MajorRelease struct { + ReleaseID string `json:"release_id"` // "11.4" + ReleaseName string `json:"release_name"` // "MariaDB Server 11.4" + ReleaseStatus string `json:"release_status"` // "Stable", "RC", "Alpha" + ReleaseSupportType string `json:"release_support_type"` // "Long Term Support", etc. +} + +// Release is one point release with its downloadable files. +type Release struct { + ReleaseID string `json:"release_id"` // "11.4.5" + ReleaseName string `json:"release_name"` // "MariaDB Server 11.4.5" + DateOfRelease string `json:"date_of_release"` // "2025-02-12" + ReleaseNotesURL string `json:"release_notes_url"` // URL + Files []File `json:"files"` + + // MajorStatus is copied from the parent MajorRelease. Not in upstream JSON. + MajorStatus string `json:"major_status,omitempty"` +} + +// File is one downloadable artifact within a release. +type File struct { + FileID int `json:"file_id"` + FileName string `json:"file_name"` + PackageType string `json:"package_type"` // "gzipped tar file", "ZIP file" + OS string `json:"os"` // "Linux", "Windows", or "" + CPU string `json:"cpu"` // "x86_64" or "" + Checksum Checksum `json:"checksum"` + FileDownloadURL string `json:"file_download_url"` +} + +// Checksum holds hash digests for a file. +type Checksum struct { + SHA256 string `json:"sha256sum"` +} + +type majorResp struct { + MajorReleases []MajorRelease `json:"major_releases"` +} + +type releaseResp struct { + Releases map[string]Release `json:"releases"` +} + +var reVersion = regexp.MustCompile(`^\d+\.\d+$`) + +// Fetch retrieves all MariaDB releases across all major series. +// +// Yields one batch per major release series. +func Fetch(ctx context.Context, client *http.Client) iter.Seq2[[]Release, error] { + return func(yield func([]Release, error) bool) { + // Step 1: list major release series. + majors, err := fetchMajors(ctx, client) + if err != nil { + yield(nil, err) + return + } + + // Step 2: fetch point releases for each series. + for _, major := range majors { + if !reVersion.MatchString(major.ReleaseID) { + continue + } + + releases, err := fetchReleases(ctx, client, major.ReleaseID) + if err != nil { + yield(nil, fmt.Errorf("mariadbdist: %s: %w", major.ReleaseID, err)) + return + } + + // Tag each release with the major status. + for i := range releases { + releases[i].MajorStatus = major.ReleaseStatus + } + + if !yield(releases, nil) { + return + } + } + } +} + +func fetchMajors(ctx context.Context, client *http.Client) ([]MajorRelease, error) { + url := "https://downloads.mariadb.org/rest-api/mariadb/" + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("mariadbdist: %w", err) + } + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("mariadbdist: fetch majors: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("mariadbdist: fetch majors: %s", resp.Status) + } + + var result majorResp + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("mariadbdist: decode majors: %w", err) + } + + return result.MajorReleases, nil +} + +func fetchReleases(ctx context.Context, client *http.Client, majorID string) ([]Release, error) { + url := fmt.Sprintf("https://downloads.mariadb.org/rest-api/mariadb/%s", majorID) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("mariadbdist: %w", err) + } + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("mariadbdist: fetch %s: %w", majorID, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("mariadbdist: fetch %s: %s", majorID, resp.Status) + } + + var result releaseResp + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("mariadbdist: decode %s: %w", majorID, err) + } + + var releases []Release + for _, r := range result.Releases { + releases = append(releases, r) + } + return releases, nil +} diff --git a/kubens/releases.conf b/kubens/releases.conf new file mode 100644 index 0000000..07d3b10 --- /dev/null +++ b/kubens/releases.conf @@ -0,0 +1,3 @@ +source = github +owner = ahmetb +repo = kubectx diff --git a/mariadb/releases.conf b/mariadb/releases.conf new file mode 100644 index 0000000..bb96d57 --- /dev/null +++ b/mariadb/releases.conf @@ -0,0 +1 @@ +source = mariadbdist diff --git a/node/releases.conf b/node/releases.conf new file mode 100644 index 0000000..2a6681f --- /dev/null +++ b/node/releases.conf @@ -0,0 +1,2 @@ +source = nodedist +url = https://nodejs.org/download/release diff --git a/pathman/releases.conf b/pathman/releases.conf new file mode 100644 index 0000000..a327cab --- /dev/null +++ b/pathman/releases.conf @@ -0,0 +1,4 @@ +source = gitea +base_url = https://git.rootprojects.org +owner = root +repo = pathman diff --git a/postgres/releases.conf b/postgres/releases.conf new file mode 100644 index 0000000..f588f49 --- /dev/null +++ b/postgres/releases.conf @@ -0,0 +1,3 @@ +source = github +owner = bnnanet +repo = postgresql-releases diff --git a/psql/releases.conf b/psql/releases.conf new file mode 100644 index 0000000..4790d6b --- /dev/null +++ b/psql/releases.conf @@ -0,0 +1 @@ +alias_of = postgres diff --git a/rg/releases.conf b/rg/releases.conf new file mode 100644 index 0000000..9a123ca --- /dev/null +++ b/rg/releases.conf @@ -0,0 +1,3 @@ +source = github +owner = BurntSushi +repo = ripgrep diff --git a/vim-commentary/releases.conf b/vim-commentary/releases.conf new file mode 100644 index 0000000..1d48090 --- /dev/null +++ b/vim-commentary/releases.conf @@ -0,0 +1,2 @@ +source = gittag +url = https://github.com/tpope/vim-commentary.git diff --git a/vim-zig/releases.conf b/vim-zig/releases.conf new file mode 100644 index 0000000..5612946 --- /dev/null +++ b/vim-zig/releases.conf @@ -0,0 +1,2 @@ +source = gittag +url = https://github.com/ziglang/zig.vim.git diff --git a/zig.vim/releases.conf b/zig.vim/releases.conf new file mode 100644 index 0000000..fe218c5 --- /dev/null +++ b/zig.vim/releases.conf @@ -0,0 +1 @@ +alias_of = vim-zig