diff --git a/cmd/webicached/main.go b/cmd/webicached/main.go index d88bcdc..c0dc806 100644 --- a/cmd/webicached/main.go +++ b/cmd/webicached/main.go @@ -50,7 +50,7 @@ import ( "github.com/webinstall/webi-installers/internal/releases/iterm2dist" "github.com/webinstall/webi-installers/internal/releases/juliadist" "github.com/webinstall/webi-installers/internal/releases/mariadbdist" - "github.com/webinstall/webi-installers/internal/releases/nodedist" + "github.com/webinstall/webi-installers/internal/releases/nodeindex" "github.com/webinstall/webi-installers/internal/releases/servicemandist" "github.com/webinstall/webi-installers/internal/releases/zigdist" "github.com/webinstall/webi-installers/internal/storage" @@ -660,7 +660,7 @@ func (wc *WebiCache) fetchNodeDist(ctx context.Context, pkgName string, conf *in // Fetch from primary URL. Tag with "official/" prefix so unofficial // entries for the same version don't overwrite. - for batch, err := range nodedist.Fetch(ctx, wc.Client, baseURL) { + for batch, err := range nodeindex.Fetch(ctx, wc.Client, baseURL) { if err != nil { return err } @@ -673,7 +673,7 @@ func (wc *WebiCache) fetchNodeDist(ctx context.Context, pkgName string, conf *in // Fetch from unofficial URL if configured (e.g. Node.js unofficial builds // which add musl, riscv64, loong64 targets). if unofficialURL := conf.Extra["unofficial_url"]; unofficialURL != "" { - for batch, err := range nodedist.Fetch(ctx, wc.Client, unofficialURL) { + for batch, err := range nodeindex.Fetch(ctx, wc.Client, unofficialURL) { if err != nil { log.Printf("warning: %s unofficial fetch: %v", pkgName, err) break diff --git a/internal/classifypkg/classifypkg.go b/internal/classifypkg/classifypkg.go index c6dbc70..899e855 100644 --- a/internal/classifypkg/classifypkg.go +++ b/internal/classifypkg/classifypkg.go @@ -34,7 +34,7 @@ import ( "github.com/webinstall/webi-installers/internal/releases/juliadist" lsddist "github.com/webinstall/webi-installers/internal/releases/lsd" "github.com/webinstall/webi-installers/internal/releases/mariadbdist" - nodedist "github.com/webinstall/webi-installers/internal/releases/node" + "github.com/webinstall/webi-installers/internal/releases/nodedist" ollamadist "github.com/webinstall/webi-installers/internal/releases/ollama" "github.com/webinstall/webi-installers/internal/releases/postgres" pwshdist "github.com/webinstall/webi-installers/internal/releases/pwsh" diff --git a/internal/releases/node/node.go b/internal/releases/node/node.go deleted file mode 100644 index a56b1a0..0000000 --- a/internal/releases/node/node.go +++ /dev/null @@ -1,39 +0,0 @@ -// Package node fetches Node.js releases from both official and unofficial -// build sources. -// -// Official builds cover the standard platforms (linux-x64, osx-arm64, win-x64, -// etc.). Unofficial builds add musl, loong64, and other targets that the -// official CI doesn't produce. -// -// Both sources use the same index format, served by [nodedist]. -package nodedist - -import ( - "context" - "iter" - "net/http" - - "github.com/webinstall/webi-installers/internal/releases/nodedist" -) - -const ( - officialURL = "https://nodejs.org/download/release" - unofficialURL = "https://unofficial-builds.nodejs.org/download/release" -) - -// Fetch retrieves Node.js releases from both official and unofficial sources. -// Yields one batch per source (official first, then unofficial). -func Fetch(ctx context.Context, client *http.Client) iter.Seq2[[]nodedist.Entry, error] { - return func(yield func([]nodedist.Entry, error) bool) { - for entries, err := range nodedist.Fetch(ctx, client, officialURL) { - if !yield(entries, err) { - return - } - } - for entries, err := range nodedist.Fetch(ctx, client, unofficialURL) { - if !yield(entries, err) { - return - } - } - } -} diff --git a/internal/releases/node/node_test.go b/internal/releases/node/node_test.go deleted file mode 100644 index 73fe3d3..0000000 --- a/internal/releases/node/node_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package nodedist_test - -import ( - "context" - "net/http" - "testing" - - node "github.com/webinstall/webi-installers/internal/releases/node" -) - -func TestFetchCombinesSources(t *testing.T) { - if testing.Short() { - t.Skip("skipping network test in short mode") - } - - ctx := context.Background() - client := &http.Client{} - - var batches int - var total int - for entries, err := range node.Fetch(ctx, client) { - if err != nil { - t.Fatalf("batch %d: %v", batches, err) - } - batches++ - total += len(entries) - } - - if batches != 2 { - t.Errorf("got %d batches, want 2 (official + unofficial)", batches) - } - if total < 100 { - t.Errorf("got %d total entries, expected at least 100", total) - } - t.Logf("fetched %d entries in %d batches", total, batches) -} diff --git a/internal/releases/nodedist/nodedist.go b/internal/releases/nodedist/nodedist.go index 7790e05..e93bb83 100644 --- a/internal/releases/nodedist/nodedist.go +++ b/internal/releases/nodedist/nodedist.go @@ -1,108 +1,39 @@ -// Package nodedist fetches a Node.js-style distribution index. +// Package nodedist fetches Node.js releases from both official and unofficial +// build sources. // -// Node.js publishes a JSON index of all releases at: +// Official builds cover the standard platforms (linux-x64, osx-arm64, win-x64, +// etc.). Unofficial builds add musl, loong64, and other targets that the +// official CI doesn't produce. // -// https://nodejs.org/download/release/index.json -// -// Unofficial builds (musl, etc.) use the same format at: -// -// https://unofficial-builds.nodejs.org/download/release/index.json -// -// This package fetches and deserializes that index. It does not classify, -// normalize, or transform the data — the caller gets what the API returns. +// Both sources use the same index format, served by [nodeindex]. package nodedist import ( "context" - "encoding/json" - "fmt" "iter" "net/http" + + "github.com/webinstall/webi-installers/internal/releases/nodeindex" ) -// Entry is one release from a Node.js distribution index. -// Fields mirror the upstream JSON schema. -type Entry struct { - Version string `json:"version"` // "v25.8.0" - Date string `json:"date"` // "2026-03-03" - Files []string `json:"files"` // ["linux-arm64", "osx-arm64-tar", ...] - NPM string `json:"npm"` // "11.11.0" - V8 string `json:"v8"` // "14.1.146.11" - UV string `json:"uv"` // "1.51.0" - Zlib string `json:"zlib"` // "1.3.1" - OpenSSL string `json:"openssl"` // "3.5.5" - Modules string `json:"modules"` // "141" - LTS LTS `json:"lts"` // false or "Jod" - Security bool `json:"security"` // true if security release -} +const ( + officialURL = "https://nodejs.org/download/release" + unofficialURL = "https://unofficial-builds.nodejs.org/download/release" +) -// LTS holds the long-term support status. The upstream API encodes this as -// either the boolean false or a codename string like "Jod" or "Iron". -// An empty string means the release is not LTS. -type LTS string - -func (l *LTS) UnmarshalJSON(data []byte) error { - // false → "" - if string(data) == "false" { - *l = "" - return nil - } - - // "Codename" → Codename - var s string - if err := json.Unmarshal(data, &s); err != nil { - return fmt.Errorf("nodedist: unexpected lts value: %s", data) - } - *l = LTS(s) - return nil -} - -func (l LTS) MarshalJSON() ([]byte, error) { - if l == "" { - return []byte("false"), nil - } - return json.Marshal(string(l)) -} - -// Fetch retrieves the Node.js distribution index from baseURL. -// -// The iterator yields one batch per HTTP response. The Node.js index API -// returns all releases in a single response, so there will be exactly one -// yield. The iterator interface exists so that callers use the same pattern -// for paginated sources (like GitHub). -// -// Standard base URLs: -// - https://nodejs.org/download/release -// - https://unofficial-builds.nodejs.org/download/release -func Fetch(ctx context.Context, client *http.Client, baseURL string) iter.Seq2[[]Entry, error] { - return func(yield func([]Entry, error) bool) { - url := baseURL + "/index.json" - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - yield(nil, fmt.Errorf("nodedist: %w", err)) - return +// Fetch retrieves Node.js releases from both official and unofficial sources. +// Yields one batch per source (official first, then unofficial). +func Fetch(ctx context.Context, client *http.Client) iter.Seq2[[]nodeindex.Entry, error] { + return func(yield func([]nodeindex.Entry, error) bool) { + for entries, err := range nodeindex.Fetch(ctx, client, officialURL) { + if !yield(entries, err) { + return + } } - req.Header.Set("Accept", "application/json") - - resp, err := client.Do(req) - if err != nil { - yield(nil, fmt.Errorf("nodedist: fetch %s: %w", url, err)) - return + for entries, err := range nodeindex.Fetch(ctx, client, unofficialURL) { + if !yield(entries, err) { + return + } } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - yield(nil, fmt.Errorf("nodedist: fetch %s: %s", url, resp.Status)) - return - } - - var entries []Entry - if err := json.NewDecoder(resp.Body).Decode(&entries); err != nil { - yield(nil, fmt.Errorf("nodedist: decode %s: %w", url, err)) - return - } - - yield(entries, nil) } } diff --git a/internal/releases/nodedist/nodedist_test.go b/internal/releases/nodedist/nodedist_test.go index d451417..eacea12 100644 --- a/internal/releases/nodedist/nodedist_test.go +++ b/internal/releases/nodedist/nodedist_test.go @@ -2,142 +2,35 @@ package nodedist_test import ( "context" - "encoding/json" "net/http" - "net/http/httptest" "testing" "github.com/webinstall/webi-installers/internal/releases/nodedist" ) -// Minimal fixture from the real Node.js dist API. -const testIndex = `[ - { - "version": "v22.14.0", - "date": "2025-02-11", - "files": ["linux-arm64", "linux-x64", "osx-arm64-tar", "win-x64-zip", "src", "headers"], - "npm": "10.9.2", - "v8": "12.4.254.21", - "uv": "1.49.2", - "zlib": "1.3.0.1-motley-82a6be0", - "openssl": "3.0.15+quic", - "modules": "127", - "lts": "Jod", - "security": false - }, - { - "version": "v23.7.0", - "date": "2025-02-04", - "files": ["linux-arm64", "linux-x64", "osx-arm64-tar", "win-x64-zip"], - "npm": "10.9.2", - "v8": "13.2.152.16", - "uv": "1.49.2", - "zlib": "1.3.0.1-motley-82a6be0", - "openssl": "3.0.15+quic", - "modules": "131", - "lts": false, - "security": true - } -]` - -func TestFetch(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/index.json" { - t.Errorf("unexpected path: %s", r.URL.Path) - http.NotFound(w, r) - return - } - w.Header().Set("Content-Type", "application/json") - w.Write([]byte(testIndex)) - })) - defer srv.Close() +func TestFetchCombinesSources(t *testing.T) { + if testing.Short() { + t.Skip("skipping network test in short mode") + } ctx := context.Background() - var got []nodedist.Entry + client := &http.Client{} - for entries, err := range nodedist.Fetch(ctx, srv.Client(), srv.URL) { + var batches int + var total int + for entries, err := range nodedist.Fetch(ctx, client) { if err != nil { - t.Fatalf("Fetch: %v", err) + t.Fatalf("batch %d: %v", batches, err) } - got = append(got, entries...) + batches++ + total += len(entries) } - if len(got) != 2 { - t.Fatalf("got %d entries, want 2", len(got)) + if batches != 2 { + t.Errorf("got %d batches, want 2 (official + unofficial)", batches) } - - // First entry: LTS release - if got[0].Version != "v22.14.0" { - t.Errorf("entry[0].Version = %q, want %q", got[0].Version, "v22.14.0") - } - if got[0].Date != "2025-02-11" { - t.Errorf("entry[0].Date = %q, want %q", got[0].Date, "2025-02-11") - } - if got[0].LTS != "Jod" { - t.Errorf("entry[0].LTS = %q, want %q", got[0].LTS, "Jod") - } - if got[0].Security { - t.Error("entry[0].Security = true, want false") - } - if len(got[0].Files) != 6 { - t.Errorf("entry[0].Files len = %d, want 6", len(got[0].Files)) - } - - // Second entry: non-LTS, security release - if got[1].Version != "v23.7.0" { - t.Errorf("entry[1].Version = %q, want %q", got[1].Version, "v23.7.0") - } - if got[1].LTS != "" { - t.Errorf("entry[1].LTS = %q, want empty (non-LTS)", got[1].LTS) - } - if !got[1].Security { - t.Error("entry[1].Security = false, want true") - } -} - -func TestFetchHTTPError(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.Error(w, "rate limited", http.StatusTooManyRequests) - })) - defer srv.Close() - - ctx := context.Background() - for _, err := range nodedist.Fetch(ctx, srv.Client(), srv.URL) { - if err == nil { - t.Fatal("expected error for 429 response") - } - return - } -} - -func TestLTSMarshalRoundTrip(t *testing.T) { - // LTS codename - entry := nodedist.Entry{LTS: "Jod"} - data, err := json.Marshal(entry) - if err != nil { - t.Fatal(err) - } - - var got nodedist.Entry - if err := json.Unmarshal(data, &got); err != nil { - t.Fatal(err) - } - if got.LTS != "Jod" { - t.Errorf("LTS roundtrip: got %q, want %q", got.LTS, "Jod") - } - - // Non-LTS - entry2 := nodedist.Entry{LTS: ""} - data2, err := json.Marshal(entry2) - if err != nil { - t.Fatal(err) - } - - var got2 nodedist.Entry - if err := json.Unmarshal(data2, &got2); err != nil { - t.Fatal(err) - } - if got2.LTS != "" { - t.Errorf("non-LTS roundtrip: got %q, want empty", got2.LTS) + if total < 100 { + t.Errorf("got %d total entries, expected at least 100", total) } + t.Logf("fetched %d entries in %d batches", total, batches) } diff --git a/internal/releases/node/variants.go b/internal/releases/nodedist/variants.go similarity index 100% rename from internal/releases/node/variants.go rename to internal/releases/nodedist/variants.go diff --git a/internal/releases/nodeindex/nodeindex.go b/internal/releases/nodeindex/nodeindex.go new file mode 100644 index 0000000..08c59d9 --- /dev/null +++ b/internal/releases/nodeindex/nodeindex.go @@ -0,0 +1,108 @@ +// Package nodeindex fetches a Node.js-style distribution index. +// +// Node.js publishes a JSON index of all releases at: +// +// https://nodejs.org/download/release/index.json +// +// Unofficial builds (musl, etc.) use the same format at: +// +// https://unofficial-builds.nodejs.org/download/release/index.json +// +// This package fetches and deserializes that index. It does not classify, +// normalize, or transform the data — the caller gets what the API returns. +package nodeindex + +import ( + "context" + "encoding/json" + "fmt" + "iter" + "net/http" +) + +// Entry is one release from a Node.js distribution index. +// Fields mirror the upstream JSON schema. +type Entry struct { + Version string `json:"version"` // "v25.8.0" + Date string `json:"date"` // "2026-03-03" + Files []string `json:"files"` // ["linux-arm64", "osx-arm64-tar", ...] + NPM string `json:"npm"` // "11.11.0" + V8 string `json:"v8"` // "14.1.146.11" + UV string `json:"uv"` // "1.51.0" + Zlib string `json:"zlib"` // "1.3.1" + OpenSSL string `json:"openssl"` // "3.5.5" + Modules string `json:"modules"` // "141" + LTS LTS `json:"lts"` // false or "Jod" + Security bool `json:"security"` // true if security release +} + +// LTS holds the long-term support status. The upstream API encodes this as +// either the boolean false or a codename string like "Jod" or "Iron". +// An empty string means the release is not LTS. +type LTS string + +func (l *LTS) UnmarshalJSON(data []byte) error { + // false → "" + if string(data) == "false" { + *l = "" + return nil + } + + // "Codename" → Codename + var s string + if err := json.Unmarshal(data, &s); err != nil { + return fmt.Errorf("nodeindex: unexpected lts value: %s", data) + } + *l = LTS(s) + return nil +} + +func (l LTS) MarshalJSON() ([]byte, error) { + if l == "" { + return []byte("false"), nil + } + return json.Marshal(string(l)) +} + +// Fetch retrieves the Node.js distribution index from baseURL. +// +// The iterator yields one batch per HTTP response. The Node.js index API +// returns all releases in a single response, so there will be exactly one +// yield. The iterator interface exists so that callers use the same pattern +// for paginated sources (like GitHub). +// +// Standard base URLs: +// - https://nodejs.org/download/release +// - https://unofficial-builds.nodejs.org/download/release +func Fetch(ctx context.Context, client *http.Client, baseURL string) iter.Seq2[[]Entry, error] { + return func(yield func([]Entry, error) bool) { + url := baseURL + "/index.json" + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + yield(nil, fmt.Errorf("nodeindex: %w", err)) + return + } + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + yield(nil, fmt.Errorf("nodeindex: fetch %s: %w", url, err)) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + yield(nil, fmt.Errorf("nodeindex: fetch %s: %s", url, resp.Status)) + return + } + + var entries []Entry + if err := json.NewDecoder(resp.Body).Decode(&entries); err != nil { + yield(nil, fmt.Errorf("nodeindex: decode %s: %w", url, err)) + return + } + + yield(entries, nil) + } +} diff --git a/internal/releases/nodeindex/nodeindex_test.go b/internal/releases/nodeindex/nodeindex_test.go new file mode 100644 index 0000000..5710673 --- /dev/null +++ b/internal/releases/nodeindex/nodeindex_test.go @@ -0,0 +1,143 @@ +package nodeindex_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/webinstall/webi-installers/internal/releases/nodeindex" +) + +// Minimal fixture from the real Node.js dist API. +const testIndex = `[ + { + "version": "v22.14.0", + "date": "2025-02-11", + "files": ["linux-arm64", "linux-x64", "osx-arm64-tar", "win-x64-zip", "src", "headers"], + "npm": "10.9.2", + "v8": "12.4.254.21", + "uv": "1.49.2", + "zlib": "1.3.0.1-motley-82a6be0", + "openssl": "3.0.15+quic", + "modules": "127", + "lts": "Jod", + "security": false + }, + { + "version": "v23.7.0", + "date": "2025-02-04", + "files": ["linux-arm64", "linux-x64", "osx-arm64-tar", "win-x64-zip"], + "npm": "10.9.2", + "v8": "13.2.152.16", + "uv": "1.49.2", + "zlib": "1.3.0.1-motley-82a6be0", + "openssl": "3.0.15+quic", + "modules": "131", + "lts": false, + "security": true + } +]` + +func TestFetch(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/index.json" { + t.Errorf("unexpected path: %s", r.URL.Path) + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(testIndex)) + })) + defer srv.Close() + + ctx := context.Background() + var got []nodeindex.Entry + + for entries, err := range nodeindex.Fetch(ctx, srv.Client(), srv.URL) { + if err != nil { + t.Fatalf("Fetch: %v", err) + } + got = append(got, entries...) + } + + if len(got) != 2 { + t.Fatalf("got %d entries, want 2", len(got)) + } + + // First entry: LTS release + if got[0].Version != "v22.14.0" { + t.Errorf("entry[0].Version = %q, want %q", got[0].Version, "v22.14.0") + } + if got[0].Date != "2025-02-11" { + t.Errorf("entry[0].Date = %q, want %q", got[0].Date, "2025-02-11") + } + if got[0].LTS != "Jod" { + t.Errorf("entry[0].LTS = %q, want %q", got[0].LTS, "Jod") + } + if got[0].Security { + t.Error("entry[0].Security = true, want false") + } + if len(got[0].Files) != 6 { + t.Errorf("entry[0].Files len = %d, want 6", len(got[0].Files)) + } + + // Second entry: non-LTS, security release + if got[1].Version != "v23.7.0" { + t.Errorf("entry[1].Version = %q, want %q", got[1].Version, "v23.7.0") + } + if got[1].LTS != "" { + t.Errorf("entry[1].LTS = %q, want empty (non-LTS)", got[1].LTS) + } + if !got[1].Security { + t.Error("entry[1].Security = false, want true") + } +} + +func TestFetchHTTPError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "rate limited", http.StatusTooManyRequests) + })) + defer srv.Close() + + ctx := context.Background() + for _, err := range nodeindex.Fetch(ctx, srv.Client(), srv.URL) { + if err == nil { + t.Fatal("expected error for 429 response") + } + return + } +} + +func TestLTSMarshalRoundTrip(t *testing.T) { + // LTS codename + entry := nodeindex.Entry{LTS: "Jod"} + data, err := json.Marshal(entry) + if err != nil { + t.Fatal(err) + } + + var got nodeindex.Entry + if err := json.Unmarshal(data, &got); err != nil { + t.Fatal(err) + } + if got.LTS != "Jod" { + t.Errorf("LTS roundtrip: got %q, want %q", got.LTS, "Jod") + } + + // Non-LTS + entry2 := nodeindex.Entry{LTS: ""} + data2, err := json.Marshal(entry2) + if err != nil { + t.Fatal(err) + } + + var got2 nodeindex.Entry + if err := json.Unmarshal(data2, &got2); err != nil { + t.Fatal(err) + } + if got2.LTS != "" { + t.Errorf("non-LTS roundtrip: got %q, want empty", got2.LTS) + } +}