diff --git a/internal/buildmeta/buildmeta.go b/internal/buildmeta/buildmeta.go index d100bb7..77b92ef 100644 --- a/internal/buildmeta/buildmeta.go +++ b/internal/buildmeta/buildmeta.go @@ -87,17 +87,6 @@ const ( ChannelDev Channel = "dev" ) -// ChannelNames lists recognized channel names in priority order. -var ChannelNames = []Channel{ - ChannelStable, - ChannelLatest, - ChannelRC, - ChannelPreview, - ChannelBeta, - ChannelAlpha, - ChannelDev, -} - // Target represents a fully resolved build target. type Target struct { OS OS @@ -109,27 +98,3 @@ type Target struct { func (t Target) Triplet() string { return string(t.OS) + "-" + string(t.Arch) + "-" + string(t.Libc) } - -// Release represents a single downloadable build artifact. -type Release struct { - Name string `json:"name"` - Version string `json:"version"` - LTS bool `json:"lts"` - Channel Channel `json:"channel"` - Date string `json:"date"` // "2024-01-15" - OS OS `json:"os"` - Arch Arch `json:"arch"` - Libc Libc `json:"libc"` - Ext Format `json:"ext"` - Download string `json:"download"` - Comment string `json:"comment,omitempty"` -} - -// PackageMeta holds aggregate metadata about a package's available releases. -type PackageMeta struct { - Name string `json:"name"` - OSes []OS `json:"oses"` - Arches []Arch `json:"arches"` - Libcs []Libc `json:"libcs"` - Formats []Format `json:"formats"` -} diff --git a/internal/httpclient/httpclient.go b/internal/httpclient/httpclient.go index 32dcc9a..aab028d 100644 --- a/internal/httpclient/httpclient.go +++ b/internal/httpclient/httpclient.go @@ -1,12 +1,10 @@ -// Package httpclient provides a secure, resilient HTTP client for upstream API -// calls. It exists because [http.DefaultClient] has no timeouts, no retry -// logic, and follows redirects from HTTPS to HTTP — none of which are -// acceptable for a server making calls to GitHub, Gitea, etc. on behalf of -// users. +// Package httpclient provides a well-configured [http.Client] for upstream +// API calls. It exists because [http.DefaultClient] has no timeouts, no TLS +// minimum, and follows redirects from HTTPS to HTTP — none of which are +// acceptable for a server calling GitHub, Gitea, etc. on behalf of users. // -// Create a [Client] with [New], then use [Client.Do] or [Client.Get]. All -// calls require [context.Context] and will automatically retry transient -// failures (429, 502, 503, 504) with backoff. +// Use [New] to create a configured client. Use [Do] to execute a request +// with automatic retries for transient failures. package httpclient import ( @@ -21,78 +19,38 @@ import ( "time" ) -// Client wraps http.Client with retry logic and resilience defaults. -type Client struct { - inner *http.Client - userAgent string - retries int - baseDelay time.Duration - maxDelay time.Duration -} +const userAgent = "Webi/2.0 (+https://webinstall.dev)" -// Option configures a Client. -type Option func(*Client) - -// WithUserAgent sets the User-Agent header for all requests. -func WithUserAgent(ua string) Option { - return func(c *Client) { c.userAgent = ua } -} - -// WithRetries sets the maximum number of retries for transient errors. -func WithRetries(n int) Option { - return func(c *Client) { c.retries = n } -} - -// WithBaseDelay sets the initial delay for exponential backoff. -func WithBaseDelay(d time.Duration) Option { - return func(c *Client) { c.baseDelay = d } -} - -// New creates a Client with secure, resilient defaults. -func New(opts ...Option) *Client { - transport := &http.Transport{ - DialContext: (&net.Dialer{ - Timeout: 10 * time.Second, - KeepAlive: 30 * time.Second, - }).DialContext, - TLSClientConfig: &tls.Config{ - MinVersion: tls.VersionTLS12, +// New returns an [http.Client] with secure, production-ready defaults: +// TLS 1.2+, timeouts at every level, connection pooling, no HTTPS→HTTP +// redirect, and a Webi User-Agent. +func New() *http.Client { + return &http.Client{ + Transport: &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + TLSClientConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + }, + TLSHandshakeTimeout: 10 * time.Second, + ResponseHeaderTimeout: 30 * time.Second, + MaxIdleConns: 100, + MaxIdleConnsPerHost: 10, + IdleConnTimeout: 90 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + ForceAttemptHTTP2: true, }, - TLSHandshakeTimeout: 10 * time.Second, - ResponseHeaderTimeout: 30 * time.Second, - MaxIdleConns: 100, - MaxIdleConnsPerHost: 10, - IdleConnTimeout: 90 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - ForceAttemptHTTP2: true, + Timeout: 60 * time.Second, + CheckRedirect: checkRedirect, } - - c := &Client{ - inner: &http.Client{ - Transport: transport, - Timeout: 60 * time.Second, - CheckRedirect: checkRedirect, - }, - userAgent: "Webi/2.0 (+https://webinstall.dev)", - retries: 3, - baseDelay: 1 * time.Second, - maxDelay: 30 * time.Second, - } - - for _, opt := range opts { - opt(c) - } - - return c } -// maxRedirects is the redirect depth limit. -const maxRedirects = 10 - // checkRedirect prevents HTTPS→HTTP downgrades and limits redirect depth. func checkRedirect(req *http.Request, via []*http.Request) error { - if len(via) >= maxRedirects { - return fmt.Errorf("stopped after %d redirects", maxRedirects) + if len(via) >= 10 { + return fmt.Errorf("stopped after %d redirects", len(via)) } if len(via) > 0 && via[0].URL.Scheme == "https" && req.URL.Scheme == "http" { return errors.New("refused redirect from https to http") @@ -100,36 +58,57 @@ func checkRedirect(req *http.Request, via []*http.Request) error { return nil } -// Do executes a request with automatic retries for transient errors. -// It sets the User-Agent header if not already present. -func (c *Client) Do(req *http.Request) (*http.Response, error) { +// Get performs a GET request with the Webi User-Agent header. +func Get(ctx context.Context, client *http.Client, url string) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", userAgent) + return client.Do(req) +} + +// Do executes a request with automatic retries for transient errors (429, +// 502, 503, 504). Retries up to 3 times with exponential backoff and jitter. +// Respects Retry-After headers. Only retries GET and HEAD (idempotent). +// +// Sets the Webi User-Agent header if not already present. +func Do(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) { if req.Header.Get("User-Agent") == "" { - req.Header.Set("User-Agent", c.userAgent) + req.Header.Set("User-Agent", userAgent) } + // Only retry idempotent methods. + idempotent := req.Method == http.MethodGet || req.Method == http.MethodHead + + const maxRetries = 3 var resp *http.Response var err error - for attempt := 0; attempt <= c.retries; attempt++ { + for attempt := range maxRetries + 1 { if attempt > 0 { - delay := c.backoff(attempt, resp) - select { - case <-req.Context().Done(): - return nil, req.Context().Err() - case <-time.After(delay): + if !idempotent { + break + } + + delay := backoff(attempt, resp) + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: } - // Close previous response body before retry. if resp != nil { resp.Body.Close() } } - resp, err = c.inner.Do(req) + resp, err = client.Do(req) if err != nil { - // Retry on transient network errors. - if req.Context().Err() != nil { - return nil, req.Context().Err() + if ctx.Err() != nil { + return nil, ctx.Err() } continue } @@ -139,62 +118,37 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) { } } - // Exhausted retries — return whatever we have. if err != nil { - return nil, fmt.Errorf("after %d retries: %w", c.retries, err) + return nil, fmt.Errorf("after %d retries: %w", maxRetries, err) } return resp, nil } -// Get is a convenience wrapper around Do for GET requests. -func (c *Client) Get(ctx context.Context, url string) (*http.Response, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, err - } - return c.Do(req) -} - -// isRetryable returns true for HTTP status codes that indicate a transient error. func isRetryable(status int) bool { - switch status { - case http.StatusTooManyRequests, // 429 - http.StatusBadGateway, // 502 - http.StatusServiceUnavailable, // 503 - http.StatusGatewayTimeout: // 504 - return true - } - return false + return status == http.StatusTooManyRequests || + status == http.StatusBadGateway || + status == http.StatusServiceUnavailable || + status == http.StatusGatewayTimeout } -// backoff calculates the delay before the next retry attempt. -// Uses exponential backoff with jitter, and respects Retry-After headers. -func (c *Client) backoff(attempt int, resp *http.Response) time.Duration { - // Check Retry-After header from previous response. +// backoff returns a delay before the next retry. Respects Retry-After, +// otherwise uses exponential backoff with jitter. +func backoff(attempt int, resp *http.Response) time.Duration { if resp != nil { if ra := resp.Header.Get("Retry-After"); ra != "" { - if seconds, err := strconv.Atoi(ra); err == nil { - d := time.Duration(seconds) * time.Second - if d > 0 && d < 5*time.Minute { - return d - } + if seconds, err := strconv.Atoi(ra); err == nil && seconds > 0 && seconds < 300 { + return time.Duration(seconds) * time.Second } } } - // Exponential backoff: baseDelay * 2^attempt + jitter - delay := c.baseDelay - for i := 1; i < attempt; i++ { - delay *= 2 - if delay > c.maxDelay { - delay = c.maxDelay - break - } + // 1s, 2s, 4s base delays + base := time.Second << (attempt - 1) + if base > 30*time.Second { + base = 30 * time.Second } - // Add jitter: ±25% - jitter := time.Duration(float64(delay) * 0.5 * rand.Float64()) - delay = delay + jitter - (delay / 4) - - return delay + // Add jitter: 75% to 125% of base + jitter := float64(base) * (0.75 + 0.5*rand.Float64()) + return time.Duration(jitter) } diff --git a/internal/uadetect/uadetect.go b/internal/uadetect/uadetect.go index c049123..66176e1 100644 --- a/internal/uadetect/uadetect.go +++ b/internal/uadetect/uadetect.go @@ -5,10 +5,11 @@ // "Darwin 23.1.0 arm64" or "Linux 6.1.0 x86_64". This package parses those // into [buildmeta.OS], [buildmeta.Arch], and [buildmeta.Libc] values so the // server can select the correct release artifact. +// +// Also handles non-uname agents like "PowerShell/7.3.0" and "MS AMD64". package uadetect import ( - "regexp" "strings" "github.com/webinstall/webi-installers/internal/buildmeta" @@ -23,136 +24,180 @@ type Result struct { // Parse extracts OS, arch, and libc from a User-Agent string. func Parse(ua string) Result { + if ua == "-" { + return Result{} + } + + tokens := tokenize(ua) + return Result{ - OS: DetectOS(ua), - Arch: DetectArch(ua), - Libc: DetectLibc(ua), + OS: matchOS(tokens), + Arch: matchArch(tokens), + Libc: matchLibc(tokens), } } -// DetectOS returns the OS from a User-Agent string. -func DetectOS(ua string) buildmeta.OS { - if ua == "-" { - return "" +// tokenize splits a User-Agent into lowercase tokens for matching. +// Splits on whitespace, '/', and ';', since UAs come in various forms: +// +// "Darwin 23.1.0 arm64" (uname -srm) +// "PowerShell/7.3.0" (PowerShell) +// "MS AMD64" (Windows shorthand) +// "Macintosh; Intel Mac OS X 10_15_7" (browser) +func tokenize(ua string) []string { + // Strip xnu kernel info that can mislead arch detection under Rosetta. + // "xnu-7195.60.75~1/RELEASE_ARM64_T8101" contains ARM64 even when + // running as x86_64. This only appears in verbose uname output. + if i := strings.Index(ua, "xnu-"); i >= 0 { + end := strings.IndexByte(ua[i:], ' ') + if end < 0 { + ua = ua[:i] + } else { + ua = ua[:i] + ua[i+end:] + } } - // Android must be tested before Linux. - if reAndroid.MatchString(ua) { + return strings.FieldsFunc(strings.ToLower(ua), func(r rune) bool { + return r == ' ' || r == '/' || r == ';' || r == '\t' + }) +} + +// matchOS identifies the operating system from tokens. +// Order matters: Android before Linux, Linux before Windows (for WSL). +func matchOS(tokens []string) buildmeta.OS { + has := func(s string) bool { + for _, t := range tokens { + if strings.Contains(t, s) { + return true + } + } + return false + } + + // Android must be checked before Linux. + if has("android") { return buildmeta.OSAndroid } - // macOS/Darwin must be tested before Linux (for edge cases) and before - // "win" (because "darwin" contains no "win", but ordering matters). - if reDarwin.MatchString(ua) { + if has("darwin") || has("macos") || has("macintosh") || has("iphone") || has("ios") || has("ipad") { return buildmeta.OSDarwin } + // "mac" alone (not in "macintosh" which is already matched) + for _, t := range tokens { + if t == "mac" { + return buildmeta.OSDarwin + } + } - // Linux must be tested before Windows because WSL User-Agents contain - // both "Linux" and sometimes "Microsoft". - if reLinux.MatchString(ua) && !reCygwin.MatchString(ua) { + // Linux before Windows because WSL UAs contain both "linux" and "microsoft". + // But exclude Cygwin/msysgit which report Linux-like strings on Windows. + if has("linux") && !has("cygwin") && !has("msysgit") { return buildmeta.OSLinux } - if reWindows.MatchString(ua) { + if has("windows") || has("win32") || has("microsoft") || has("powershell") { return buildmeta.OSWindows } + for _, t := range tokens { + if t == "ms" || t == "win" { + return buildmeta.OSWindows + } + } - // Try Linux again after Windows (for plain "curl" or "wget"). - if reLinuxLoose.MatchString(ua) { + // Fallback: curl and wget imply a POSIX system, almost always Linux. + if has("curl") || has("wget") { return buildmeta.OSLinux } return "" } -// DetectArch returns the CPU architecture from a User-Agent string. -func DetectArch(ua string) buildmeta.Arch { - if ua == "-" { - return "" +// matchArch identifies the CPU architecture from tokens. +// More specific patterns are checked before less specific ones. +func matchArch(tokens []string) buildmeta.Arch { + has := func(s string) bool { + for _, t := range tokens { + if strings.Contains(t, s) { + return true + } + } + return false + } + exact := func(s string) bool { + for _, t := range tokens { + if t == s { + return true + } + } + return false } - // Strip macOS kernel release arch info that can mislead detection. - // e.g. "xnu-7195.60.75~1/RELEASE_ARM64_T8101 x86_64" under Rosetta - ua = reXNU.ReplaceAllString(ua, "") - - // Order matters — more specific patterns first. - if reARM64.MatchString(ua) { + // ARM 64-bit (most specific first) + if has("aarch64") || has("arm64") || has("armv8") { return buildmeta.ArchARM64 } - if reARMv7.MatchString(ua) { + + // ARM 32-bit variants + if has("armv7") || has("arm32") { return buildmeta.ArchARMv7 } - if reARMv6.MatchString(ua) { + if has("armv6") { return buildmeta.ArchARMv6 } - if rePPC64LE.MatchString(ua) { + // Bare "arm" without a version qualifier → armv6 (conservative). + if exact("arm") { + return buildmeta.ArchARMv6 + } + + // POWER (check before generic 64-bit) + if has("ppc64le") { return buildmeta.ArchPPC64LE } - if rePPC64.MatchString(ua) { + if has("ppc64") { return buildmeta.ArchPPC64 } - if reMIPS64.MatchString(ua) { + + // MIPS (check before generic 64-bit) + if has("mips64") { return buildmeta.ArchMIPS64 } - if reMIPS.MatchString(ua) { + if has("mips") { return buildmeta.ArchMIPS } - // amd64 must come after ppc64/mips64 (both contain "64"). - if reAMD64.MatchString(ua) { + + // x86-64 + if has("x86_64") || has("amd64") || exact("x64") { return buildmeta.ArchAMD64 } - // x86 must come after x86_64/amd64. - if reX86.MatchString(ua) { + + // x86 32-bit (after x86_64 to avoid false match) + if has("i386") || has("i686") || exact("x86") { return buildmeta.ArchX86 } return "" } -// DetectLibc returns the C library variant from a User-Agent string. -func DetectLibc(ua string) buildmeta.Libc { - if ua == "-" { - return "" +// matchLibc identifies the C library from tokens. +func matchLibc(tokens []string) buildmeta.Libc { + has := func(s string) bool { + for _, t := range tokens { + if strings.Contains(t, s) { + return true + } + } + return false } - lower := strings.ToLower(ua) - - if reMusl.MatchString(lower) { + if has("musl") { return buildmeta.LibcMusl } - if reMSVC.MatchString(lower) { + if has("msvc") || has("windows") || has("microsoft") { return buildmeta.LibcMSVC } - if reGNU.MatchString(lower) { + if has("gnu") || has("glibc") || has("linux") { return buildmeta.LibcGNU } - // Default: no specific libc requirement detected. return buildmeta.LibcNone } - -// Compiled regexes — allocated once. -var ( - reAndroid = regexp.MustCompile(`(?i)Android`) - reDarwin = regexp.MustCompile(`(?i)iOS|iPhone|Macintosh|Darwin|OS\s*X|macOS|mac`) - reLinux = regexp.MustCompile(`(?i)Linux`) - reCygwin = regexp.MustCompile(`(?i)cygwin|msysgit`) - reWindows = regexp.MustCompile(`(?i)(\b|^)ms(\b|$)|Microsoft|Windows|win32|win|PowerShell`) - reLinuxLoose = regexp.MustCompile(`(?i)Linux|curl|wget`) - - reXNU = regexp.MustCompile(`xnu-\S*RELEASE_\S*`) - - reARM64 = regexp.MustCompile(`(?i)(\b|_)(aarch64|arm64|arm8|armv8)`) - reARMv7 = regexp.MustCompile(`(?i)(\b|_)(aarch|arm7|armv7|arm32)`) - reARMv6 = regexp.MustCompile(`(?i)(\b|_)(arm6|armv6|arm(\b|_))`) - rePPC64LE = regexp.MustCompile(`(?i)ppc64le`) - rePPC64 = regexp.MustCompile(`(?i)ppc64`) - reMIPS64 = regexp.MustCompile(`(?i)mips64`) - reMIPS = regexp.MustCompile(`(?i)mips`) - reAMD64 = regexp.MustCompile(`(?i)(amd64|x86_64|x64|_64)\b`) - reX86 = regexp.MustCompile(`(?i)(\b|_)(3|6|x|_)86\b`) - - reMusl = regexp.MustCompile(`(\b|_)musl(\b|_)`) - reMSVC = regexp.MustCompile(`(\b|_)(msvc|windows|microsoft)(\b|_)`) - reGNU = regexp.MustCompile(`(\b|_)(gnu|glibc|linux)(\b|_)`) -) diff --git a/internal/uadetect/uadetect_test.go b/internal/uadetect/uadetect_test.go index 961ce09..a7664c2 100644 --- a/internal/uadetect/uadetect_test.go +++ b/internal/uadetect/uadetect_test.go @@ -7,21 +7,18 @@ import ( "github.com/webinstall/webi-installers/internal/uadetect" ) -func TestDetectOS(t *testing.T) { +func TestOS(t *testing.T) { tests := []struct { ua string want buildmeta.OS }{ - // macOS / Darwin + // uname -srm style {"Darwin 23.1.0 arm64", buildmeta.OSDarwin}, {"Darwin 20.2.0 x86_64", buildmeta.OSDarwin}, - {"Macintosh; Intel Mac OS X 10_15_7", buildmeta.OSDarwin}, - - // Linux {"Linux 6.1.0-18-amd64 x86_64", buildmeta.OSLinux}, {"Linux 5.15.0 aarch64", buildmeta.OSLinux}, - // WSL (Linux, not Windows) + // WSL: Linux, not Windows (contains "microsoft" in kernel release) {"Linux 5.15.146.1-microsoft-standard-WSL2 x86_64", buildmeta.OSLinux}, // Windows @@ -29,28 +26,31 @@ func TestDetectOS(t *testing.T) { {"PowerShell/7.3.0", buildmeta.OSWindows}, {"Microsoft Windows 10.0.19045", buildmeta.OSWindows}, - // Android + // Android before Linux {"Android 13 aarch64", buildmeta.OSAndroid}, - // Minimal agents + // Browser-style + {"Macintosh; Intel Mac OS X 10_15_7", buildmeta.OSDarwin}, + + // Minimal agents → assume Linux {"curl/8.1.2", buildmeta.OSLinux}, {"wget/1.21", buildmeta.OSLinux}, - // Dash means unknown + // Explicit unknown {"-", ""}, } for _, tt := range tests { t.Run(tt.ua, func(t *testing.T) { - got := uadetect.DetectOS(tt.ua) + got := uadetect.Parse(tt.ua).OS if got != tt.want { - t.Errorf("DetectOS(%q) = %q, want %q", tt.ua, got, tt.want) + t.Errorf("Parse(%q).OS = %q, want %q", tt.ua, got, tt.want) } }) } } -func TestDetectArch(t *testing.T) { +func TestArch(t *testing.T) { tests := []struct { ua string want buildmeta.Arch @@ -63,7 +63,7 @@ func TestDetectArch(t *testing.T) { {"Linux 5.10.0 armv6l", buildmeta.ArchARMv6}, {"Linux 5.4.0 ppc64le", buildmeta.ArchPPC64LE}, - // Rosetta: kernel says ARM64 but uname reports x86_64 + // Rosetta: xnu kernel info says ARM64 but actual arch is x86_64 {"Darwin 20.2.0 Darwin Kernel Version 20.2.0; root:xnu-7195.60.75~1/RELEASE_ARM64_T8101 x86_64", buildmeta.ArchAMD64}, {"-", ""}, @@ -71,15 +71,15 @@ func TestDetectArch(t *testing.T) { for _, tt := range tests { t.Run(tt.ua, func(t *testing.T) { - got := uadetect.DetectArch(tt.ua) + got := uadetect.Parse(tt.ua).Arch if got != tt.want { - t.Errorf("DetectArch(%q) = %q, want %q", tt.ua, got, tt.want) + t.Errorf("Parse(%q).Arch = %q, want %q", tt.ua, got, tt.want) } }) } } -func TestDetectLibc(t *testing.T) { +func TestLibc(t *testing.T) { tests := []struct { ua string want buildmeta.Libc @@ -95,15 +95,15 @@ func TestDetectLibc(t *testing.T) { for _, tt := range tests { t.Run(tt.ua, func(t *testing.T) { - got := uadetect.DetectLibc(tt.ua) + got := uadetect.Parse(tt.ua).Libc if got != tt.want { - t.Errorf("DetectLibc(%q) = %q, want %q", tt.ua, got, tt.want) + t.Errorf("Parse(%q).Libc = %q, want %q", tt.ua, got, tt.want) } }) } } -func TestParse(t *testing.T) { +func TestFullParse(t *testing.T) { r := uadetect.Parse("Darwin 23.1.0 arm64") if r.OS != buildmeta.OSDarwin { t.Errorf("OS = %q, want %q", r.OS, buildmeta.OSDarwin)