ref(internal): rewrite buildmeta, uadetect, httpclient from scratch

buildmeta: remove premature Release/PackageMeta structs and
ChannelNames slice — keep only the shared vocabulary types.

uadetect: replace regex-based matching with token-based matching.
Split UA on whitespace/slash/semicolon, match lowercase tokens.
Strip xnu kernel info for Rosetta. Single Parse() entry point.

httpclient: return plain *http.Client from New(). Make Do() and Get()
free functions. Only retry idempotent methods (GET/HEAD).
This commit is contained in:
AJ ONeal
2026-03-08 22:07:50 -06:00
parent 1374bca46b
commit 43ab591061
4 changed files with 223 additions and 259 deletions

View File

@@ -87,17 +87,6 @@ const (
ChannelDev Channel = "dev"
)
// ChannelNames lists recognized channel names in priority order.
var ChannelNames = []Channel{
ChannelStable,
ChannelLatest,
ChannelRC,
ChannelPreview,
ChannelBeta,
ChannelAlpha,
ChannelDev,
}
// Target represents a fully resolved build target.
type Target struct {
OS OS
@@ -109,27 +98,3 @@ type Target struct {
func (t Target) Triplet() string {
return string(t.OS) + "-" + string(t.Arch) + "-" + string(t.Libc)
}
// Release represents a single downloadable build artifact.
type Release struct {
Name string `json:"name"`
Version string `json:"version"`
LTS bool `json:"lts"`
Channel Channel `json:"channel"`
Date string `json:"date"` // "2024-01-15"
OS OS `json:"os"`
Arch Arch `json:"arch"`
Libc Libc `json:"libc"`
Ext Format `json:"ext"`
Download string `json:"download"`
Comment string `json:"comment,omitempty"`
}
// PackageMeta holds aggregate metadata about a package's available releases.
type PackageMeta struct {
Name string `json:"name"`
OSes []OS `json:"oses"`
Arches []Arch `json:"arches"`
Libcs []Libc `json:"libcs"`
Formats []Format `json:"formats"`
}

View File

@@ -1,12 +1,10 @@
// Package httpclient provides a secure, resilient HTTP client for upstream API
// calls. It exists because [http.DefaultClient] has no timeouts, no retry
// logic, and follows redirects from HTTPS to HTTP — none of which are
// acceptable for a server making calls to GitHub, Gitea, etc. on behalf of
// users.
// Package httpclient provides a well-configured [http.Client] for upstream
// API calls. It exists because [http.DefaultClient] has no timeouts, no TLS
// minimum, and follows redirects from HTTPS to HTTP — none of which are
// acceptable for a server calling GitHub, Gitea, etc. on behalf of users.
//
// Create a [Client] with [New], then use [Client.Do] or [Client.Get]. All
// calls require [context.Context] and will automatically retry transient
// failures (429, 502, 503, 504) with backoff.
// Use [New] to create a configured client. Use [Do] to execute a request
// with automatic retries for transient failures.
package httpclient
import (
@@ -21,78 +19,38 @@ import (
"time"
)
// Client wraps http.Client with retry logic and resilience defaults.
type Client struct {
inner *http.Client
userAgent string
retries int
baseDelay time.Duration
maxDelay time.Duration
}
const userAgent = "Webi/2.0 (+https://webinstall.dev)"
// Option configures a Client.
type Option func(*Client)
// WithUserAgent sets the User-Agent header for all requests.
func WithUserAgent(ua string) Option {
return func(c *Client) { c.userAgent = ua }
}
// WithRetries sets the maximum number of retries for transient errors.
func WithRetries(n int) Option {
return func(c *Client) { c.retries = n }
}
// WithBaseDelay sets the initial delay for exponential backoff.
func WithBaseDelay(d time.Duration) Option {
return func(c *Client) { c.baseDelay = d }
}
// New creates a Client with secure, resilient defaults.
func New(opts ...Option) *Client {
transport := &http.Transport{
DialContext: (&net.Dialer{
Timeout: 10 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
TLSClientConfig: &tls.Config{
MinVersion: tls.VersionTLS12,
// New returns an [http.Client] with secure, production-ready defaults:
// TLS 1.2+, timeouts at every level, connection pooling, no HTTPS→HTTP
// redirect, and a Webi User-Agent.
func New() *http.Client {
return &http.Client{
Transport: &http.Transport{
DialContext: (&net.Dialer{
Timeout: 10 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
TLSClientConfig: &tls.Config{
MinVersion: tls.VersionTLS12,
},
TLSHandshakeTimeout: 10 * time.Second,
ResponseHeaderTimeout: 30 * time.Second,
MaxIdleConns: 100,
MaxIdleConnsPerHost: 10,
IdleConnTimeout: 90 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
ForceAttemptHTTP2: true,
},
TLSHandshakeTimeout: 10 * time.Second,
ResponseHeaderTimeout: 30 * time.Second,
MaxIdleConns: 100,
MaxIdleConnsPerHost: 10,
IdleConnTimeout: 90 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
ForceAttemptHTTP2: true,
Timeout: 60 * time.Second,
CheckRedirect: checkRedirect,
}
c := &Client{
inner: &http.Client{
Transport: transport,
Timeout: 60 * time.Second,
CheckRedirect: checkRedirect,
},
userAgent: "Webi/2.0 (+https://webinstall.dev)",
retries: 3,
baseDelay: 1 * time.Second,
maxDelay: 30 * time.Second,
}
for _, opt := range opts {
opt(c)
}
return c
}
// maxRedirects is the redirect depth limit.
const maxRedirects = 10
// checkRedirect prevents HTTPS→HTTP downgrades and limits redirect depth.
func checkRedirect(req *http.Request, via []*http.Request) error {
if len(via) >= maxRedirects {
return fmt.Errorf("stopped after %d redirects", maxRedirects)
if len(via) >= 10 {
return fmt.Errorf("stopped after %d redirects", len(via))
}
if len(via) > 0 && via[0].URL.Scheme == "https" && req.URL.Scheme == "http" {
return errors.New("refused redirect from https to http")
@@ -100,36 +58,57 @@ func checkRedirect(req *http.Request, via []*http.Request) error {
return nil
}
// Do executes a request with automatic retries for transient errors.
// It sets the User-Agent header if not already present.
func (c *Client) Do(req *http.Request) (*http.Response, error) {
// Get performs a GET request with the Webi User-Agent header.
func Get(ctx context.Context, client *http.Client, url string) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", userAgent)
return client.Do(req)
}
// Do executes a request with automatic retries for transient errors (429,
// 502, 503, 504). Retries up to 3 times with exponential backoff and jitter.
// Respects Retry-After headers. Only retries GET and HEAD (idempotent).
//
// Sets the Webi User-Agent header if not already present.
func Do(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
if req.Header.Get("User-Agent") == "" {
req.Header.Set("User-Agent", c.userAgent)
req.Header.Set("User-Agent", userAgent)
}
// Only retry idempotent methods.
idempotent := req.Method == http.MethodGet || req.Method == http.MethodHead
const maxRetries = 3
var resp *http.Response
var err error
for attempt := 0; attempt <= c.retries; attempt++ {
for attempt := range maxRetries + 1 {
if attempt > 0 {
delay := c.backoff(attempt, resp)
select {
case <-req.Context().Done():
return nil, req.Context().Err()
case <-time.After(delay):
if !idempotent {
break
}
delay := backoff(attempt, resp)
timer := time.NewTimer(delay)
select {
case <-ctx.Done():
timer.Stop()
return nil, ctx.Err()
case <-timer.C:
}
// Close previous response body before retry.
if resp != nil {
resp.Body.Close()
}
}
resp, err = c.inner.Do(req)
resp, err = client.Do(req)
if err != nil {
// Retry on transient network errors.
if req.Context().Err() != nil {
return nil, req.Context().Err()
if ctx.Err() != nil {
return nil, ctx.Err()
}
continue
}
@@ -139,62 +118,37 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
}
}
// Exhausted retries — return whatever we have.
if err != nil {
return nil, fmt.Errorf("after %d retries: %w", c.retries, err)
return nil, fmt.Errorf("after %d retries: %w", maxRetries, err)
}
return resp, nil
}
// Get is a convenience wrapper around Do for GET requests.
func (c *Client) Get(ctx context.Context, url string) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
return c.Do(req)
}
// isRetryable returns true for HTTP status codes that indicate a transient error.
func isRetryable(status int) bool {
switch status {
case http.StatusTooManyRequests, // 429
http.StatusBadGateway, // 502
http.StatusServiceUnavailable, // 503
http.StatusGatewayTimeout: // 504
return true
}
return false
return status == http.StatusTooManyRequests ||
status == http.StatusBadGateway ||
status == http.StatusServiceUnavailable ||
status == http.StatusGatewayTimeout
}
// backoff calculates the delay before the next retry attempt.
// Uses exponential backoff with jitter, and respects Retry-After headers.
func (c *Client) backoff(attempt int, resp *http.Response) time.Duration {
// Check Retry-After header from previous response.
// backoff returns a delay before the next retry. Respects Retry-After,
// otherwise uses exponential backoff with jitter.
func backoff(attempt int, resp *http.Response) time.Duration {
if resp != nil {
if ra := resp.Header.Get("Retry-After"); ra != "" {
if seconds, err := strconv.Atoi(ra); err == nil {
d := time.Duration(seconds) * time.Second
if d > 0 && d < 5*time.Minute {
return d
}
if seconds, err := strconv.Atoi(ra); err == nil && seconds > 0 && seconds < 300 {
return time.Duration(seconds) * time.Second
}
}
}
// Exponential backoff: baseDelay * 2^attempt + jitter
delay := c.baseDelay
for i := 1; i < attempt; i++ {
delay *= 2
if delay > c.maxDelay {
delay = c.maxDelay
break
}
// 1s, 2s, 4s base delays
base := time.Second << (attempt - 1)
if base > 30*time.Second {
base = 30 * time.Second
}
// Add jitter: ±25%
jitter := time.Duration(float64(delay) * 0.5 * rand.Float64())
delay = delay + jitter - (delay / 4)
return delay
// Add jitter: 75% to 125% of base
jitter := float64(base) * (0.75 + 0.5*rand.Float64())
return time.Duration(jitter)
}

View File

@@ -5,10 +5,11 @@
// "Darwin 23.1.0 arm64" or "Linux 6.1.0 x86_64". This package parses those
// into [buildmeta.OS], [buildmeta.Arch], and [buildmeta.Libc] values so the
// server can select the correct release artifact.
//
// Also handles non-uname agents like "PowerShell/7.3.0" and "MS AMD64".
package uadetect
import (
"regexp"
"strings"
"github.com/webinstall/webi-installers/internal/buildmeta"
@@ -23,136 +24,180 @@ type Result struct {
// Parse extracts OS, arch, and libc from a User-Agent string.
func Parse(ua string) Result {
if ua == "-" {
return Result{}
}
tokens := tokenize(ua)
return Result{
OS: DetectOS(ua),
Arch: DetectArch(ua),
Libc: DetectLibc(ua),
OS: matchOS(tokens),
Arch: matchArch(tokens),
Libc: matchLibc(tokens),
}
}
// DetectOS returns the OS from a User-Agent string.
func DetectOS(ua string) buildmeta.OS {
if ua == "-" {
return ""
// tokenize splits a User-Agent into lowercase tokens for matching.
// Splits on whitespace, '/', and ';', since UAs come in various forms:
//
// "Darwin 23.1.0 arm64" (uname -srm)
// "PowerShell/7.3.0" (PowerShell)
// "MS AMD64" (Windows shorthand)
// "Macintosh; Intel Mac OS X 10_15_7" (browser)
func tokenize(ua string) []string {
// Strip xnu kernel info that can mislead arch detection under Rosetta.
// "xnu-7195.60.75~1/RELEASE_ARM64_T8101" contains ARM64 even when
// running as x86_64. This only appears in verbose uname output.
if i := strings.Index(ua, "xnu-"); i >= 0 {
end := strings.IndexByte(ua[i:], ' ')
if end < 0 {
ua = ua[:i]
} else {
ua = ua[:i] + ua[i+end:]
}
}
// Android must be tested before Linux.
if reAndroid.MatchString(ua) {
return strings.FieldsFunc(strings.ToLower(ua), func(r rune) bool {
return r == ' ' || r == '/' || r == ';' || r == '\t'
})
}
// matchOS identifies the operating system from tokens.
// Order matters: Android before Linux, Linux before Windows (for WSL).
func matchOS(tokens []string) buildmeta.OS {
has := func(s string) bool {
for _, t := range tokens {
if strings.Contains(t, s) {
return true
}
}
return false
}
// Android must be checked before Linux.
if has("android") {
return buildmeta.OSAndroid
}
// macOS/Darwin must be tested before Linux (for edge cases) and before
// "win" (because "darwin" contains no "win", but ordering matters).
if reDarwin.MatchString(ua) {
if has("darwin") || has("macos") || has("macintosh") || has("iphone") || has("ios") || has("ipad") {
return buildmeta.OSDarwin
}
// "mac" alone (not in "macintosh" which is already matched)
for _, t := range tokens {
if t == "mac" {
return buildmeta.OSDarwin
}
}
// Linux must be tested before Windows because WSL User-Agents contain
// both "Linux" and sometimes "Microsoft".
if reLinux.MatchString(ua) && !reCygwin.MatchString(ua) {
// Linux before Windows because WSL UAs contain both "linux" and "microsoft".
// But exclude Cygwin/msysgit which report Linux-like strings on Windows.
if has("linux") && !has("cygwin") && !has("msysgit") {
return buildmeta.OSLinux
}
if reWindows.MatchString(ua) {
if has("windows") || has("win32") || has("microsoft") || has("powershell") {
return buildmeta.OSWindows
}
for _, t := range tokens {
if t == "ms" || t == "win" {
return buildmeta.OSWindows
}
}
// Try Linux again after Windows (for plain "curl" or "wget").
if reLinuxLoose.MatchString(ua) {
// Fallback: curl and wget imply a POSIX system, almost always Linux.
if has("curl") || has("wget") {
return buildmeta.OSLinux
}
return ""
}
// DetectArch returns the CPU architecture from a User-Agent string.
func DetectArch(ua string) buildmeta.Arch {
if ua == "-" {
return ""
// matchArch identifies the CPU architecture from tokens.
// More specific patterns are checked before less specific ones.
func matchArch(tokens []string) buildmeta.Arch {
has := func(s string) bool {
for _, t := range tokens {
if strings.Contains(t, s) {
return true
}
}
return false
}
exact := func(s string) bool {
for _, t := range tokens {
if t == s {
return true
}
}
return false
}
// Strip macOS kernel release arch info that can mislead detection.
// e.g. "xnu-7195.60.75~1/RELEASE_ARM64_T8101 x86_64" under Rosetta
ua = reXNU.ReplaceAllString(ua, "")
// Order matters — more specific patterns first.
if reARM64.MatchString(ua) {
// ARM 64-bit (most specific first)
if has("aarch64") || has("arm64") || has("armv8") {
return buildmeta.ArchARM64
}
if reARMv7.MatchString(ua) {
// ARM 32-bit variants
if has("armv7") || has("arm32") {
return buildmeta.ArchARMv7
}
if reARMv6.MatchString(ua) {
if has("armv6") {
return buildmeta.ArchARMv6
}
if rePPC64LE.MatchString(ua) {
// Bare "arm" without a version qualifier → armv6 (conservative).
if exact("arm") {
return buildmeta.ArchARMv6
}
// POWER (check before generic 64-bit)
if has("ppc64le") {
return buildmeta.ArchPPC64LE
}
if rePPC64.MatchString(ua) {
if has("ppc64") {
return buildmeta.ArchPPC64
}
if reMIPS64.MatchString(ua) {
// MIPS (check before generic 64-bit)
if has("mips64") {
return buildmeta.ArchMIPS64
}
if reMIPS.MatchString(ua) {
if has("mips") {
return buildmeta.ArchMIPS
}
// amd64 must come after ppc64/mips64 (both contain "64").
if reAMD64.MatchString(ua) {
// x86-64
if has("x86_64") || has("amd64") || exact("x64") {
return buildmeta.ArchAMD64
}
// x86 must come after x86_64/amd64.
if reX86.MatchString(ua) {
// x86 32-bit (after x86_64 to avoid false match)
if has("i386") || has("i686") || exact("x86") {
return buildmeta.ArchX86
}
return ""
}
// DetectLibc returns the C library variant from a User-Agent string.
func DetectLibc(ua string) buildmeta.Libc {
if ua == "-" {
return ""
// matchLibc identifies the C library from tokens.
func matchLibc(tokens []string) buildmeta.Libc {
has := func(s string) bool {
for _, t := range tokens {
if strings.Contains(t, s) {
return true
}
}
return false
}
lower := strings.ToLower(ua)
if reMusl.MatchString(lower) {
if has("musl") {
return buildmeta.LibcMusl
}
if reMSVC.MatchString(lower) {
if has("msvc") || has("windows") || has("microsoft") {
return buildmeta.LibcMSVC
}
if reGNU.MatchString(lower) {
if has("gnu") || has("glibc") || has("linux") {
return buildmeta.LibcGNU
}
// Default: no specific libc requirement detected.
return buildmeta.LibcNone
}
// Compiled regexes — allocated once.
var (
reAndroid = regexp.MustCompile(`(?i)Android`)
reDarwin = regexp.MustCompile(`(?i)iOS|iPhone|Macintosh|Darwin|OS\s*X|macOS|mac`)
reLinux = regexp.MustCompile(`(?i)Linux`)
reCygwin = regexp.MustCompile(`(?i)cygwin|msysgit`)
reWindows = regexp.MustCompile(`(?i)(\b|^)ms(\b|$)|Microsoft|Windows|win32|win|PowerShell`)
reLinuxLoose = regexp.MustCompile(`(?i)Linux|curl|wget`)
reXNU = regexp.MustCompile(`xnu-\S*RELEASE_\S*`)
reARM64 = regexp.MustCompile(`(?i)(\b|_)(aarch64|arm64|arm8|armv8)`)
reARMv7 = regexp.MustCompile(`(?i)(\b|_)(aarch|arm7|armv7|arm32)`)
reARMv6 = regexp.MustCompile(`(?i)(\b|_)(arm6|armv6|arm(\b|_))`)
rePPC64LE = regexp.MustCompile(`(?i)ppc64le`)
rePPC64 = regexp.MustCompile(`(?i)ppc64`)
reMIPS64 = regexp.MustCompile(`(?i)mips64`)
reMIPS = regexp.MustCompile(`(?i)mips`)
reAMD64 = regexp.MustCompile(`(?i)(amd64|x86_64|x64|_64)\b`)
reX86 = regexp.MustCompile(`(?i)(\b|_)(3|6|x|_)86\b`)
reMusl = regexp.MustCompile(`(\b|_)musl(\b|_)`)
reMSVC = regexp.MustCompile(`(\b|_)(msvc|windows|microsoft)(\b|_)`)
reGNU = regexp.MustCompile(`(\b|_)(gnu|glibc|linux)(\b|_)`)
)

View File

@@ -7,21 +7,18 @@ import (
"github.com/webinstall/webi-installers/internal/uadetect"
)
func TestDetectOS(t *testing.T) {
func TestOS(t *testing.T) {
tests := []struct {
ua string
want buildmeta.OS
}{
// macOS / Darwin
// uname -srm style
{"Darwin 23.1.0 arm64", buildmeta.OSDarwin},
{"Darwin 20.2.0 x86_64", buildmeta.OSDarwin},
{"Macintosh; Intel Mac OS X 10_15_7", buildmeta.OSDarwin},
// Linux
{"Linux 6.1.0-18-amd64 x86_64", buildmeta.OSLinux},
{"Linux 5.15.0 aarch64", buildmeta.OSLinux},
// WSL (Linux, not Windows)
// WSL: Linux, not Windows (contains "microsoft" in kernel release)
{"Linux 5.15.146.1-microsoft-standard-WSL2 x86_64", buildmeta.OSLinux},
// Windows
@@ -29,28 +26,31 @@ func TestDetectOS(t *testing.T) {
{"PowerShell/7.3.0", buildmeta.OSWindows},
{"Microsoft Windows 10.0.19045", buildmeta.OSWindows},
// Android
// Android before Linux
{"Android 13 aarch64", buildmeta.OSAndroid},
// Minimal agents
// Browser-style
{"Macintosh; Intel Mac OS X 10_15_7", buildmeta.OSDarwin},
// Minimal agents → assume Linux
{"curl/8.1.2", buildmeta.OSLinux},
{"wget/1.21", buildmeta.OSLinux},
// Dash means unknown
// Explicit unknown
{"-", ""},
}
for _, tt := range tests {
t.Run(tt.ua, func(t *testing.T) {
got := uadetect.DetectOS(tt.ua)
got := uadetect.Parse(tt.ua).OS
if got != tt.want {
t.Errorf("DetectOS(%q) = %q, want %q", tt.ua, got, tt.want)
t.Errorf("Parse(%q).OS = %q, want %q", tt.ua, got, tt.want)
}
})
}
}
func TestDetectArch(t *testing.T) {
func TestArch(t *testing.T) {
tests := []struct {
ua string
want buildmeta.Arch
@@ -63,7 +63,7 @@ func TestDetectArch(t *testing.T) {
{"Linux 5.10.0 armv6l", buildmeta.ArchARMv6},
{"Linux 5.4.0 ppc64le", buildmeta.ArchPPC64LE},
// Rosetta: kernel says ARM64 but uname reports x86_64
// Rosetta: xnu kernel info says ARM64 but actual arch is x86_64
{"Darwin 20.2.0 Darwin Kernel Version 20.2.0; root:xnu-7195.60.75~1/RELEASE_ARM64_T8101 x86_64", buildmeta.ArchAMD64},
{"-", ""},
@@ -71,15 +71,15 @@ func TestDetectArch(t *testing.T) {
for _, tt := range tests {
t.Run(tt.ua, func(t *testing.T) {
got := uadetect.DetectArch(tt.ua)
got := uadetect.Parse(tt.ua).Arch
if got != tt.want {
t.Errorf("DetectArch(%q) = %q, want %q", tt.ua, got, tt.want)
t.Errorf("Parse(%q).Arch = %q, want %q", tt.ua, got, tt.want)
}
})
}
}
func TestDetectLibc(t *testing.T) {
func TestLibc(t *testing.T) {
tests := []struct {
ua string
want buildmeta.Libc
@@ -95,15 +95,15 @@ func TestDetectLibc(t *testing.T) {
for _, tt := range tests {
t.Run(tt.ua, func(t *testing.T) {
got := uadetect.DetectLibc(tt.ua)
got := uadetect.Parse(tt.ua).Libc
if got != tt.want {
t.Errorf("DetectLibc(%q) = %q, want %q", tt.ua, got, tt.want)
t.Errorf("Parse(%q).Libc = %q, want %q", tt.ua, got, tt.want)
}
})
}
}
func TestParse(t *testing.T) {
func TestFullParse(t *testing.T) {
r := uadetect.Parse("Darwin 23.1.0 arm64")
if r.OS != buildmeta.OSDarwin {
t.Errorf("OS = %q, want %q", r.OS, buildmeta.OSDarwin)