fix(mariadb-galera): remove obsolete package

Galera is bundled into MariaDB 10.1+ — no separate downloads exist.
The MariaDB REST API confirms this: "There are no longer separate
MariaDB Galera Cluster releases for MariaDB 10.1 and above."

cmd/webicached/main.go

@@ -35,6 +35,7 @@ import (
 
 	"github.com/joho/godotenv"
 	"github.com/webinstall/webi-installers/internal/classifypkg"
+	"github.com/webinstall/webi-installers/internal/httpclient"
 	"github.com/webinstall/webi-installers/internal/installerconf"
 	"github.com/webinstall/webi-installers/internal/rawcache"
 	"github.com/webinstall/webi-installers/internal/releases/chromedist"
@@ -166,10 +167,10 @@ func main() {
 		auth = &githubish.Auth{Token: cfg.token}
 	}
 
-	client := &http.Client{Timeout: 30 * time.Second}
+	client := httpclient.New()
 	if cfg.pageDelay > 0 {
 		client.Transport = &delayTransport{
-			base:  http.DefaultTransport,
+			base:  client.Transport,
 			delay: cfg.pageDelay,
 		}
 	}
@@ -206,11 +207,11 @@ func main() {
 	if err != nil {
 		log.Fatalf("discover: %v", err)
 	}
+	nameSet := make(map[string]bool, len(filterPkgs))
+	for _, a := range filterPkgs {
+		nameSet[a] = true
+	}
 	if len(filterPkgs) > 0 {
-		nameSet := make(map[string]bool, len(filterPkgs))
-		for _, a := range filterPkgs {
-			nameSet[a] = true
-		}
 		var filtered []pkgConf
 		for _, p := range packages {
 			if nameSet[p.name] {
@@ -227,8 +228,41 @@ func main() {
 		}
 	}
 
+	// rescanNew appends any conf files added since the last scan.
+	// Returns true when at least one new package was added so the caller
+	// can restart the batch loop and process new packages immediately.
+	rescanNew := func() bool {
+		discovered, err := discover(wc.ConfDir)
+		if err != nil {
+			log.Printf("rescan: %v", err)
+			return false
+		}
+		known := make(map[string]bool, len(real))
+		for _, p := range real {
+			known[p.name] = true
+		}
+		added := false
+		for _, p := range discovered {
+			if p.conf.AliasOf != "" || known[p.name] {
+				continue
+			}
+			if len(filterPkgs) > 0 && !nameSet[p.name] {
+				continue
+			}
+			log.Printf("discovered new package: %s (source=%s)", p.name, p.conf.Source)
+			real = append(real, p)
+			added = true
+		}
+		return added
+	}
+
 	log.Printf("refreshing %d packages, interval %s, batch size 20 (ctrl-c to stop)", len(real), cfg.interval)
 	for {
+		// Rescan before computing staleness so newly added conf files are
+		// included immediately. New packages have a zero timestamp and sort
+		// to the front of the stale list, so they are processed next.
+		rescanNew()
+
 		stale := wc.stalest(real)
 		if len(stale) == 0 {
 			log.Printf("all packages fresh, sleeping %s", cfg.interval)
@@ -252,6 +286,10 @@ func main() {
 			}
 			cancel()
 			time.Sleep(cfg.interval)
+			// Rescan mid-batch so new packages preempt remaining batch items.
+			if rescanNew() {
+				break
+			}
 		}
 	}
 }
@@ -294,14 +332,13 @@ func (wc *WebiCache) stalest(packages []pkgConf) []pkgConf {
 	for _, pkg := range packages {
 		data, err := wc.Store.Load(ctx, pkg.name)
 		var t time.Time
-		hasAssets := false
 		if err == nil && data != nil {
 			t = data.UpdatedAt
-			hasAssets = len(data.Assets) > 0
 		}
-		// Never fetched, or has no assets despite having a timestamp
-		// (e.g. classified from empty rawcache), or older than 10 minutes.
-		if t.IsZero() || !hasAssets || time.Since(t) > 10*time.Minute {
+		// Never fetched, or older than 10 minutes.
+		// 0-asset results are not treated as perpetually stale — packages that
+		// produce no classifiable assets (e.g. galera) respect the timestamp.
+		if t.IsZero() || time.Since(t) > 10*time.Minute {
 			stale = append(stale, stamped{pkg: pkg, updatedAt: t})
 		}
 	}
@@ -531,7 +568,14 @@ func (wc *WebiCache) fetchRaw(ctx context.Context, pkg pkgConf, shallow bool) er
 	// commit hashes. Git entries are classified from this data in
 	// refreshPackage, not from the main raw cache.
 	if pkg.conf.GitURL != "" && pkg.conf.Source != "gittag" {
-		if err := wc.fetchGitTagSupplementary(ctx, pkg.name, pkg.conf.GitURL, shallow); err != nil {
+		gitShallow := shallow
+		if !wc.Shallow {
+			gd, gdErr := rawcache.Open(filepath.Join(wc.RawDir, "_gittag", pkg.name))
+			if gdErr == nil && !gd.Populated() {
+				gitShallow = false
+			}
+		}
+		if err := wc.fetchGitTagSupplementary(ctx, pkg.name, pkg.conf.GitURL, gitShallow); err != nil {
 			log.Printf("  %s: supplementary gittag fetch: %v", pkg.name, err)
 		}
 	}

ffmpeg/releases.conf

@@ -1,3 +1,4 @@
+source = ffmpegdist
 github_releases = eugeneware/ffmpeg-static
 asset_filter = ffmpeg
 version_prefix = b

internal/classifypkg/classifypkg.go

@@ -166,6 +166,8 @@ func classifySource(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]st
 		return classifyMariaDBDist(d)
 	case "zigdist":
 		return classifyZigDist(d)
+	case "ffmpegdist":
+		return classifyFFmpegDist(d)
 	default:
 		return nil, nil
 	}
@@ -464,6 +466,86 @@ func classifyGitHub(pkg string, conf *installerconf.Conf, d *rawcache.Dir) ([]st
 	return assets, nil
 }
 
+var ffmpegOSMap = map[string]string{
+	"linux":  "linux",
+	"darwin": "darwin",
+	"win32":  "windows",
+}
+
+var ffmpegArchMap = map[string]string{
+	"x64":   "x86_64",
+	"ia32":  "x86",
+	"arm64": "aarch64",
+	"arm":   "armv7",
+}
+
+// classifyFFmpegDist handles eugeneware/ffmpeg-static releases.
+// Upstream uses non-standard names (x64, ia32, win32, arm) and ships both
+// bare binaries and .gz-compressed copies. Only bare binaries are kept —
+// the install template has no handler for single-file .gz extraction.
+func classifyFFmpegDist(d *rawcache.Dir) ([]storage.Asset, error) {
+	releases, err := ReadAllRaw(d)
+	if err != nil {
+		return nil, err
+	}
+
+	var assets []storage.Asset
+	for _, data := range releases {
+		var rel ghRelease
+		if err := json.Unmarshal(data, &rel); err != nil {
+			continue
+		}
+		if rel.Draft {
+			continue
+		}
+
+		version := strings.TrimPrefix(rel.TagName, "b")
+
+		channel := "stable"
+		if rel.Prerelease {
+			channel = "beta"
+		}
+
+		date := ""
+		if len(rel.PublishedAt) >= 10 {
+			date = rel.PublishedAt[:10]
+		}
+
+		for _, a := range rel.Assets {
+			if strings.Contains(a.Name, ".") {
+				continue
+			}
+			if !strings.HasPrefix(a.Name, "ffmpeg-") {
+				continue
+			}
+
+			parts := strings.SplitN(a.Name, "-", 3)
+			if len(parts) != 3 {
+				continue
+			}
+
+			os, osOK := ffmpegOSMap[parts[1]]
+			arch, archOK := ffmpegArchMap[parts[2]]
+			if !osOK || !archOK {
+				continue
+			}
+
+			assets = append(assets, storage.Asset{
+				Filename: a.Name,
+				Version:  version,
+				Channel:  channel,
+				OS:       os,
+				Arch:     arch,
+				Format:   "",
+				Download: a.BrowserDownloadURL,
+				Date:     date,
+			})
+		}
+	}
+
+	return assets, nil
+}
+
 // classifyServiceman handles serviceman's dual-repo layout: binary releases
 // from therootcompany/serviceman (≤v0.8.x) and source-only releases from
 // bnnanet/serviceman (v0.9.x+). Emits binary assets where available, plus

internal/httpclient/httpclient.go

@@ -0,0 +1,154 @@
+// Package httpclient provides a well-configured [http.Client] for upstream
+// API calls. It exists because [http.DefaultClient] has no timeouts, no TLS
+// minimum, and follows redirects from HTTPS to HTTP — none of which are
+// acceptable for a server calling GitHub, Gitea, etc. on behalf of users.
+//
+// Use [New] to create a configured client. Use [Do] to execute a request
+// with automatic retries for transient failures.
+package httpclient
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"math/rand/v2"
+	"net"
+	"net/http"
+	"strconv"
+	"time"
+)
+
+const userAgent = "Webi/2.0 (+https://webinstall.dev)"
+
+// New returns an [http.Client] with secure, production-ready defaults:
+// TLS 1.2+, timeouts at every level, connection pooling, no HTTPS→HTTP
+// redirect, and a Webi User-Agent.
+func New() *http.Client {
+	return &http.Client{
+		Transport: &http.Transport{
+			DialContext: (&net.Dialer{
+				Timeout:   10 * time.Second,
+				KeepAlive: 30 * time.Second,
+			}).DialContext,
+			TLSClientConfig: &tls.Config{
+				MinVersion: tls.VersionTLS12,
+			},
+			TLSHandshakeTimeout:  10 * time.Second,
+			ResponseHeaderTimeout: 30 * time.Second,
+			MaxIdleConns:          100,
+			MaxIdleConnsPerHost:   10,
+			IdleConnTimeout:       90 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+			ForceAttemptHTTP2:     true,
+		},
+		Timeout:       60 * time.Second,
+		CheckRedirect: checkRedirect,
+	}
+}
+
+// checkRedirect prevents HTTPS→HTTP downgrades and limits redirect depth.
+func checkRedirect(req *http.Request, via []*http.Request) error {
+	if len(via) >= 10 {
+		return fmt.Errorf("stopped after %d redirects", len(via))
+	}
+	if len(via) > 0 && via[0].URL.Scheme == "https" && req.URL.Scheme == "http" {
+		return errors.New("refused redirect from https to http")
+	}
+	return nil
+}
+
+// Get performs a GET request with the Webi User-Agent header.
+func Get(ctx context.Context, client *http.Client, url string) (*http.Response, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", userAgent)
+	return client.Do(req)
+}
+
+// Do executes a request with automatic retries for transient errors (429,
+// 502, 503, 504). Retries up to 3 times with exponential backoff and jitter.
+// Respects Retry-After headers. Only retries GET and HEAD (idempotent).
+//
+// Sets the Webi User-Agent header if not already present.
+func Do(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
+	if req.Header.Get("User-Agent") == "" {
+		req.Header.Set("User-Agent", userAgent)
+	}
+
+	// Only retry idempotent methods.
+	idempotent := req.Method == http.MethodGet || req.Method == http.MethodHead
+
+	const maxRetries = 3
+	var resp *http.Response
+	var err error
+
+	for attempt := range maxRetries + 1 {
+		if attempt > 0 {
+			if !idempotent {
+				break
+			}
+
+			delay := backoff(attempt, resp)
+			timer := time.NewTimer(delay)
+			select {
+			case <-ctx.Done():
+				timer.Stop()
+				return nil, ctx.Err()
+			case <-timer.C:
+			}
+
+			if resp != nil {
+				resp.Body.Close()
+			}
+		}
+
+		resp, err = client.Do(req)
+		if err != nil {
+			if ctx.Err() != nil {
+				return nil, ctx.Err()
+			}
+			continue
+		}
+
+		if !isRetryable(resp.StatusCode) {
+			return resp, nil
+		}
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("after %d retries: %w", maxRetries, err)
+	}
+	return resp, nil
+}
+
+func isRetryable(status int) bool {
+	return status == http.StatusTooManyRequests ||
+		status == http.StatusBadGateway ||
+		status == http.StatusServiceUnavailable ||
+		status == http.StatusGatewayTimeout
+}
+
+// backoff returns a delay before the next retry. Respects Retry-After,
+// otherwise uses exponential backoff with jitter.
+func backoff(attempt int, resp *http.Response) time.Duration {
+	if resp != nil {
+		if ra := resp.Header.Get("Retry-After"); ra != "" {
+			if seconds, err := strconv.Atoi(ra); err == nil && seconds > 0 && seconds < 300 {
+				return time.Duration(seconds) * time.Second
+			}
+		}
+	}
+
+	// 1s, 2s, 4s base delays
+	base := time.Second << (attempt - 1)
+	if base > 30*time.Second {
+		base = 30 * time.Second
+	}
+
+	// Add jitter: 75% to 125% of base
+	jitter := float64(base) * (0.75 + 0.5*rand.Float64())
+	return time.Duration(jitter)
+}

internal/installerconf/installerconf.go

@@ -162,6 +162,8 @@ func Read(path string) (*Conf, error) {
 	c := &Conf{}
 
 	// Infer source from primary key, falling back to explicit "source".
+	// When both github_releases and source are set, parse the repo ref
+	// from github_releases but use the explicit source for classification.
 	switch {
 	// GitHub binary releases.
 	case raw["github_releases"] != "":
@@ -213,6 +215,13 @@ func Read(path string) (*Conf, error) {
 	default:
 	}
 
+	// Explicit "source" overrides the inferred source when both are present.
+	// This lets packages like ffmpeg use github_releases for fetching but
+	// a custom classifier for classification.
+	if raw["source"] != "" && c.Source != "" {
+		c.Source = raw["source"]
+	}
+
 	// git_url can appear alongside any source type (e.g. github_sources)
 	// to provide a git clone fallback. When it's the only key, it's the
 	// primary source (gittag).

mariadb-galera/releases.conf

@@ -1,2 +0,0 @@
-source = mariadbdist
-asset_filter = galera