Compare commits

...

7 Commits

Author SHA1 Message Date
AJ ONeal
d67ac8d9f2 feat(webicached,webid): add pgstore backend and --pg flag
Adds an optional PostgreSQL storage backend for webicached and webid.
When --pg is given a DSN, the daemon writes/reads classified assets to/from
Postgres instead of the legacy fsstore. Brings in pgx/v5 as a direct dependency.
2026-05-16 21:58:59 -06:00
AJ ONeal
b79c5529e1 chore(deps): upgrade pgx v5.8.0 → v5.9.2 2026-05-16 21:58:49 -06:00
AJ ONeal
ac59e4728e feat(webid): add HTTP API server (releases, resolve, installer scripts)
Serves the HTTP API for webinstall.dev:
- GET /api/releases/{pkg}.json — classified release list from fsstore
- GET /v1/resolve/{pkg} — resolve OS/arch/version to a specific asset
- GET /api/installers/{pkg}.sh — render installer shell script
- GET /api/installers/{pkg}.ps1 — render installer PowerShell script

Git-clone packages include git_tag and git_commit_hash in responses.
UA detection infers OS/arch from User-Agent when not query-specified.
2026-05-16 21:53:05 -06:00
AJ ONeal
59b2956d60 feat(webid): add UA detection and platform resolve packages 2026-05-16 21:53:05 -06:00
AJ ONeal
75bd1a3cf9 feat(resolver): add platform/version resolver for webid 2026-05-16 21:53:05 -06:00
AJ ONeal
89e12d22f3 feat(render): add installer script renderer for webid 2026-05-16 21:53:05 -06:00
AJ ONeal
23064a6db7 fix(webicached): don't treat 0-asset packages as perpetually stale
Packages that produce no classifiable assets (e.g. mariadb-galera with
the galera asset_filter) were being refetched every batch because
!hasAssets marked them stale regardless of timestamp. The hasAssets
condition was intended for the startup case (classified from empty
rawcache), but those packages are already caught by t.IsZero() on first
run. Respect the timestamp for 0-asset results as for any other package.
2026-05-16 21:53:01 -06:00
20 changed files with 4634 additions and 10 deletions

View File

@@ -17,6 +17,7 @@ $Env:WEBI_HOST = 'https://webinstall.dev'
#$Env:PKG_NAME = node
#$Env:WEBI_VERSION = v12.16.2
#$Env:WEBI_GIT_TAG = 12.16.2
#$Env:WEBI_GIT_COMMIT_HASH =
#$Env:WEBI_PKG_URL = "https://.../node-....zip"
#$Env:WEBI_PKG_FILE = "node-v12.16.2-win-x64.zip"
#$Env:WEBI_PKG_PATHNAME = "node-v12.16.2-win-x64.zip"

View File

@@ -15,6 +15,7 @@ __bootstrap_webi() {
# TODO not sure if BUILD is the best name for this
#WEBI_BUILD=
#WEBI_GIT_TAG=
#WEBI_GIT_COMMIT_HASH=
#WEBI_LTS=
#WEBI_CHANNEL=
#WEBI_EXT=

View File

@@ -55,6 +55,7 @@ import (
"github.com/webinstall/webi-installers/internal/releases/zigdist"
"github.com/webinstall/webi-installers/internal/storage"
"github.com/webinstall/webi-installers/internal/storage/fsstore"
"github.com/webinstall/webi-installers/internal/storage/pgstore"
)
var (
@@ -78,6 +79,7 @@ type MainConfig struct {
envFile string
confDir string
cacheDir string
pgDSN string
rawDir string
token string
once bool
@@ -156,11 +158,20 @@ func main() {
cfg.token = os.Getenv("GITHUB_TOKEN")
}
fss, err := fsstore.New(cfg.cacheDir)
if err != nil {
log.Fatalf("fsstore: %v", err)
var store storage.Store
if cfg.pgDSN != "" {
pg, err := pgstore.New(context.Background(), cfg.pgDSN)
if err != nil {
log.Fatalf("pgstore: %v", err)
}
store = pg
} else {
fss, err := fsstore.New(cfg.cacheDir)
if err != nil {
log.Fatalf("fsstore: %v", err)
}
store = fss
}
var store storage.Store = fss
var auth *githubish.Auth
if cfg.token != "" {
@@ -298,6 +309,7 @@ func registerFlags(fs *flag.FlagSet, cfg *MainConfig) {
fs.StringVar(&cfg.envFile, "envfile", "", "path to .env file to load before running")
fs.StringVar(&cfg.confDir, "conf", ".", "root directory containing {pkg}/releases.conf files")
fs.StringVar(&cfg.cacheDir, "legacy", "~/.cache/webi/legacy", "legacy cache directory (fsstore root)")
fs.StringVar(&cfg.pgDSN, "pg", "", "PostgreSQL DSN (enables pgstore; mutually exclusive with -legacy)")
fs.StringVar(&cfg.rawDir, "raw", "~/.cache/webi/raw", "raw cache directory for upstream responses")
fs.StringVar(&cfg.token, "token", "", "GitHub API token (or set $GITHUB_TOKEN)")
fs.BoolVar(&cfg.once, "once", false, "run once then exit (no periodic refresh)")
@@ -332,14 +344,13 @@ func (wc *WebiCache) stalest(packages []pkgConf) []pkgConf {
for _, pkg := range packages {
data, err := wc.Store.Load(ctx, pkg.name)
var t time.Time
hasAssets := false
if err == nil && data != nil {
t = data.UpdatedAt
hasAssets = len(data.Assets) > 0
}
// Never fetched, or has no assets despite having a timestamp
// (e.g. classified from empty rawcache), or older than 10 minutes.
if t.IsZero() || !hasAssets || time.Since(t) > 10*time.Minute {
// Never fetched, or older than 10 minutes.
// 0-asset results are not treated as perpetually stale — packages that
// produce no classifiable assets (e.g. galera) respect the timestamp.
if t.IsZero() || time.Since(t) > 10*time.Minute {
stale = append(stale, stamped{pkg: pkg, updatedAt: t})
}
}

146
cmd/webid/bootstrap_test.go Normal file
View File

@@ -0,0 +1,146 @@
package main
import (
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
// TestBootstrapCurlPipe verifies the /{pkg} route returns the curl-pipe bootstrap.
func TestBootstrapCurlPipe(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/bat@stable")
if code != 200 {
t.Fatalf("status %d: %s", code, body[:min(len(body), 200)])
}
// Should contain the bootstrap env vars.
if !strings.Contains(body, "WEBI_PKG=") {
t.Error("missing WEBI_PKG= in bootstrap")
}
if !strings.Contains(body, "WEBI_HOST=") {
t.Error("missing WEBI_HOST= in bootstrap")
}
if !strings.Contains(body, "WEBI_CHECKSUM=") {
t.Error("missing WEBI_CHECKSUM= in bootstrap")
}
// Should NOT contain the full installer (install.sh content).
// The bootstrap just downloads and runs webi.
if strings.Contains(body, "pkg_install()") {
t.Error("bootstrap should not contain pkg_install — that's the full installer")
}
t.Logf("bootstrap size: %d bytes", len(body))
}
// TestInstallerFull verifies /api/installers/{pkg}.sh returns the full installer.
func TestInstallerFull(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
// Use a webi-style User-Agent so the server can detect platform.
code, body := getWithUA(t, ts, "/api/installers/bat@stable.sh", "aarch64/unknown Darwin/24.2.0 libc")
if code != 200 {
t.Fatalf("status %d: %s", code, body[:min(len(body), 500)])
}
// Should contain resolved release info.
if !strings.Contains(body, "WEBI_VERSION=") {
t.Error("missing WEBI_VERSION= in installer")
}
if !strings.Contains(body, "WEBI_PKG_URL=") {
t.Error("missing WEBI_PKG_URL= in installer")
}
if !strings.Contains(body, "PKG_NAME=") {
t.Error("missing PKG_NAME= in installer")
}
// Should contain the package's install.sh content (embedded).
if !strings.Contains(body, "pkg_") {
t.Error("installer should contain pkg_ functions from install.sh")
}
t.Logf("installer size: %d bytes", len(body))
}
// TestInstallerPowerShell verifies /api/installers/{pkg}.ps1 returns a PowerShell installer.
func TestInstallerPowerShell(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "node"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := getWithUA(t, ts, "/api/installers/node@stable.ps1", "AMD64/unknown Windows/10.0.19045 msvc")
if code != 200 {
t.Fatalf("status %d: %s", code, body[:min(len(body), 500)])
}
if !strings.Contains(body, "$Env:WEBI_VERSION") {
t.Error("missing $Env:WEBI_VERSION in PS1 installer")
}
if !strings.Contains(body, "$Env:WEBI_PKG_URL") {
t.Error("missing $Env:WEBI_PKG_URL in PS1 installer")
}
if !strings.Contains(body, "$Env:PKG_NAME") {
t.Error("missing $Env:PKG_NAME in PS1 installer")
}
t.Logf("PS1 installer size: %d bytes", len(body))
}
// TestInstallerSelfHosted verifies selfhosted packages get a script without resolution.
func TestInstallerSelfHosted(t *testing.T) {
_, ts := newTestServer(t)
// ssh-utils is selfhosted — has install.sh but no releases.conf.
code, body := getWithUA(t, ts, "/api/installers/ssh-utils.sh", "aarch64/unknown Darwin/24.2.0 libc")
if code == 404 {
t.Skip("ssh-utils not available as installer")
}
if code != 200 {
t.Skipf("status %d (selfhosted may not render without cache): %s", code, body[:min(len(body), 200)])
}
t.Logf("selfhosted installer size: %d bytes", len(body))
}
// TestBootstrapUnknownPackage verifies 404 for unknown packages.
func TestBootstrapUnknownPackage(t *testing.T) {
_, ts := newTestServer(t)
code, _ := get(t, ts, "/nonexistent-package-xyz")
if code != 404 {
t.Errorf("expected 404, got %d", code)
}
}
// getWithUA fetches a URL with a custom User-Agent header.
func getWithUA(t *testing.T, ts *httptest.Server, path, ua string) (int, string) {
t.Helper()
req, err := http.NewRequest("GET", ts.URL+path, nil)
if err != nil {
t.Fatalf("new request: %v", err)
}
req.Header.Set("User-Agent", ua)
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("GET %s: %v", path, err)
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
return resp.StatusCode, string(body)
}

1000
cmd/webid/main.go Normal file

File diff suppressed because it is too large Load Diff

298
cmd/webid/main_test.go Normal file
View File

@@ -0,0 +1,298 @@
package main
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/webinstall/webi-installers/internal/resolve"
"github.com/webinstall/webi-installers/internal/storage"
"github.com/webinstall/webi-installers/internal/storage/fsstore"
)
// newTestServer creates a server backed by the real _cache directory
// and returns an httptest.Server with proper routing (so PathValue works).
func newTestServer(t *testing.T) (*server, *httptest.Server) {
t.Helper()
cacheDir := filepath.Join("..", "..", "_cache")
if _, err := os.Stat(cacheDir); err != nil {
t.Skipf("no cache dir at %s", cacheDir)
}
store, err := fsstore.New(cacheDir)
if err != nil {
t.Fatalf("fsstore: %v", err)
}
srv := &server{
store: store,
installersDir: filepath.Join("..", ".."),
packages: make(map[string]*packageCache),
}
// Load packages.
monthDir := time.Now().Format("2006-01")
dir := filepath.Join(store.Root(), monthDir)
entries, err := os.ReadDir(dir)
if err != nil {
t.Fatalf("readdir: %v", err)
}
for _, e := range entries {
if !strings.HasSuffix(e.Name(), ".json") {
continue
}
pkg := strings.TrimSuffix(e.Name(), ".json")
pd, err := store.Load(context.Background(), pkg)
if err != nil || pd == nil || len(pd.Assets) == 0 {
continue
}
pc := &packageCache{
assets: pd.Assets,
dists: assetsToDists(pd.Assets),
}
pc.catalog = resolve.Survey(pc.dists)
srv.packages[pkg] = pc
}
mux := http.NewServeMux()
mux.HandleFunc("GET /api/releases/{rest...}", srv.handleReleasesAPI)
mux.HandleFunc("GET /v1/releases/{rest...}", srv.handleV1Releases)
mux.HandleFunc("GET /v1/resolve/{rest...}", srv.handleV1Resolve)
mux.HandleFunc("GET /api/installers/{rest...}", srv.handleInstaller)
mux.HandleFunc("GET /api/debug", srv.handleDebug)
mux.HandleFunc("GET /{pkgSpec}", srv.handleBootstrap)
ts := httptest.NewServer(mux)
t.Cleanup(ts.Close)
return srv, ts
}
// get fetches a URL from the test server and returns the body.
func get(t *testing.T, ts *httptest.Server, path string) (int, string) {
t.Helper()
resp, err := http.Get(ts.URL + path)
if err != nil {
t.Fatalf("GET %s: %v", path, err)
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
return resp.StatusCode, string(body)
}
// TestLegacyJSONFormat verifies our JSON output matches the production format.
func TestLegacyJSONFormat(t *testing.T) {
srv, ts := newTestServer(t)
packages := []string{"bat", "node", "go", "jq"}
for _, pkg := range packages {
t.Run(pkg, func(t *testing.T) {
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/api/releases/"+pkg+".json?limit=5")
if code != http.StatusOK {
t.Fatalf("status %d: %s", code, body)
}
body = strings.TrimSpace(body)
// Must be a JSON array, not an object.
if !strings.HasPrefix(body, "[") {
t.Fatalf("expected JSON array, got: %.100s", body)
}
var releases []legacyRelease
if err := json.Unmarshal([]byte(body), &releases); err != nil {
t.Fatalf("decode: %v", err)
}
if len(releases) == 0 {
t.Fatal("no releases returned")
}
// Check field format conventions.
for i, r := range releases {
if strings.HasPrefix(r.Version, "v") {
t.Errorf("release[%d]: version %q should not have v prefix", i, r.Version)
}
if strings.HasPrefix(r.Ext, ".") {
t.Errorf("release[%d]: ext %q should not have . prefix", i, r.Ext)
}
if r.OS == "darwin" {
t.Errorf("release[%d]: os should be 'macos' not 'darwin'", i)
}
if r.Arch == "x86_64" {
t.Errorf("release[%d]: arch should be 'amd64' not 'x86_64'", i)
}
if r.Arch == "aarch64" {
t.Errorf("release[%d]: arch should be 'arm64' not 'aarch64'", i)
}
if r.Libc == "" {
t.Errorf("release[%d]: libc should be 'none' not empty", i)
}
if r.Download == "" {
t.Errorf("release[%d]: download URL is empty", i)
}
}
})
}
}
// TestLegacyTabFormat verifies our .tab output uses real TSV.
func TestLegacyTabFormat(t *testing.T) {
srv, ts := newTestServer(t)
packages := []string{"bat", "node", "go"}
for _, pkg := range packages {
t.Run(pkg, func(t *testing.T) {
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/api/releases/"+pkg+".tab?limit=3")
if code != http.StatusOK {
t.Fatalf("status %d: %s", code, body)
}
lines := strings.Split(strings.TrimSpace(body), "\n")
if len(lines) == 0 {
t.Fatal("no lines returned")
}
for i, line := range lines {
fields := strings.Split(line, "\t")
// Expect 11 tab-separated fields:
// version, lts, channel, date, os, arch, ext, hash, download, (empty), libc
if len(fields) != 11 {
t.Errorf("line[%d]: expected 11 tab fields, got %d: %q", i, len(fields), line)
continue
}
version := fields[0]
lts := fields[1]
ext := fields[6]
if strings.HasPrefix(version, "v") {
t.Errorf("line[%d]: version %q should not have v prefix", i, version)
}
if lts != "-" && lts != "lts" {
t.Errorf("line[%d]: lts should be '-' or 'lts', got %q", i, lts)
}
if strings.HasPrefix(ext, ".") {
t.Errorf("line[%d]: ext %q should not have . prefix", i, ext)
}
}
})
}
}
// TestLegacyJSONAgainstProduction compares our output against live production.
// Run with: WEBI_TEST_PROD=1 go test -run TestLegacyJSONAgainstProduction
func TestLegacyJSONAgainstProduction(t *testing.T) {
if os.Getenv("WEBI_TEST_PROD") == "" {
t.Skip("set WEBI_TEST_PROD=1 to compare against production")
}
srv, ts := newTestServer(t)
packages := []string{"bat", "node", "go", "jq", "rg"}
for _, pkg := range packages {
t.Run(pkg, func(t *testing.T) {
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
// Fetch from production.
prodURL := fmt.Sprintf("https://webinstall.dev/api/releases/%s.json?limit=3", pkg)
prodResp, err := http.Get(prodURL)
if err != nil {
t.Fatalf("fetch production: %v", err)
}
defer prodResp.Body.Close()
prodBody, _ := io.ReadAll(prodResp.Body)
var prodReleases []legacyRelease
if err := json.Unmarshal(prodBody, &prodReleases); err != nil {
t.Fatalf("decode production: %v\nbody: %.500s", err, string(prodBody))
}
// Fetch from local.
_, localBody := get(t, ts, "/api/releases/"+pkg+".json?limit=3")
var localReleases []legacyRelease
if err := json.Unmarshal([]byte(localBody), &localReleases); err != nil {
t.Fatalf("decode local: %v", err)
}
if len(prodReleases) == 0 || len(localReleases) == 0 {
t.Skip("empty releases")
}
// Compare the first release's format.
prod := prodReleases[0]
local := localReleases[0]
if strings.HasPrefix(local.Version, "v") != strings.HasPrefix(prod.Version, "v") {
t.Errorf("version prefix mismatch: prod=%q local=%q", prod.Version, local.Version)
}
if strings.HasPrefix(local.Ext, ".") != strings.HasPrefix(prod.Ext, ".") {
t.Errorf("ext prefix mismatch: prod=%q local=%q", prod.Ext, local.Ext)
}
if prod.OS == "macos" && local.OS == "darwin" {
t.Error("OS: prod uses 'macos', local uses 'darwin'")
}
if prod.Arch == "amd64" && local.Arch == "x86_64" {
t.Error("Arch: prod uses 'amd64', local uses 'x86_64'")
}
if prod.Arch == "arm64" && local.Arch == "aarch64" {
t.Error("Arch: prod uses 'arm64', local uses 'aarch64'")
}
t.Logf("prod[0]: version=%q os=%q arch=%q ext=%q libc=%q",
prod.Version, prod.OS, prod.Arch, prod.Ext, prod.Libc)
t.Logf("local[0]: version=%q os=%q arch=%q ext=%q libc=%q",
local.Version, local.OS, local.Arch, local.Ext, local.Libc)
})
}
}
// TestSortOrder verifies releases come back newest-first.
func TestSortOrder(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
_, body := get(t, ts, "/api/releases/"+pkg+".json?limit=20")
var releases []legacyRelease
if err := json.Unmarshal([]byte(body), &releases); err != nil {
t.Fatalf("decode: %v", err)
}
if len(releases) < 2 {
t.Skip("need at least 2 releases")
}
// First release should be newest (or equal) version.
first := releases[0].Date
last := releases[len(releases)-1].Date
if first < last {
t.Errorf("not newest-first: first=%q last=%q", first, last)
}
}
// Ensure imports are used.
var _ = storage.Asset{}

459
cmd/webid/v1api.go Normal file
View File

@@ -0,0 +1,459 @@
package main
import (
"bytes"
"encoding/csv"
"encoding/json"
"fmt"
"net/http"
"slices"
"strings"
"github.com/jszwec/csvutil"
"github.com/webinstall/webi-installers/internal/buildmeta"
"github.com/webinstall/webi-installers/internal/lexver"
"github.com/webinstall/webi-installers/internal/resolver"
"github.com/webinstall/webi-installers/internal/storage"
)
// v1Release is a single release in the new API TSV format.
// Field order matters for csvutil — it determines column order.
// Fields are designed to be easy to consume with cut/grep/sort.
type v1Release struct {
Version string `csv:"version"`
Channel string `csv:"channel"`
LTS string `csv:"lts"`
Date string `csv:"date"`
OS string `csv:"os"`
Arch string `csv:"arch"`
Libc string `csv:"libc"`
Format string `csv:"format"`
Variants string `csv:"variants"` // space-separated
Download string `csv:"download"`
Filename string `csv:"filename"`
}
// v1ResolveResult is the response for /v1/resolve/{pkg}.
type v1ResolveResult struct {
Version string `csv:"version" json:"version"`
Channel string `csv:"channel" json:"channel"`
LTS string `csv:"lts" json:"lts"`
Date string `csv:"date" json:"date"`
OS string `csv:"os" json:"os"`
Arch string `csv:"arch" json:"arch"`
Libc string `csv:"libc" json:"libc"`
Format string `csv:"format" json:"format"`
Variants string `csv:"variants" json:"variants"`
Download string `csv:"download" json:"download"`
Filename string `csv:"filename" json:"filename"`
Triplet string `csv:"triplet" json:"triplet"`
}
// handleV1Releases serves /v1/releases/{pkg}.tsv (or .json)
// with Go-native naming and TSV-first format.
//
// Query params:
//
// os — filter by OS (darwin, linux, windows)
// arch — filter by arch (aarch64, x86_64, armv7l)
// libc — filter by libc (gnu, musl, msvc)
// channel — release channel (stable, beta, rc, alpha)
// version — version prefix filter (e.g. "1.20")
// lts — if "true", only LTS releases
// format — filter by format (e.g. "tar.gz")
// variant — filter by variant (e.g. "rocm")
// limit — max results (default 1000)
func (s *server) handleV1Releases(w http.ResponseWriter, r *http.Request) {
rest := r.PathValue("rest")
pkg, version, format, err := parseReleasePath(rest)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
pc := s.getPackage(pkg)
if pc == nil {
if s.isSelfHosted(pkg) {
s.v1ServeEmpty(w, format)
return
}
http.Error(w, fmt.Sprintf("package %q not found", pkg), http.StatusNotFound)
return
}
q := r.URL.Query()
osStr := q.Get("os")
archStr := q.Get("arch")
libcStr := q.Get("libc")
channelStr := q.Get("channel")
ltsStr := q.Get("lts")
formatFilter := q.Get("format")
variantStr := q.Get("variant")
limitStr := q.Get("limit")
// Use version from URL path or query.
if version == "" {
version = q.Get("version")
}
// Handle channel selectors in version field.
switch strings.ToLower(version) {
case "stable", "latest":
version = ""
if channelStr == "" {
channelStr = "stable"
}
case "lts":
version = ""
ltsStr = "true"
case "beta", "pre", "preview":
version = ""
if channelStr == "" {
channelStr = "beta"
}
case "rc":
version = ""
if channelStr == "" {
channelStr = "rc"
}
case "alpha", "dev":
version = ""
if channelStr == "" {
channelStr = "alpha"
}
}
lts := ltsStr == "true" || ltsStr == "1"
limit := 1000
if limitStr != "" {
fmt.Sscanf(limitStr, "%d", &limit)
}
// Filter assets directly (not via resolve.Dist).
filtered := filterAssets(pc.assets, osStr, archStr, libcStr, channelStr, version, formatFilter, variantStr, lts, limit)
// Sort newest-first.
sortAssetsDescending(filtered)
switch format {
case "json":
s.v1ServeJSON(w, filtered)
case "tab":
s.v1ServeTSV(w, filtered)
default:
http.Error(w, "unsupported format: "+format+" (use .json or .tab)", http.StatusBadRequest)
}
}
// handleV1Resolve serves /v1/resolve/{pkg}.tsv (or .json)
// It resolves the single best asset for a given platform.
//
// Query params:
//
// os — target OS (required)
// arch — target arch (required)
// libc — target libc
// version — version prefix
// channel — release channel
// lts — if "true", only LTS
// format — preferred formats (comma-separated, in preference order)
// variant — preferred variant
func (s *server) handleV1Resolve(w http.ResponseWriter, r *http.Request) {
rest := r.PathValue("rest")
pkg, version, format, err := parseReleasePath(rest)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
pc := s.getPackage(pkg)
if pc == nil {
http.Error(w, fmt.Sprintf("package %q not found", pkg), http.StatusNotFound)
return
}
q := r.URL.Query()
osStr := q.Get("os")
archStr := q.Get("arch")
libcStr := q.Get("libc")
channelStr := q.Get("channel")
ltsStr := q.Get("lts")
formatsStr := q.Get("format")
variantStr := q.Get("variant")
if version == "" {
version = q.Get("version")
}
// Handle channel selectors in version field.
switch strings.ToLower(version) {
case "stable", "latest":
version = ""
if channelStr == "" {
channelStr = "stable"
}
case "lts":
version = ""
ltsStr = "true"
case "beta", "pre", "preview":
version = ""
if channelStr == "" {
channelStr = "beta"
}
case "rc":
version = ""
if channelStr == "" {
channelStr = "rc"
}
case "alpha", "dev":
version = ""
if channelStr == "" {
channelStr = "alpha"
}
}
lts := ltsStr == "true" || ltsStr == "1"
var formats []string
if formatsStr != "" {
formats = strings.Split(formatsStr, ",")
}
req := resolver.Request{
OS: osStr,
Arch: archStr,
Libc: libcStr,
Version: version,
Channel: channelStr,
LTS: lts,
Formats: formats,
Variant: variantStr,
}
res, err := resolver.Resolve(pc.assets, req)
if err != nil {
http.Error(w, fmt.Sprintf("no match for %s: %v", pkg, err), http.StatusNotFound)
return
}
result := assetToV1Resolve(res)
switch format {
case "json":
w.Header().Set("Content-Type", "application/json")
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
enc.Encode(result)
case "tab":
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
data, err := marshalTSV([]v1ResolveResult{result})
if err != nil {
http.Error(w, "encode error: "+err.Error(), http.StatusInternalServerError)
return
}
w.Write(data)
default:
http.Error(w, "unsupported format: "+format, http.StatusBadRequest)
}
}
func assetToV1Release(a storage.Asset) v1Release {
lts := "-"
if a.LTS {
lts = "lts"
}
channel := a.Channel
if channel == "" {
channel = "stable"
}
libc := a.Libc
if libc == "" {
libc = "-"
}
return v1Release{
Version: a.Version,
Channel: channel,
LTS: lts,
Date: a.Date,
OS: a.OS,
Arch: a.Arch,
Libc: libc,
Format: a.Format,
Variants: strings.Join(a.Variants, " "),
Download: a.Download,
Filename: a.Filename,
}
}
func assetToV1Resolve(res resolver.Result) v1ResolveResult {
a := res.Asset
lts := "-"
if a.LTS {
lts = "lts"
}
channel := a.Channel
if channel == "" {
channel = "stable"
}
libc := a.Libc
if libc == "" {
libc = "-"
}
return v1ResolveResult{
Version: a.Version,
Channel: channel,
LTS: lts,
Date: a.Date,
OS: a.OS,
Arch: a.Arch,
Libc: libc,
Format: a.Format,
Variants: strings.Join(a.Variants, " "),
Download: a.Download,
Filename: a.Filename,
Triplet: res.Triplet,
}
}
func (s *server) v1ServeTSV(w http.ResponseWriter, assets []storage.Asset) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
releases := make([]v1Release, len(assets))
for i, a := range assets {
releases[i] = assetToV1Release(a)
}
data, err := marshalTSV(releases)
if err != nil {
http.Error(w, "encode error: "+err.Error(), http.StatusInternalServerError)
return
}
w.Write(data)
}
func (s *server) v1ServeJSON(w http.ResponseWriter, assets []storage.Asset) {
w.Header().Set("Content-Type", "application/json")
releases := make([]v1Release, len(assets))
for i, a := range assets {
releases[i] = assetToV1Release(a)
}
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
enc.Encode(releases)
}
func (s *server) v1ServeEmpty(w http.ResponseWriter, format string) {
switch format {
case "json":
w.Header().Set("Content-Type", "application/json")
w.Write([]byte("[]\n"))
case "tab":
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
// Just the header.
data, _ := marshalTSV([]v1Release{})
w.Write(data)
}
}
// filterAssets filters storage.Asset slices directly.
func filterAssets(assets []storage.Asset, osStr, archStr, libcStr, channel, version, formatFilter, variant string, lts bool, limit int) []storage.Asset {
var result []storage.Asset
for _, a := range assets {
if osStr != "" && a.OS != osStr && a.OS != "ANYOS" && a.OS != "" {
continue
}
if archStr != "" && a.Arch != archStr && a.Arch != "ANYARCH" && a.Arch != "" {
continue
}
if libcStr != "" && a.Libc != "" && a.Libc != "none" && a.Libc != libcStr {
continue
}
if lts && !a.LTS {
continue
}
if channel != "" && a.Channel != channel {
continue
}
if version != "" {
v := strings.TrimPrefix(a.Version, "v")
vq := strings.TrimPrefix(version, "v")
if !strings.HasPrefix(v, vq) {
continue
}
}
if formatFilter != "" && !strings.Contains(a.Format, formatFilter) {
continue
}
if variant != "" {
if !hasVariant(a.Variants, variant) {
continue
}
}
result = append(result, a)
if len(result) >= limit {
break
}
}
return result
}
// sortAssetsDescending sorts assets newest-first by version.
func sortAssetsDescending(assets []storage.Asset) {
slices.SortStableFunc(assets, func(a, b storage.Asset) int {
va := lexver.Parse(strings.TrimPrefix(a.Version, "v"))
vb := lexver.Parse(strings.TrimPrefix(b.Version, "v"))
return lexver.Compare(vb, va) // descending
})
}
// hasVariant checks if the variant list contains the wanted variant.
// This is a copy of resolver.hasVariant since it's unexported.
func hasVariant(variants []string, want string) bool {
for _, v := range variants {
if v == want {
return true
}
}
return false
}
// marshalTSV encodes a slice of structs as tab-separated values with a header.
// Uses csvutil for struct-to-CSV mapping, with csv.Writer set to tab delimiter.
func marshalTSV[T any](records []T) ([]byte, error) {
var buf bytes.Buffer
w := csv.NewWriter(&buf)
w.Comma = '\t'
enc := csvutil.NewEncoder(w)
for _, r := range records {
if err := enc.Encode(r); err != nil {
return nil, err
}
}
w.Flush()
if err := w.Error(); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// normalizeV1Arch maps query arch names to canonical Go names.
func normalizeV1Arch(s string) string {
switch strings.ToLower(s) {
case "amd64":
return string(buildmeta.ArchAMD64) // "x86_64"
case "arm64":
return string(buildmeta.ArchARM64) // "aarch64"
default:
return s
}
}

273
cmd/webid/v1api_test.go Normal file
View File

@@ -0,0 +1,273 @@
package main
import (
"encoding/json"
"strings"
"testing"
)
// TestV1ReleasesTSV verifies the v1 releases endpoint returns proper TSV.
func TestV1ReleasesTSV(t *testing.T) {
srv, ts := newTestServer(t)
packages := []string{"bat", "node", "go"}
for _, pkg := range packages {
t.Run(pkg, func(t *testing.T) {
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/v1/releases/"+pkg+".tab?limit=5")
if code != 200 {
t.Fatalf("status %d: %s", code, body)
}
lines := strings.Split(strings.TrimSpace(body), "\n")
if len(lines) < 2 {
t.Fatal("expected header + data rows")
}
// First line should be header.
header := lines[0]
fields := strings.Split(header, "\t")
expectedHeaders := []string{
"version",
"channel",
"lts",
"date",
"os",
"arch",
"libc",
"format",
"variants",
"download",
"filename",
}
if len(fields) != len(expectedHeaders) {
t.Fatalf("expected %d columns, got %d: %q", len(expectedHeaders), len(fields), header)
}
for i, want := range expectedHeaders {
if fields[i] != want {
t.Errorf("column[%d]: want %q, got %q", i, want, fields[i])
}
}
// Data rows should have same number of fields.
for i, line := range lines[1:] {
dataFields := strings.Split(line, "\t")
if len(dataFields) != len(expectedHeaders) {
t.Errorf("row[%d]: expected %d fields, got %d: %q", i, len(expectedHeaders), len(dataFields), line)
}
}
})
}
}
// TestV1ReleasesJSON verifies the v1 releases JSON format.
func TestV1ReleasesJSON(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/v1/releases/"+pkg+".json?limit=3")
if code != 200 {
t.Fatalf("status %d: %s", code, body)
}
var releases []v1Release
if err := json.Unmarshal([]byte(body), &releases); err != nil {
t.Fatalf("decode: %v", err)
}
if len(releases) == 0 {
t.Fatal("no releases")
}
// v1 API uses Go-native naming — no mapping.
for i, r := range releases {
if r.Version == "" {
t.Errorf("release[%d]: empty version", i)
}
if r.Download == "" {
t.Errorf("release[%d]: empty download", i)
}
if r.Channel == "" {
t.Errorf("release[%d]: empty channel (should be 'stable' or similar)", i)
}
}
}
// TestV1Resolve verifies the v1 resolve endpoint.
func TestV1Resolve(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
tests := []struct {
name string
query string
wantOS string
}{
{
name: "linux amd64",
query: "?os=linux&arch=x86_64",
wantOS: "linux",
},
{
name: "darwin arm64",
query: "?os=darwin&arch=aarch64",
wantOS: "darwin",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
code, body := get(t, ts, "/v1/resolve/"+pkg+".json"+tt.query)
if code != 200 {
t.Fatalf("status %d: %s", code, body)
}
var result v1ResolveResult
if err := json.Unmarshal([]byte(body), &result); err != nil {
t.Fatalf("decode: %v", err)
}
if result.Version == "" {
t.Error("empty version")
}
if result.Download == "" {
t.Error("empty download")
}
if result.OS != tt.wantOS {
t.Errorf("os: want %q, got %q", tt.wantOS, result.OS)
}
if result.Triplet == "" {
t.Error("empty triplet")
}
t.Logf("resolved: %s %s %s %s → %s", result.Version, result.OS, result.Arch, result.Format, result.Download)
})
}
}
// TestV1ResolveTSV verifies the TSV format for resolve.
func TestV1ResolveTSV(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/v1/resolve/"+pkg+".tab?os=linux&arch=x86_64")
if code != 200 {
t.Fatalf("status %d: %s", code, body)
}
lines := strings.Split(strings.TrimSpace(body), "\n")
if len(lines) != 2 {
t.Fatalf("expected 2 lines (header + result), got %d", len(lines))
}
header := strings.Split(lines[0], "\t")
data := strings.Split(lines[1], "\t")
if len(header) != len(data) {
t.Fatalf("header has %d fields, data has %d", len(header), len(data))
}
// Should have a "triplet" column.
hasTriplet := false
for _, h := range header {
if h == "triplet" {
hasTriplet = true
}
}
if !hasTriplet {
t.Error("missing triplet column in header")
}
}
// TestV1ResolveJQ verifies jq resolves to binaries, not git.
func TestV1ResolveJQ(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "jq"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/v1/resolve/"+pkg+".json?os=darwin&arch=aarch64")
if code != 200 {
t.Fatalf("status %d: %s", code, body)
}
var result v1ResolveResult
if err := json.Unmarshal([]byte(body), &result); err != nil {
t.Fatalf("decode: %v", err)
}
if result.Format == "git" {
t.Errorf("resolved to git instead of binary: %+v", result)
}
if result.OS == "" {
t.Errorf("resolved to empty OS (git asset): %+v", result)
}
t.Logf("jq resolved: version=%s os=%s arch=%s format=%s → %s",
result.Version, result.OS, result.Arch, result.Format, result.Download)
}
// TestV1ReleasesFilterOS verifies OS filtering works.
func TestV1ReleasesFilterOS(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/v1/releases/"+pkg+".json?os=darwin&limit=10")
if code != 200 {
t.Fatalf("status %d: %s", code, body)
}
var releases []v1Release
if err := json.Unmarshal([]byte(body), &releases); err != nil {
t.Fatalf("decode: %v", err)
}
for i, r := range releases {
if r.OS != "darwin" && r.OS != "ANYOS" && r.OS != "" {
t.Errorf("release[%d]: os=%q, expected darwin", i, r.OS)
}
}
}
// TestV1NoQuotedFields verifies TSV output has no quoted fields.
func TestV1NoQuotedFields(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/v1/releases/"+pkg+".tab?limit=20")
if code != 200 {
t.Fatalf("status %d: %s", code, body)
}
lines := strings.Split(strings.TrimSpace(body), "\n")
for i, line := range lines {
if strings.Contains(line, "\"") {
t.Errorf("line[%d] contains quotes: %s", i, line)
}
}
}

15
go.mod
View File

@@ -2,4 +2,17 @@ module github.com/webinstall/webi-installers
go 1.26.1
require github.com/joho/godotenv v1.5.1
require (
github.com/jackc/pgx/v5 v5.9.2
github.com/joho/godotenv v1.5.1
github.com/jszwec/csvutil v1.10.0
github.com/therootcompany/golib/http/middleware/v2 v2.0.1
)
require (
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
golang.org/x/sync v0.17.0 // indirect
golang.org/x/text v0.29.0 // indirect
)

30
go.sum
View File

@@ -1,2 +1,32 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/jszwec/csvutil v1.10.0 h1:upMDUxhQKqZ5ZDCs/wy+8Kib8rZR8I8lOR34yJkdqhI=
github.com/jszwec/csvutil v1.10.0/go.mod h1:/E4ONrmGkwmWsk9ae9jpXnv9QT8pLHEPcCirMFhxG9I=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/therootcompany/golib/http/middleware/v2 v2.0.1 h1:VNKpHcwyEW7cMct7/eO4fyrxwIQk2ycb6juVXSPs2Sk=
github.com/therootcompany/golib/http/middleware/v2 v2.0.1/go.mod h1:g5gb9qBidw74nW6/mwIauTKMpOKchiN2l0gt5qzJ2aQ=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

267
internal/render/render.go Normal file
View File

@@ -0,0 +1,267 @@
// Package render generates installer scripts by injecting release
// metadata into the package-install template.
//
// The template uses shell-style variable markers:
//
// #WEBI_VERSION= → WEBI_VERSION='1.2.3'
// #export WEBI_PKG_URL= → export WEBI_PKG_URL='https://...'
//
// The package's install.sh is injected at the {{ installer }} marker.
package render
import (
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
)
// Params holds all the values to inject into the installer template.
type Params struct {
// Host is the base URL of the webi server (e.g. "https://webinstall.dev").
Host string
// Checksum is the webi.sh bootstrap script checksum (first 8 hex chars of SHA-1).
Checksum string
// Package name (e.g. "bat", "node").
PkgName string
// Tag is the version selector from the URL (e.g. "20", "stable", "").
Tag string
// OS, Arch, Libc are the detected platform strings.
OS string
Arch string
Libc string
// Resolved release info.
Version string
Major string
Minor string
Patch string
Build string
GitTag string
GitBranch string
GitCommitHash string
LTS string // "true" or "false"
Channel string
Ext string // archive extension (e.g. "tar.gz", "zip")
Formats string // comma-separated format list
// Download info.
PkgURL string // download URL
PkgFile string // filename
// Releases API URL for this request.
ReleasesURL string
// CSV line for WEBI_CSV.
CSV string
// Package catalog info.
PkgStable string
PkgLatest string
PkgOSes string // space-separated
PkgArches string // space-separated
PkgLibcs string // space-separated
PkgFormats string // space-separated
}
// Bash renders a complete bash installer script by injecting params
// into the template and splicing in the package's install.sh.
func Bash(tplPath, installersDir, pkgName string, p Params) (string, error) {
tpl, err := os.ReadFile(tplPath)
if err != nil {
return "", fmt.Errorf("render: read template: %w", err)
}
// Read the package's install.sh.
installPath := filepath.Join(installersDir, pkgName, "install.sh")
installSh, err := os.ReadFile(installPath)
if err != nil {
return "", fmt.Errorf("render: read %s/install.sh: %w", pkgName, err)
}
text := string(tpl)
// Inject environment variables.
vars := []struct {
name string
value string
}{
{"WEBI_CHECKSUM", p.Checksum},
{"WEBI_PKG", p.PkgName + "@" + p.Tag},
{"WEBI_HOST", p.Host},
{"WEBI_OS", p.OS},
{"WEBI_ARCH", p.Arch},
{"WEBI_LIBC", p.Libc},
{"WEBI_TAG", p.Tag},
{"WEBI_RELEASES", p.ReleasesURL},
{"WEBI_CSV", p.CSV},
{"WEBI_VERSION", p.Version},
{"WEBI_MAJOR", p.Major},
{"WEBI_MINOR", p.Minor},
{"WEBI_PATCH", p.Patch},
{"WEBI_BUILD", p.Build},
{"WEBI_GIT_BRANCH", p.GitBranch},
{"WEBI_GIT_TAG", p.GitTag},
{"WEBI_GIT_COMMIT_HASH", p.GitCommitHash},
{"WEBI_LTS", p.LTS},
{"WEBI_CHANNEL", p.Channel},
{"WEBI_EXT", p.Ext},
{"WEBI_FORMATS", p.Formats},
{"WEBI_PKG_URL", p.PkgURL},
{"WEBI_PKG_PATHNAME", p.PkgFile},
{"WEBI_PKG_FILE", p.PkgFile},
{"PKG_NAME", p.PkgName},
{"PKG_STABLE", p.PkgStable},
{"PKG_LATEST", p.PkgLatest},
{"PKG_OSES", p.PkgOSes},
{"PKG_ARCHES", p.PkgArches},
{"PKG_LIBCS", p.PkgLibcs},
{"PKG_FORMATS", p.PkgFormats},
}
for _, v := range vars {
text = InjectVar(text, v.name, v.value)
}
// Inject the installer script at the {{ installer }} marker.
// The marker sits inside __init_installer() at 8-space indent.
// Production pads every line of install.sh to match, and replaces
// the entire line (including leading whitespace).
padded := padScript(string(installSh), " ")
text = replaceMarkerLine(text, "{{ installer }}", padded)
return text, nil
}
// PowerShell renders a complete PowerShell installer script by injecting
// params into the template and splicing in the package's install.ps1.
func PowerShell(tplPath, installersDir, pkgName string, p Params) (string, error) {
tpl, err := os.ReadFile(tplPath)
if err != nil {
return "", fmt.Errorf("render: read template: %w", err)
}
installPath := filepath.Join(installersDir, pkgName, "install.ps1")
installPs1, err := os.ReadFile(installPath)
if err != nil {
return "", fmt.Errorf("render: read %s/install.ps1: %w", pkgName, err)
}
text := string(tpl)
vars := []struct {
name string
value string
}{
{"WEBI_PKG", p.PkgName + "@" + p.Tag},
{"WEBI_HOST", p.Host},
{"WEBI_VERSION", p.Version},
{"WEBI_GIT_TAG", p.GitTag},
{"WEBI_GIT_COMMIT_HASH", p.GitCommitHash},
{"WEBI_PKG_URL", p.PkgURL},
{"WEBI_PKG_FILE", p.PkgFile},
{"WEBI_PKG_PATHNAME", p.PkgFile},
{"PKG_NAME", p.PkgName},
}
for _, v := range vars {
text = InjectPSVar(text, v.name, v.value)
}
// PS1 marker is at column 0, no padding needed.
text = replaceMarkerLine(text, "{{ installer }}", string(installPs1))
return text, nil
}
// InjectPSVar replaces a PowerShell template variable line with its value.
// Matches lines like:
//
// #$Env:WEBI_VERSION = v12.16.2
// $Env:WEBI_HOST = 'https://webinstall.dev'
func InjectPSVar(text, name, value string) string {
p := getPSVarPattern(name)
return p.ReplaceAllString(text, "${1}$$Env:"+name+" = '"+sanitizePSValue(value)+"'")
}
var psVarPatterns = map[string]*regexp.Regexp{}
func getPSVarPattern(name string) *regexp.Regexp {
if p, ok := psVarPatterns[name]; ok {
return p
}
// Match: optional leading whitespace, optional #, $Env:NAME, =, rest of line
p := regexp.MustCompile(`(?m)^([ \t]*)#?\$Env:` + regexp.QuoteMeta(name) + `\s*=.*$`)
psVarPatterns[name] = p
return p
}
// sanitizePSValue escapes single quotes for PowerShell single-quoted strings.
// In PowerShell, single quotes inside single-quoted strings are doubled: ''
func sanitizePSValue(s string) string {
return strings.ReplaceAll(s, "'", "''")
}
// varPattern matches shell variable declarations in the template.
// Matches lines like:
//
// #WEBI_VERSION=
// #export WEBI_PKG_URL=
// #WEBI_OS=
var varPatterns = map[string]*regexp.Regexp{}
func getVarPattern(name string) *regexp.Regexp {
if p, ok := varPatterns[name]; ok {
return p
}
// Match: optional leading whitespace, optional #, optional export, the var name, =, rest of line
p := regexp.MustCompile(`(?m)^([ \t]*)#?([ \t]*)(export[ \t]+)?[ \t]*(` + regexp.QuoteMeta(name) + `)=.*$`)
varPatterns[name] = p
return p
}
// InjectVar replaces a template variable line with its value.
// It matches lines like:
//
// #WEBI_VERSION=
// #export WEBI_PKG_URL=
// export WEBI_HOST=
//
// and replaces them with the value in single quotes.
func InjectVar(text, name, value string) string {
p := getVarPattern(name)
return p.ReplaceAllString(text, "${1}${3}"+name+"='"+sanitizeShellValue(value)+"'")
}
// sanitizeShellValue ensures a value is safe to embed in single quotes.
// Single quotes in shell can't be escaped inside single quotes, so we
// close-quote, add escaped quote, re-open quote: 'foo'\''bar'
func sanitizeShellValue(s string) string {
return strings.ReplaceAll(s, "'", `'\''`)
}
// padScript prepends each line of a script with the given indent string.
// This matches production behavior where install.sh content is indented
// to align with the surrounding template code.
func padScript(script, indent string) string {
lines := strings.Split(script, "\n")
for i, line := range lines {
if line != "" {
lines[i] = indent + line
}
}
return strings.Join(lines, "\n")
}
// replaceMarkerLine replaces an entire line containing the marker
// (including any leading whitespace) with the replacement text.
// This matches production's regex: /\s*#?\s*{{ installer }}/
func replaceMarkerLine(text, marker, replacement string) string {
re := regexp.MustCompile(`(?m)^[ \t]*#?[ \t]*` + regexp.QuoteMeta(marker) + `[^\n]*`)
return re.ReplaceAllLiteralString(text, replacement)
}

View File

@@ -0,0 +1,90 @@
package render
import (
"strings"
"testing"
)
func TestInjectVar(t *testing.T) {
tests := []struct {
name string
input string
key string
value string
want string
}{
{
name: "commented var",
input: " #WEBI_VERSION=",
key: "WEBI_VERSION",
value: "1.2.3",
want: " WEBI_VERSION='1.2.3'",
},
{
name: "commented export var",
input: " #export WEBI_PKG_URL=",
key: "WEBI_PKG_URL",
value: "https://example.com/foo.tar.gz",
want: " export WEBI_PKG_URL='https://example.com/foo.tar.gz'",
},
{
name: "existing value replaced",
input: " export WEBI_HOST=",
key: "WEBI_HOST",
value: "https://webinstall.dev",
want: " export WEBI_HOST='https://webinstall.dev'",
},
{
name: "value with single quotes",
input: " #PKG_NAME=",
key: "PKG_NAME",
value: "it's-a-test",
want: " PKG_NAME='it'\\''s-a-test'",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := InjectVar(tt.input, tt.key, tt.value)
if strings.TrimSpace(got) != strings.TrimSpace(tt.want) {
t.Errorf("got %q\nwant %q", got, tt.want)
}
})
}
}
func TestInjectVarInTemplate(t *testing.T) {
tpl := `#!/bin/sh
__bootstrap_webi() {
#PKG_NAME=
#WEBI_OS=
#WEBI_ARCH=
#WEBI_VERSION=
export WEBI_HOST=
WEBI_PKG_DOWNLOAD=""
`
result := tpl
result = InjectVar(result, "PKG_NAME", "bat")
result = InjectVar(result, "WEBI_OS", "linux")
result = InjectVar(result, "WEBI_ARCH", "x86_64")
result = InjectVar(result, "WEBI_VERSION", "0.26.1")
result = InjectVar(result, "WEBI_HOST", "https://webinstall.dev")
if !strings.Contains(result, "PKG_NAME='bat'") {
t.Error("PKG_NAME not injected")
}
if !strings.Contains(result, "WEBI_OS='linux'") {
t.Error("WEBI_OS not injected")
}
if !strings.Contains(result, "WEBI_VERSION='0.26.1'") {
t.Error("WEBI_VERSION not injected")
}
if !strings.Contains(result, "export WEBI_HOST='https://webinstall.dev'") {
t.Error("WEBI_HOST not injected")
}
// Should not have #PKG_NAME= anymore.
if strings.Contains(result, "#PKG_NAME=") {
t.Error("#PKG_NAME= should have been replaced")
}
}

303
internal/resolve/resolve.go Normal file
View File

@@ -0,0 +1,303 @@
// Package resolve picks the best release for a given platform query.
//
// Given a set of classified distributables and a target query (OS, arch,
// libc, format preferences, version constraint), it returns the single
// best matching release — or nil if nothing matches.
package resolve
import (
"strings"
"github.com/webinstall/webi-installers/internal/buildmeta"
"github.com/webinstall/webi-installers/internal/lexver"
)
// Dist is one downloadable distributable — matches the CSV row from classify.
type Dist struct {
Package string
Version string
Channel string
OS string
Arch string
Libc string
Format string
Download string
Filename string
SHA256 string
Size int64
LTS bool
Date string
Extra string // extra version info for sorting
GitTag string // original git tag or branch — only for format="git"
GitCommitHash string // short commit hash — only for format="git"
Variants []string // build qualifiers: "installer", "rocm", "fxdependent", etc.
}
// Query describes what the caller wants.
type Query struct {
OS buildmeta.OS
Arch buildmeta.Arch
Libc buildmeta.Libc
Formats []string // acceptable formats (e.g. ".tar.gz", ".zip"), in preference order
Channel string // "stable" (default), "beta", etc.
Version string // version prefix constraint ("24", "24.14", ""), empty = latest
Variants []string // if non-empty, only match assets with these variants
}
// Match is the resolved release.
type Match struct {
Version string
OS string
Arch string
Libc string
Format string
Download string
Filename string
LTS bool
Date string
Channel string
}
// Best finds the single best release matching the query.
// Returns nil if nothing matches.
func Best(dists []Dist, q Query) *Match {
channel := q.Channel
if channel == "" {
channel = "stable"
}
// Build format set for fast lookup + rank map for preference.
formatRank := make(map[string]int, len(q.Formats))
for i, f := range q.Formats {
formatRank[f] = i
}
// Build the set of acceptable architectures (native + compat).
compatArches := buildmeta.CompatArches(q.OS, q.Arch)
archRank := make(map[string]int, len(compatArches))
for i, a := range compatArches {
archRank[string(a)] = i
}
// Parse version prefix for constraint matching.
var versionPrefix lexver.Version
hasVersionConstraint := q.Version != ""
if hasVersionConstraint {
versionPrefix = lexver.Parse(q.Version)
}
var best *candidate
for i := range dists {
d := &dists[i]
// Channel filter.
if channel == "stable" && d.Channel != "stable" && d.Channel != "" {
continue
}
// OS filter: exact match, POSIX fallback, or ANYOS.
if !osMatches(q.OS, d.OS) {
continue
}
// Arch filter (including compat arches).
// Empty arch, ANYARCH, or "*" means "universal/platform-agnostic" —
// accept it but rank it lower than an exact match.
aRank, archOK := archRank[d.Arch]
if !archOK && (d.Arch == "" || d.Arch == "*" || d.Arch == string(buildmeta.ArchAny)) {
// Universal binary — rank after all specific arches.
aRank = len(compatArches)
archOK = true
}
if !archOK {
continue
}
// Libc filter.
if !libcMatches(q.OS, q.Libc, d.Libc) {
continue
}
// Format filter.
// Empty format means bare binary — accept as last resort.
fRank, formatOK := formatRank[d.Format]
if !formatOK && d.Format == "" {
// Bare binary — rank after all explicit formats.
fRank = len(q.Formats)
formatOK = true
}
if !formatOK && len(q.Formats) > 0 {
continue
}
if !formatOK {
fRank = 999
}
// Version constraint.
ver := lexver.Parse(d.Version)
if hasVersionConstraint && !ver.HasPrefix(versionPrefix) {
continue
}
c := &candidate{
dist: d,
ver: ver,
archRank: aRank,
formatRank: fRank,
hasVariants: len(d.Variants) > 0,
}
if best == nil || c.betterThan(best) {
best = c
}
}
if best == nil {
return nil
}
d := best.dist
return &Match{
Version: d.Version,
OS: d.OS,
Arch: d.Arch,
Libc: d.Libc,
Format: d.Format,
Download: d.Download,
Filename: d.Filename,
LTS: d.LTS,
Date: d.Date,
Channel: d.Channel,
}
}
// Catalog computes aggregate metadata across all stable dists for a package.
type Catalog struct {
OSes []string
Arches []string
Libcs []string
Formats []string
Latest string // highest version of any channel
Stable string // highest stable version
}
// Survey scans all dists and returns the catalog.
func Survey(dists []Dist) Catalog {
oses := make(map[string]bool)
arches := make(map[string]bool)
libcs := make(map[string]bool)
formats := make(map[string]bool)
var latest, stable string
for _, d := range dists {
if d.OS != "" {
oses[d.OS] = true
}
if d.Arch != "" {
arches[d.Arch] = true
}
if d.Libc != "" {
libcs[d.Libc] = true
}
if d.Format != "" {
formats[d.Format] = true
}
v := lexver.Parse(d.Version)
if latest == "" || lexver.Compare(v, lexver.Parse(latest)) > 0 {
latest = d.Version
}
if d.Channel == "stable" || d.Channel == "" {
if stable == "" || lexver.Compare(v, lexver.Parse(stable)) > 0 {
stable = d.Version
}
}
}
return Catalog{
OSes: sortedKeys(oses),
Arches: sortedKeys(arches),
Libcs: sortedKeys(libcs),
Formats: sortedKeys(formats),
Latest: latest,
Stable: stable,
}
}
type candidate struct {
dist *Dist
ver lexver.Version
archRank int
formatRank int
hasVariants bool // true if dist has variant qualifiers (GPU, installer, etc.)
}
// betterThan returns true if c is a better match than other.
// Priority: version (higher) > base over variant > arch rank (lower=native) > format rank (lower=preferred).
func (c *candidate) betterThan(other *candidate) bool {
cmp := lexver.Compare(c.ver, other.ver)
if cmp != 0 {
return cmp > 0
}
// Prefer base build over variant builds (rocm, installer, etc.)
if c.hasVariants != other.hasVariants {
return !c.hasVariants
}
if c.archRank != other.archRank {
return c.archRank < other.archRank
}
return c.formatRank < other.formatRank
}
// osMatches checks whether a dist's OS is acceptable for the query.
// Matches exact OS, ANYOS (universal), and POSIX compatibility levels
// (posix_2017 matches any non-Windows OS).
func osMatches(want buildmeta.OS, have string) bool {
if have == string(want) {
return true
}
if have == string(buildmeta.OSAny) {
return true
}
// POSIX assets run on any non-Windows system.
if want != buildmeta.OSWindows {
if have == string(buildmeta.OSPosix2017) || have == string(buildmeta.OSPosix2024) {
return true
}
}
return false
}
// libcMatches checks whether a dist's libc is acceptable for the query.
func libcMatches(os buildmeta.OS, want buildmeta.Libc, have string) bool {
// Darwin and Windows don't use libc tagging — accept anything.
if os == buildmeta.OSDarwin || os == buildmeta.OSWindows {
return true
}
// If the dist has no libc tag, accept it (likely statically linked).
if have == "" || have == "none" || have == string(buildmeta.LibcNone) {
return true
}
// If the query has no libc preference, accept any.
if want == "" || want == buildmeta.LibcNone {
return true
}
return have == string(want)
}
func sortedKeys(m map[string]bool) []string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
// Simple insertion sort — these are tiny sets.
for i := 1; i < len(keys); i++ {
for j := i; j > 0 && strings.Compare(keys[j-1], keys[j]) > 0; j-- {
keys[j-1], keys[j] = keys[j], keys[j-1]
}
}
return keys
}

View File

@@ -0,0 +1,416 @@
// Package resolver selects the best release asset for a given platform
// and version constraint.
//
// The resolver takes a package's full asset list and a request describing
// what the client needs (OS, arch, libc, version prefix, channel, format
// preferences). It returns the single best matching asset or an error.
//
// Resolution order:
// 1. Filter assets by channel (inclusive: @stable includes stable+lts)
// 2. Sort versions descending, filter by version prefix if given
// 3. For each candidate version, try compatible platform triplets
// (OS × CompatArches fallback × libc) in preference order
// 4. Among platform matches, pick the best format
// 5. Among format matches, prefer assets without build variants
package resolver
import (
"errors"
"slices"
"strings"
"github.com/webinstall/webi-installers/internal/buildmeta"
"github.com/webinstall/webi-installers/internal/lexver"
"github.com/webinstall/webi-installers/internal/storage"
)
// ErrNoMatch is returned when no asset matches the request.
var ErrNoMatch = errors.New("resolver: no matching asset")
// Request describes what the client is looking for.
type Request struct {
// OS is the target operating system (e.g. "linux", "darwin", "windows").
OS string
// Arch is the target architecture (e.g. "aarch64", "x86_64").
Arch string
// Libc is the preferred C library (e.g. "gnu", "musl", "msvc").
// Empty means no preference — the resolver tries all libc values.
Libc string
// Version is a version prefix constraint (e.g. "1.20", "1", "").
// Empty means latest. Exact versions like "1.20.3" also work.
Version string
// Channel selects the release stability level. Values:
// ""/"stable" — stable and LTS only (default)
// "lts" — LTS releases only
// "rc" — rc + stable + LTS
// "beta" — beta + rc + stable + LTS
// "alpha" — everything (alpha + beta + rc + stable + LTS)
// "pre" — alias for beta (package-specific meaning)
Channel string
// LTS when true selects only LTS-flagged releases.
LTS bool
// Formats lists acceptable archive formats in preference order.
// If empty, a default preference order is used.
Formats []string
// Variant selects a specific build variant (e.g. "rocm", "jetpack6").
// If empty, assets with variants are deprioritized.
Variant string
}
// Result holds the resolved asset and metadata about the match.
type Result struct {
// Asset is the selected download.
Asset storage.Asset
// Version is the matched version string.
Version string
// Triplet is the matched platform triplet (os-arch-libc).
Triplet string
}
// Resolve finds the best matching asset for the given request.
func Resolve(assets []storage.Asset, req Request) (Result, error) {
if len(assets) == 0 {
return Result{}, ErrNoMatch
}
// Parse the version prefix for filtering.
var versionPrefix lexver.Version
hasPrefix := req.Version != ""
if hasPrefix {
versionPrefix = lexver.Parse(req.Version)
}
// Build the channel filter.
channelOK := channelFilter(req.Channel, req.LTS)
// Parse and sort all unique versions descending.
type versionEntry struct {
parsed lexver.Version
raw string
}
seen := make(map[string]bool)
var versions []versionEntry
for _, a := range assets {
if seen[a.Version] {
continue
}
seen[a.Version] = true
v := lexver.Parse(a.Version)
v.Raw = a.Version
versions = append(versions, versionEntry{parsed: v, raw: a.Version})
}
slices.SortFunc(versions, func(a, b versionEntry) int {
return lexver.Compare(b.parsed, a.parsed) // descending
})
// Build platform fallback list: ordered (os, arch, libc) combinations.
triplets := enumerateTriplets(req.OS, req.Arch, req.Libc)
// Build format preference list.
formats := req.Formats
if len(formats) == 0 {
formats = defaultFormats(req.OS)
}
// Index assets by version+triplet for fast lookup.
// Assets with empty OS/Arch (like git repos) use "" keys.
type tripletKey struct {
version string
os string
arch string
libc string
}
index := make(map[tripletKey][]storage.Asset)
for _, a := range assets {
key := tripletKey{
version: a.Version,
os: a.OS,
arch: a.Arch,
libc: a.Libc,
}
index[key] = append(index[key], a)
}
// Walk versions in descending order.
for _, ve := range versions {
// Check version prefix.
if hasPrefix && !ve.parsed.HasPrefix(versionPrefix) {
continue
}
// Check channel.
if !channelOK(ve.parsed.Channel, ve.raw) {
continue
}
// Try each compatible triplet.
for _, tri := range triplets {
key := tripletKey{
version: ve.raw,
os: tri.os,
arch: tri.arch,
libc: tri.libc,
}
candidates := index[key]
if len(candidates) == 0 {
continue
}
// Pick the best asset from candidates.
best, ok := pickBest(candidates, formats, req.Variant, req.LTS)
if !ok {
continue
}
triplet := tri.os + "-" + tri.arch + "-" + tri.libc
return Result{
Asset: best,
Version: ve.raw,
Triplet: triplet,
}, nil
}
}
return Result{}, ErrNoMatch
}
// channelFilter returns a function that checks whether a given channel
// is acceptable for the requested channel level.
func channelFilter(requested string, ltsOnly bool) func(channel string, version string) bool {
if ltsOnly {
return func(_ string, _ string) bool {
// LTS filtering happens at the asset level, not version level.
// We let all versions through and filter by LTS flag later.
// Actually, LTS is per-asset, so we handle it in pickBest.
return true
}
}
requested = strings.ToLower(requested)
if requested == "" {
requested = "stable"
}
if requested == "pre" {
requested = "beta"
}
if requested == "latest" {
requested = "stable"
}
// channelRank maps channel names to a numeric rank.
// Higher rank = less stable. A request for rank N accepts
// everything at rank N or below.
rank := func(ch string) int {
ch = strings.ToLower(ch)
switch ch {
case "", "stable":
return 0
case "rc":
return 1
case "beta", "preview":
return 2
case "alpha", "dev":
return 3
default:
return 2 // unknown pre-release channels default to beta-level
}
}
maxRank := rank(requested)
return func(channel string, _ string) bool {
return rank(channel) <= maxRank
}
}
type platformTriple struct {
os string
arch string
libc string
}
// enumerateTriplets builds the ordered list of platform combinations to try.
// It uses CompatArches for arch fallback and tries multiple libc values.
func enumerateTriplets(osStr, archStr, libcStr string) []platformTriple {
// OS candidates: specific OS first, then POSIX compat, then any.
var oses []string
switch osStr {
case "windows":
oses = []string{"windows", "ANYOS", ""}
case "android":
oses = []string{"android", "linux", "posix_2024", "posix_2017", "ANYOS", ""}
case "":
oses = []string{"ANYOS", ""}
default:
oses = []string{osStr, "posix_2024", "posix_2017", "ANYOS", ""}
}
// Arch candidates: use CompatArches for fallback chain.
arches := buildmeta.CompatArches(buildmeta.OS(osStr), buildmeta.Arch(archStr))
var archStrs []string
for _, a := range arches {
archStrs = append(archStrs, string(a))
}
// Also try ANYARCH and empty (for platform-agnostic assets like git repos).
archStrs = append(archStrs, "ANYARCH", "")
// Libc candidates.
var libcs []string
if libcStr != "" {
libcs = []string{libcStr, "none", ""}
} else {
// No preference: try all common options.
switch osStr {
case "linux":
// none first (static, no deps), then gnu, musl, empty.
libcs = []string{"none", "gnu", "musl", ""}
case "windows":
// none first (no deps), msvc last (needs vcredist).
libcs = []string{"none", "msvc", ""}
default:
libcs = []string{"none", ""}
}
}
var triplets []platformTriple
for _, os := range oses {
for _, arch := range archStrs {
for _, libc := range libcs {
triplets = append(triplets, platformTriple{
os: os,
arch: arch,
libc: libc,
})
}
}
}
return triplets
}
// pickBest selects the best asset from a set of candidates for the same
// version and platform. Prefers the requested variant (or no-variant if
// none requested), then picks by format preference.
func pickBest(candidates []storage.Asset, formats []string, wantVariant string, ltsOnly bool) (storage.Asset, bool) {
// Filter by LTS if requested.
if ltsOnly {
var lts []storage.Asset
for _, a := range candidates {
if a.LTS {
lts = append(lts, a)
}
}
if len(lts) == 0 {
return storage.Asset{}, false
}
candidates = lts
}
// Separate into variant-matched and non-variant pools.
var preferred []storage.Asset
var fallback []storage.Asset
for _, a := range candidates {
if wantVariant != "" {
// User requested a specific variant.
if hasVariant(a.Variants, wantVariant) {
preferred = append(preferred, a)
} else if len(a.Variants) == 0 {
fallback = append(fallback, a)
}
} else {
// No variant requested: prefer no-variant assets.
if len(a.Variants) == 0 {
preferred = append(preferred, a)
} else {
fallback = append(fallback, a)
}
}
}
// Try preferred pool first, then fallback.
for _, pool := range [][]storage.Asset{preferred, fallback} {
if len(pool) == 0 {
continue
}
if best, ok := pickByFormat(pool, formats); ok {
return best, true
}
}
return storage.Asset{}, false
}
// pickByFormat selects the asset with the most preferred format.
func pickByFormat(assets []storage.Asset, formats []string) (storage.Asset, bool) {
for _, fmt := range formats {
for _, a := range assets {
if a.Format == fmt {
return a, true
}
}
}
// No format match — return the first asset as last resort.
if len(assets) > 0 {
return assets[0], true
}
return storage.Asset{}, false
}
func hasVariant(variants []string, want string) bool {
for _, v := range variants {
if v == want {
return true
}
}
return false
}
// defaultFormats returns the format preference order for an OS.
// zst is preferred as the modern standard, but availability varies.
func defaultFormats(os string) []string {
switch os {
case "windows":
return []string{
".tar.zst",
".tar.xz",
".zip",
".tar.gz",
".exe.xz",
".7z",
".exe",
".msi",
"git",
}
case "darwin":
return []string{
".tar.zst",
".tar.xz",
".zip",
".tar.gz",
".gz",
".app.zip",
".dmg",
".pkg",
"git",
}
default:
// Linux and other POSIX.
return []string{
".tar.zst",
".tar.xz",
".tar.gz",
".gz",
".zip",
".xz",
"git",
}
}
}

View File

@@ -0,0 +1,290 @@
package resolver_test
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"github.com/webinstall/webi-installers/internal/resolver"
"github.com/webinstall/webi-installers/internal/storage"
)
func loadAssets(t *testing.T, pkg string) []storage.Asset {
t.Helper()
cacheDir := filepath.Join("..", "..", "_cache", "2026-03")
path := filepath.Join(cacheDir, pkg+".json")
data, err := os.ReadFile(path)
if err != nil {
t.Skipf("no cache file for %s: %v", pkg, err)
}
var lc storage.LegacyCache
if err := json.Unmarshal(data, &lc); err != nil {
t.Fatalf("parse %s: %v", pkg, err)
}
pd := storage.ImportLegacy(lc)
return pd.Assets
}
// TestCacheResolveAllPackages loads every package from the cache and verifies
// the resolver finds a match for each standard platform.
func TestCacheResolveAllPackages(t *testing.T) {
cacheDir := filepath.Join("..", "..", "_cache", "2026-03")
entries, err := os.ReadDir(cacheDir)
if err != nil {
t.Skipf("no cache dir: %v", err)
}
var pkgs []string
for _, e := range entries {
if strings.HasSuffix(e.Name(), ".json") {
pkgs = append(pkgs, strings.TrimSuffix(e.Name(), ".json"))
}
}
if len(pkgs) < 50 {
t.Fatalf("expected at least 50 packages, got %d", len(pkgs))
}
platforms := []struct {
name string
os string
arch string
}{
{"darwin-arm64", "darwin", "aarch64"},
{"darwin-amd64", "darwin", "x86_64"},
{"linux-amd64", "linux", "x86_64"},
{"linux-arm64", "linux", "aarch64"},
{"windows-amd64", "windows", "x86_64"},
}
for _, pkg := range pkgs {
t.Run(pkg, func(t *testing.T) {
assets := loadAssets(t, pkg)
if len(assets) == 0 {
t.Skip("no releases")
}
// Determine which OSes this package has.
osSet := make(map[string]bool)
for _, a := range assets {
if a.OS != "" {
osSet[a.OS] = true
}
}
// Also check for platform-agnostic assets.
hasAgnostic := false
for _, a := range assets {
if a.OS == "" {
hasAgnostic = true
break
}
}
for _, plat := range platforms {
supported := osSet[plat.os] ||
osSet["ANYOS"] ||
hasAgnostic ||
(plat.os != "windows" && (osSet["posix_2017"] || osSet["posix_2024"]))
if !supported {
continue
}
t.Run(plat.name, func(t *testing.T) {
res, err := resolver.Resolve(assets, resolver.Request{
OS: plat.os,
Arch: plat.arch,
})
if err != nil {
// Not a test failure — some packages don't have
// all arch builds. Log for visibility.
t.Logf("WARN: no match for %s on %s (has OSes: %v)",
pkg, plat.name, sortedOSes(osSet))
return
}
if res.Version == "" {
t.Error("matched but Version is empty")
}
if res.Asset.Download == "" {
t.Error("matched but Download is empty")
}
})
}
})
}
}
// TestCacheKnownPackages verifies specific packages resolve correctly.
var knownPackages = []struct {
pkg string
version string // expected latest stable version prefix
platforms []string
}{
{"bat", "0.26", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"caddy", "2.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"delta", "0.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"fd", "10.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"fzf", "0.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"gh", "2.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"rg", "", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"node", "", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"terraform", "", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"zig", "", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
}
func TestCacheKnownPackages(t *testing.T) {
platMap := map[string]resolver.Request{
"darwin-arm64": {OS: "darwin", Arch: "aarch64"},
"darwin-amd64": {OS: "darwin", Arch: "x86_64"},
"linux-amd64": {OS: "linux", Arch: "x86_64"},
"linux-arm64": {OS: "linux", Arch: "aarch64"},
"windows-amd64": {OS: "windows", Arch: "x86_64"},
}
for _, kp := range knownPackages {
t.Run(kp.pkg, func(t *testing.T) {
assets := loadAssets(t, kp.pkg)
for _, platName := range kp.platforms {
req := platMap[platName]
t.Run(platName, func(t *testing.T) {
res, err := resolver.Resolve(assets, req)
if err != nil {
t.Fatalf("no match for %s on %s", kp.pkg, platName)
}
if kp.version != "" {
v := strings.TrimPrefix(res.Version, "v")
if !strings.HasPrefix(v, kp.version) {
t.Errorf("Version = %q, want prefix %q", res.Version, kp.version)
}
}
})
}
})
}
}
// TestCacheVersionConstraints tests version pinning with real data.
func TestCacheVersionConstraints(t *testing.T) {
tests := []struct {
pkg string
version string
wantPfx string
}{
{"bat", "0.25", "0.25"},
{"bat", "0.26", "0.26"},
{"gh", "2.40", "2.40"},
{"node", "20", "20."},
{"node", "22", "22."},
}
for _, tt := range tests {
t.Run(tt.pkg+"@"+tt.version, func(t *testing.T) {
assets := loadAssets(t, tt.pkg)
res, err := resolver.Resolve(assets, resolver.Request{
OS: "linux",
Arch: "x86_64",
Version: tt.version,
})
if err != nil {
t.Fatalf("no match for %s@%s", tt.pkg, tt.version)
}
v := strings.TrimPrefix(res.Version, "v")
if !strings.HasPrefix(v, tt.wantPfx) {
t.Errorf("Version = %q, want prefix %q", res.Version, tt.wantPfx)
}
})
}
}
// TestCacheArchFallback verifies Rosetta-style fallback with real data.
func TestCacheArchFallback(t *testing.T) {
// awless only has amd64 builds — macOS ARM64 should fall back.
assets := loadAssets(t, "awless")
res, err := resolver.Resolve(assets, resolver.Request{
OS: "darwin",
Arch: "aarch64",
})
if err != nil {
t.Fatal("expected Rosetta 2 fallback for awless")
}
if res.Asset.Arch != "x86_64" {
t.Errorf("Arch = %q, want x86_64", res.Asset.Arch)
}
}
// TestCacheGitPackages verifies git-only packages resolve on any platform.
func TestCacheGitPackages(t *testing.T) {
gitPkgs := []string{"vim-essentials", "vim-spell"}
for _, pkg := range gitPkgs {
t.Run(pkg, func(t *testing.T) {
assets := loadAssets(t, pkg)
if len(assets) == 0 {
t.Skip("no releases")
}
// Should work on any platform.
for _, plat := range []struct {
os, arch string
}{
{"linux", "x86_64"},
{"darwin", "aarch64"},
{"windows", "x86_64"},
} {
res, err := resolver.Resolve(assets, resolver.Request{
OS: plat.os,
Arch: plat.arch,
})
if err != nil {
t.Errorf("expected match on %s-%s", plat.os, plat.arch)
continue
}
if res.Asset.Format != "git" {
t.Errorf("format = %q, want git", res.Asset.Format)
}
}
})
}
}
// TestCacheLibcPreference tests explicit libc selection.
// bat is Rust — its musl builds are static (tagged 'none').
func TestCacheLibcPreference(t *testing.T) {
assets := loadAssets(t, "bat")
// Musl host requesting bat: gets static musl build (tagged 'none').
res, err := resolver.Resolve(assets, resolver.Request{
OS: "linux",
Arch: "x86_64",
Libc: "musl",
})
if err != nil {
t.Fatal("expected match for musl host")
}
if res.Asset.Libc != "none" {
t.Errorf("Libc = %q, want none (static musl)", res.Asset.Libc)
}
// Explicit gnu.
res, err = resolver.Resolve(assets, resolver.Request{
OS: "linux",
Arch: "x86_64",
Libc: "gnu",
})
if err != nil {
t.Fatal("expected gnu match")
}
if res.Asset.Libc != "gnu" {
t.Errorf("Libc = %q, want gnu", res.Asset.Libc)
}
}
func sortedOSes(m map[string]bool) []string {
var keys []string
for k := range m {
keys = append(keys, k)
}
return keys
}

View File

@@ -0,0 +1,397 @@
package resolver
import (
"testing"
"github.com/webinstall/webi-installers/internal/storage"
)
func TestResolveSimple(t *testing.T) {
assets := []storage.Asset{
{
Filename: "bat-v0.25.0-x86_64-unknown-linux-musl.tar.gz",
Version: "0.25.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Libc: "musl",
Format: ".tar.gz",
Download: "https://example.com/bat-0.25.0-linux-x86_64.tar.gz",
},
{
Filename: "bat-v0.26.0-x86_64-unknown-linux-musl.tar.gz",
Version: "0.26.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Libc: "musl",
Format: ".tar.gz",
Download: "https://example.com/bat-0.26.0-linux-x86_64.tar.gz",
},
{
Filename: "bat-v0.26.0-aarch64-unknown-linux-musl.tar.gz",
Version: "0.26.0",
Channel: "stable",
OS: "linux",
Arch: "aarch64",
Libc: "musl",
Format: ".tar.gz",
Download: "https://example.com/bat-0.26.0-linux-aarch64.tar.gz",
},
{
Filename: "bat-v0.26.0-x86_64-pc-windows-msvc.zip",
Version: "0.26.0",
Channel: "stable",
OS: "windows",
Arch: "x86_64",
Libc: "msvc",
Format: ".zip",
Download: "https://example.com/bat-0.26.0-windows-x86_64.zip",
},
{
Filename: "bat-v0.26.0-x86_64-apple-darwin.tar.gz",
Version: "0.26.0",
Channel: "stable",
OS: "darwin",
Arch: "x86_64",
Format: ".tar.gz",
Download: "https://example.com/bat-0.26.0-darwin-x86_64.tar.gz",
},
}
t.Run("latest linux x86_64", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
})
if err != nil {
t.Fatal(err)
}
if res.Version != "0.26.0" {
t.Errorf("version = %q, want 0.26.0", res.Version)
}
if res.Asset.OS != "linux" {
t.Errorf("os = %q, want linux", res.Asset.OS)
}
if res.Asset.Arch != "x86_64" {
t.Errorf("arch = %q, want x86_64", res.Asset.Arch)
}
})
t.Run("latest linux aarch64", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "aarch64",
})
if err != nil {
t.Fatal(err)
}
if res.Version != "0.26.0" {
t.Errorf("version = %q, want 0.26.0", res.Version)
}
if res.Asset.Arch != "aarch64" {
t.Errorf("arch = %q, want aarch64", res.Asset.Arch)
}
})
t.Run("version prefix 0.25", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
Version: "0.25",
})
if err != nil {
t.Fatal(err)
}
if res.Version != "0.25.0" {
t.Errorf("version = %q, want 0.25.0", res.Version)
}
})
t.Run("darwin arm64 falls back to x86_64", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "darwin",
Arch: "aarch64",
})
if err != nil {
t.Fatal(err)
}
if res.Asset.Arch != "x86_64" {
t.Errorf("arch = %q, want x86_64 (Rosetta fallback)", res.Asset.Arch)
}
})
t.Run("no match returns error", func(t *testing.T) {
_, err := Resolve(assets, Request{
OS: "freebsd",
Arch: "x86_64",
})
if err != ErrNoMatch {
t.Errorf("err = %v, want ErrNoMatch", err)
}
})
t.Run("windows gets zip", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "windows",
Arch: "x86_64",
})
if err != nil {
t.Fatal(err)
}
if res.Asset.Format != ".zip" {
t.Errorf("format = %q, want .zip", res.Asset.Format)
}
})
}
func TestResolveChannels(t *testing.T) {
assets := []storage.Asset{
{
Filename: "tool-v2.0.0-rc1-linux-x86_64.tar.gz",
Version: "2.0.0-rc1",
Channel: "rc",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
},
{
Filename: "tool-v1.5.0-linux-x86_64.tar.gz",
Version: "1.5.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
},
{
Filename: "tool-v2.0.0-beta2-linux-x86_64.tar.gz",
Version: "2.0.0-beta2",
Channel: "beta",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
},
}
t.Run("stable skips rc and beta", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
})
if err != nil {
t.Fatal(err)
}
if res.Version != "1.5.0" {
t.Errorf("version = %q, want 1.5.0", res.Version)
}
})
t.Run("rc includes rc and stable", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
Channel: "rc",
})
if err != nil {
t.Fatal(err)
}
if res.Version != "2.0.0-rc1" {
t.Errorf("version = %q, want 2.0.0-rc1", res.Version)
}
})
t.Run("beta includes beta, rc, and stable", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
Channel: "beta",
})
if err != nil {
t.Fatal(err)
}
// beta2 sorts after rc1 for the same numeric version (2.0.0),
// but rc1 is more stable. However, the user asked for beta channel
// which includes everything — and beta sorts before rc alphabetically.
// With lexver: 2.0.0-rc1 > 2.0.0-beta2 (rc > beta alphabetically).
if res.Version != "2.0.0-rc1" {
t.Errorf("version = %q, want 2.0.0-rc1", res.Version)
}
})
}
func TestResolveVariants(t *testing.T) {
assets := []storage.Asset{
{
Filename: "ollama-linux-amd64.tgz",
Version: "0.6.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
},
{
Filename: "ollama-linux-amd64-rocm.tgz",
Version: "0.6.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
Variants: []string{"rocm"},
},
}
t.Run("no variant prefers plain", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
})
if err != nil {
t.Fatal(err)
}
if len(res.Asset.Variants) != 0 {
t.Errorf("variants = %v, want empty", res.Asset.Variants)
}
})
t.Run("explicit variant selects it", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
Variant: "rocm",
})
if err != nil {
t.Fatal(err)
}
if !hasVariant(res.Asset.Variants, "rocm") {
t.Errorf("variants = %v, want [rocm]", res.Asset.Variants)
}
})
}
func TestResolveFormatPreference(t *testing.T) {
assets := []storage.Asset{
{
Filename: "tool-v1.0.0-linux-x86_64.tar.gz",
Version: "1.0.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
},
{
Filename: "tool-v1.0.0-linux-x86_64.tar.xz",
Version: "1.0.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.xz",
},
{
Filename: "tool-v1.0.0-linux-x86_64.tar.zst",
Version: "1.0.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.zst",
},
}
t.Run("default prefers zst", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
})
if err != nil {
t.Fatal(err)
}
if res.Asset.Format != ".tar.zst" {
t.Errorf("format = %q, want .tar.zst", res.Asset.Format)
}
})
t.Run("explicit format preference", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
Formats: []string{".tar.gz"},
})
if err != nil {
t.Fatal(err)
}
if res.Asset.Format != ".tar.gz" {
t.Errorf("format = %q, want .tar.gz", res.Asset.Format)
}
})
}
func TestResolveGitAssets(t *testing.T) {
assets := []storage.Asset{
{
Filename: "vim-commentary-v1.2",
Version: "1.2",
Channel: "stable",
Format: "git",
Download: "https://github.com/tpope/vim-commentary.git",
},
{
Filename: "vim-commentary-v1.1",
Version: "1.1",
Channel: "stable",
Format: "git",
Download: "https://github.com/tpope/vim-commentary.git",
},
}
t.Run("git assets match any platform", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
})
if err != nil {
t.Fatal(err)
}
if res.Version != "1.2" {
t.Errorf("version = %q, want 1.2", res.Version)
}
if res.Asset.Format != "git" {
t.Errorf("format = %q, want git", res.Asset.Format)
}
})
}
func TestResolveLTS(t *testing.T) {
assets := []storage.Asset{
{
Filename: "node-v22.0.0-linux-x64.tar.gz",
Version: "22.0.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
LTS: false,
},
{
Filename: "node-v20.15.0-linux-x64.tar.gz",
Version: "20.15.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
LTS: true,
},
}
t.Run("LTS selects older LTS version", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
LTS: true,
})
if err != nil {
t.Fatal(err)
}
if res.Version != "20.15.0" {
t.Errorf("version = %q, want 20.15.0", res.Version)
}
})
}

View File

@@ -67,6 +67,12 @@ func (la LegacyAsset) ToAsset() Asset {
arch = "x86_64"
case "arm64":
arch = "aarch64"
case "armv7l":
arch = "armv7"
case "armv6l":
arch = "armv6"
case "arm":
arch = "armv5"
case "*":
arch = ""
}

View File

@@ -0,0 +1,295 @@
// Package pgstore implements [storage.Store] on PostgreSQL.
//
// Schema uses double-buffering: two asset generations per package (0 and 1).
// The active generation pointer in webi_packages is updated atomically on
// Commit, so readers always see a complete consistent snapshot.
//
// Write path:
//
// BeginRefresh → clears inactive generation, returns tx
// Put → stages assets in-memory
// Commit → bulk-inserts assets (COPY), swaps generation pointer
//
// Read path:
//
// Load → reads active generation from webi_packages, fetches assets
//
// Connection string format: standard libpq / pgx DSN, e.g.:
//
// postgres://user:pass@host/dbname?sslmode=require
// host=localhost user=webi dbname=webi sslmode=disable
package pgstore
import (
"context"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/webinstall/webi-installers/internal/storage"
)
// Schema holds the DDL for creating the required tables.
// Run once on startup or deploy to ensure the schema exists.
const Schema = `
CREATE TABLE IF NOT EXISTS webi_packages (
name TEXT NOT NULL PRIMARY KEY,
active_gen SMALLINT NOT NULL DEFAULT 0,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS webi_assets (
id BIGSERIAL PRIMARY KEY,
pkg TEXT NOT NULL,
gen SMALLINT NOT NULL,
filename TEXT NOT NULL DEFAULT '',
version TEXT NOT NULL DEFAULT '',
lts BOOLEAN NOT NULL DEFAULT FALSE,
channel TEXT NOT NULL DEFAULT '',
date TEXT NOT NULL DEFAULT '',
os TEXT NOT NULL DEFAULT '',
arch TEXT NOT NULL DEFAULT '',
libc TEXT NOT NULL DEFAULT '',
format TEXT NOT NULL DEFAULT '',
download TEXT NOT NULL DEFAULT '',
extra TEXT NOT NULL DEFAULT '',
variants TEXT[] NOT NULL DEFAULT '{}'
);
CREATE INDEX IF NOT EXISTS webi_assets_pkg_gen ON webi_assets (pkg, gen);
`
// Store is a PostgreSQL-backed asset store.
type Store struct {
pool *pgxpool.Pool
}
// New opens a connection pool to the given DSN and applies the schema.
// Returns an error if the connection or schema creation fails.
func New(ctx context.Context, dsn string) (*Store, error) {
cfg, err := pgxpool.ParseConfig(dsn)
if err != nil {
return nil, fmt.Errorf("pgstore: parse dsn: %w", err)
}
pool, err := pgxpool.NewWithConfig(ctx, cfg)
if err != nil {
return nil, fmt.Errorf("pgstore: connect: %w", err)
}
if err := applySchema(ctx, pool); err != nil {
pool.Close()
return nil, err
}
return &Store{pool: pool}, nil
}
// Close releases the connection pool.
func (s *Store) Close() {
s.pool.Close()
}
// ListPackages returns the names of all packages in the store.
func (s *Store) ListPackages(ctx context.Context) ([]string, error) {
rows, err := s.pool.Query(ctx,
`SELECT name FROM webi_packages ORDER BY name`,
)
if err != nil {
return nil, fmt.Errorf("pgstore: list packages: %w", err)
}
defer rows.Close()
var pkgs []string
for rows.Next() {
var name string
if err := rows.Scan(&name); err != nil {
return nil, fmt.Errorf("pgstore: scan package name: %w", err)
}
pkgs = append(pkgs, name)
}
return pkgs, rows.Err()
}
// Load returns all assets for a package using the active generation.
// Returns nil (not an error) if the package is not cached.
func (s *Store) Load(ctx context.Context, pkg string) (*storage.PackageData, error) {
// Fetch active generation and updated_at.
var gen int16
var updatedAt time.Time
err := s.pool.QueryRow(ctx,
`SELECT active_gen, updated_at FROM webi_packages WHERE name = $1`,
pkg,
).Scan(&gen, &updatedAt)
if err == pgx.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("pgstore: load %s: %w", pkg, err)
}
// Fetch all assets for this generation.
rows, err := s.pool.Query(ctx, `
SELECT filename, version, lts, channel, date,
os, arch, libc, format, download, extra, variants
FROM webi_assets
WHERE pkg = $1 AND gen = $2
ORDER BY id
`, pkg, gen)
if err != nil {
return nil, fmt.Errorf("pgstore: load assets %s: %w", pkg, err)
}
defer rows.Close()
var assets []storage.Asset
for rows.Next() {
var a storage.Asset
if err := rows.Scan(
&a.Filename, &a.Version, &a.LTS, &a.Channel, &a.Date,
&a.OS, &a.Arch, &a.Libc, &a.Format, &a.Download,
&a.Extra, &a.Variants,
); err != nil {
return nil, fmt.Errorf("pgstore: scan asset %s: %w", pkg, err)
}
assets = append(assets, a)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("pgstore: rows %s: %w", pkg, err)
}
return &storage.PackageData{
Assets: assets,
UpdatedAt: updatedAt,
}, nil
}
// BeginRefresh starts a write transaction for a package.
// It determines the inactive generation and clears it, ready for new data.
func (s *Store) BeginRefresh(ctx context.Context, pkg string) (storage.RefreshTx, error) {
// Determine which generation to write into (the inactive one).
var activeGen int16
err := s.pool.QueryRow(ctx,
`SELECT active_gen FROM webi_packages WHERE name = $1`,
pkg,
).Scan(&activeGen)
if err != nil && err != pgx.ErrNoRows {
return nil, fmt.Errorf("pgstore: begin refresh %s: %w", pkg, err)
}
// If package doesn't exist yet, activeGen defaults to 0 and we write to gen 1.
// If package exists, we write to the inactive generation (1 - activeGen).
var writeGen int16
if err == pgx.ErrNoRows {
writeGen = 1
} else {
writeGen = 1 - activeGen
}
// Clear the write generation so we start fresh.
if _, err := s.pool.Exec(ctx,
`DELETE FROM webi_assets WHERE pkg = $1 AND gen = $2`,
pkg, writeGen,
); err != nil {
return nil, fmt.Errorf("pgstore: clear gen %d for %s: %w", writeGen, pkg, err)
}
return &refreshTx{
pool: s.pool,
pkg: pkg,
gen: writeGen,
}, nil
}
// refreshTx is an in-progress write for one package.
type refreshTx struct {
pool *pgxpool.Pool
pkg string
gen int16
assets []storage.Asset
}
// Put stages assets for writing. May be called multiple times.
func (tx *refreshTx) Put(assets []storage.Asset) error {
tx.assets = append(tx.assets, assets...)
return nil
}
// Commit bulk-inserts all staged assets, then atomically swaps the
// active generation pointer in webi_packages.
func (tx *refreshTx) Commit(ctx context.Context) error {
if len(tx.assets) == 0 {
return tx.swapGeneration(ctx)
}
// Build rows for pgx.CopyFromRows.
rows := make([][]any, len(tx.assets))
for i, a := range tx.assets {
variants := a.Variants
if variants == nil {
variants = []string{}
}
rows[i] = []any{
tx.pkg,
tx.gen,
a.Filename,
a.Version,
a.LTS,
a.Channel,
a.Date,
a.OS,
a.Arch,
a.Libc,
a.Format,
a.Download,
a.Extra,
variants,
}
}
cols := []string{
"pkg", "gen",
"filename", "version", "lts", "channel", "date",
"os", "arch", "libc", "format", "download", "extra", "variants",
}
_, err := tx.pool.CopyFrom(ctx,
pgx.Identifier{"webi_assets"},
cols,
pgx.CopyFromRows(rows),
)
if err != nil {
return fmt.Errorf("pgstore: copy assets %s: %w", tx.pkg, err)
}
return tx.swapGeneration(ctx)
}
// swapGeneration atomically updates the active generation pointer.
func (tx *refreshTx) swapGeneration(ctx context.Context) error {
_, err := tx.pool.Exec(ctx, `
INSERT INTO webi_packages (name, active_gen, updated_at)
VALUES ($1, $2, now())
ON CONFLICT (name)
DO UPDATE SET active_gen = $2, updated_at = now()
`, tx.pkg, tx.gen)
if err != nil {
return fmt.Errorf("pgstore: swap gen %s: %w", tx.pkg, err)
}
tx.assets = nil
return nil
}
// Rollback discards all staged assets without writing anything.
func (tx *refreshTx) Rollback() error {
tx.assets = nil
return nil
}
// applySchema runs the schema DDL idempotently.
func applySchema(ctx context.Context, pool *pgxpool.Pool) error {
if _, err := pool.Exec(ctx, Schema); err != nil {
return fmt.Errorf("pgstore: apply schema: %w", err)
}
return nil
}

View File

@@ -0,0 +1,247 @@
// Package uadetect identifies the requesting agent's OS, CPU architecture,
// and libc so the server can select the correct release artifact.
//
// An agent identifies itself through multiple signals:
// - The User-Agent header: Webi's bootstrap scripts send "$(uname -srm)",
// e.g. "Darwin 23.1.0 arm64". Browsers, curl, and PowerShell send their
// own UA strings.
// - Query parameters: ?os=linux&arch=arm64 are an explicit declaration
// that takes precedence over the header.
//
// Use [FromRequest] to detect from an HTTP request (preferred).
// Use [Parse] to detect from a raw UA string.
package uadetect
import (
"net/http"
"strings"
"github.com/webinstall/webi-installers/internal/buildmeta"
)
// Result holds the detected platform info from a User-Agent string.
type Result struct {
OS buildmeta.OS
Arch buildmeta.Arch
Libc buildmeta.Libc
}
// FromRequest detects the agent's platform from an HTTP request.
// Query parameters ?os and ?arch override the User-Agent header.
func FromRequest(r *http.Request) Result {
qOS := r.URL.Query().Get("os")
qArch := r.URL.Query().Get("arch")
var ua string
switch {
case qOS != "" && qArch != "":
ua = qOS + " " + qArch
case qOS != "":
ua = qOS
case qArch != "":
ua = qArch
default:
ua = r.Header.Get("User-Agent")
}
return Parse(ua)
}
// Parse extracts OS, arch, and libc from a User-Agent string.
func Parse(ua string) Result {
if ua == "-" {
return Result{}
}
tokens := tokenize(ua)
return Result{
OS: matchOS(tokens),
Arch: matchArch(tokens),
Libc: matchLibc(tokens),
}
}
// tokenize splits a User-Agent into lowercase tokens for matching.
// Splits on whitespace, '/', and ';', since UAs come in various forms:
//
// "Darwin 23.1.0 arm64" (uname -srm)
// "PowerShell/7.3.0" (PowerShell)
// "MS AMD64" (Windows shorthand)
// "Macintosh; Intel Mac OS X 10_15_7" (browser)
func tokenize(ua string) []string {
// Strip xnu kernel info that can mislead arch detection under Rosetta.
// "xnu-7195.60.75~1/RELEASE_ARM64_T8101" contains ARM64 even when
// running as x86_64. This only appears in verbose uname output.
if i := strings.Index(ua, "xnu-"); i >= 0 {
end := strings.IndexByte(ua[i:], ' ')
if end < 0 {
ua = ua[:i]
} else {
ua = ua[:i] + ua[i+end:]
}
}
return strings.FieldsFunc(strings.ToLower(ua), func(r rune) bool {
return r == ' ' || r == '/' || r == ';' || r == '\t'
})
}
// matchOS identifies the operating system from tokens.
// Order matters: Android before Linux, Linux before Windows (for WSL).
func matchOS(tokens []string) buildmeta.OS {
has := func(s string) bool {
for _, t := range tokens {
if strings.Contains(t, s) {
return true
}
}
return false
}
// Android must be checked before Linux.
if has("android") {
return buildmeta.OSAndroid
}
if has("darwin") || has("macos") || has("macintosh") || has("iphone") || has("ios") || has("ipad") {
return buildmeta.OSDarwin
}
// "mac" alone (not in "macintosh" which is already matched)
for _, t := range tokens {
if t == "mac" {
return buildmeta.OSDarwin
}
}
// FreeBSD before Linux (both are POSIX, but FreeBSD never reports "linux").
if has("freebsd") {
return buildmeta.OSFreeBSD
}
// Linux before Windows because WSL UAs contain both "linux" and "microsoft".
// But exclude Cygwin/Msys/MINGW which report Linux-like strings on Windows.
if has("linux") && !has("cygwin") && !has("msysgit") && !has("msys") && !has("mingw") {
return buildmeta.OSLinux
}
// Cygwin, Msys, and MINGW are Windows environments.
if has("windows") || has("win32") || has("microsoft") || has("powershell") ||
has("cygwin") || has("msys") || has("mingw") {
return buildmeta.OSWindows
}
for _, t := range tokens {
if t == "ms" || t == "win" {
return buildmeta.OSWindows
}
}
// Fallback: curl and wget imply a POSIX system, almost always Linux.
if has("curl") || has("wget") {
return buildmeta.OSLinux
}
return ""
}
// matchArch identifies the CPU architecture from tokens.
// More specific patterns are checked before less specific ones.
func matchArch(tokens []string) buildmeta.Arch {
has := func(s string) bool {
for _, t := range tokens {
if strings.Contains(t, s) {
return true
}
}
return false
}
exact := func(s string) bool {
for _, t := range tokens {
if t == s {
return true
}
}
return false
}
// ARM 64-bit (most specific first)
if has("aarch64") || has("arm64") || has("armv8") {
return buildmeta.ArchARM64
}
// ARM 32-bit variants
if has("armv7") || has("arm32") {
return buildmeta.ArchARMv7
}
if has("armv6") {
return buildmeta.ArchARMv6
}
// Bare "arm" without a version qualifier → armv6 (conservative).
if exact("arm") {
return buildmeta.ArchARMv6
}
// POWER (check before generic 64-bit)
if has("ppc64le") {
return buildmeta.ArchPPC64LE
}
if has("ppc64") {
return buildmeta.ArchPPC64
}
// s390x (IBM Z)
if has("s390x") {
return buildmeta.ArchS390X
}
// RISC-V
if has("riscv64") {
return buildmeta.ArchRISCV64
}
// MIPS (check before generic 64-bit)
if has("mips64") {
return buildmeta.ArchMIPS64
}
if has("mips") {
return buildmeta.ArchMIPS
}
// x86-64
if has("x86_64") || has("amd64") || exact("x64") {
return buildmeta.ArchAMD64
}
// x86 32-bit (after x86_64 to avoid false match)
if has("i386") || has("i686") || exact("x86") {
return buildmeta.ArchX86
}
return ""
}
// matchLibc identifies the C library from tokens.
func matchLibc(tokens []string) buildmeta.Libc {
has := func(s string) bool {
for _, t := range tokens {
if strings.Contains(t, s) {
return true
}
}
return false
}
if has("musl") {
return buildmeta.LibcMusl
}
// Don't match "microsoft" — it appears in WSL kernel version strings
// (e.g. "5.15.146.1-microsoft-standard-WSL2") and doesn't indicate MSVC.
if has("msvc") || has("windows") {
return buildmeta.LibcMSVC
}
if has("gnu") || has("glibc") || has("linux") {
return buildmeta.LibcGNU
}
return buildmeta.LibcNone
}

81
scripts/deploy-webid.sh Executable file
View File

@@ -0,0 +1,81 @@
#!/bin/sh
set -e
set -u
# Build and deploy webid to a target host
g_host="${1:-next.webi.sh}"
g_bin="webid"
g_out="agents/tmp/${g_bin}"
g_remote_bin="~/bin/${g_bin}"
case "${g_host}" in
beta.webi.sh) g_remote_conf="~/srv/beta.webinstall.dev/installers/" ;;
next.webi.sh) g_remote_conf="~/srv/next.webinstall.dev/installers/" ;;
*) g_remote_conf="~/srv/webid/installers/" ;;
esac
fn_build() {
b_version="$(git describe --tags --always 2> /dev/null || echo '0.0.0-dev')"
b_commit="$(git rev-parse --short HEAD)"
b_date="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
b_ldflags="-X main.version=${b_version} -X main.commit=${b_commit} -X main.date=${b_date}"
printf 'Building %s %s %s (%s)...\n' "${g_bin}" "${b_version}" "${b_commit}" "${b_date}"
GOOS=linux GOARCH=amd64 GOAMD64=v2 go build -ldflags "${b_ldflags}" -o "${g_out}" ./cmd/webid
printf 'Built: %s\n' "${g_out}"
}
fn_deploy() {
printf 'Stopping %s on %s...\n' "${g_bin}" "${g_host}"
ssh "${g_host}" "~/.local/bin/serviceman stop ${g_bin}" 2> /dev/null || true
printf 'Uploading binary...\n'
scp "${g_out}" "${g_host}:${g_remote_bin}"
printf 'Syncing install scripts and templates...\n'
rsync -av \
--exclude='_cache' --exclude='.git' --exclude='agents' \
--exclude='bin' --exclude='cmd' --exclude='internal' \
--exclude='docs' --exclude='scripts' --exclude='node_modules' \
--include='*/' --include='install.sh' --include='install.ps1' \
--include='_webi/*.tpl.sh' --include='_webi/*.tpl.ps1' \
--exclude='*' \
./ "${g_host}:${g_remote_conf}"
}
fn_start() {
printf 'Starting %s...\n' "${g_bin}"
ssh "${g_host}" "~/.local/bin/serviceman start ${g_bin}" || {
printf 'Service not configured. Run serviceman add on the host:\n'
printf ' serviceman add --name %s \\\n' "${g_bin}"
printf ' --workdir %s -- \\\n' "${g_remote_conf}"
printf ' %s \\\n' "${g_remote_bin}"
printf ' --addr :3082 \\\n'
printf ' --legacy ~/.cache/webi/legacy \\\n'
printf ' --installers %s\n' "${g_remote_conf}"
exit 1
}
}
fn_verify() {
printf 'Waiting 3s for startup...\n'
sleep 3
printf 'Checking version...\n'
ssh "${g_host}" "${g_remote_bin} -V"
printf 'Checking health...\n'
ssh "${g_host}" "curl -s http://localhost:3082/api/releases/bat.json | head -c 100"
printf '\n'
printf 'Checking logs...\n'
ssh "${g_host}" "sudo journalctl -u ${g_bin} --no-pager -n 5"
}
fn_build
fn_deploy
fn_start
fn_verify
printf '\nDone. %s deployed to %s.\n' "${g_bin}" "${g_host}"