feat(webid): add HTTP API server (releases, resolve, installer scripts)

Serves the HTTP API for webinstall.dev:
- GET /api/releases/{pkg}.json — classified release list from fsstore
- GET /v1/resolve/{pkg} — resolve OS/arch/version to a specific asset
- GET /api/installers/{pkg}.sh — render installer shell script
- GET /api/installers/{pkg}.ps1 — render installer PowerShell script

Git-clone packages include git_tag and git_commit_hash in responses.
UA detection infers OS/arch from User-Agent when not query-specified.
This commit is contained in:
AJ ONeal
2026-05-16 21:53:05 -06:00
parent 59b2956d60
commit ac59e4728e
10 changed files with 2259 additions and 2 deletions
+1
View File
@@ -17,6 +17,7 @@ $Env:WEBI_HOST = 'https://webinstall.dev'
#$Env:PKG_NAME = node
#$Env:WEBI_VERSION = v12.16.2
#$Env:WEBI_GIT_TAG = 12.16.2
#$Env:WEBI_GIT_COMMIT_HASH =
#$Env:WEBI_PKG_URL = "https://.../node-....zip"
#$Env:WEBI_PKG_FILE = "node-v12.16.2-win-x64.zip"
#$Env:WEBI_PKG_PATHNAME = "node-v12.16.2-win-x64.zip"
+1
View File
@@ -15,6 +15,7 @@ __bootstrap_webi() {
# TODO not sure if BUILD is the best name for this
#WEBI_BUILD=
#WEBI_GIT_TAG=
#WEBI_GIT_COMMIT_HASH=
#WEBI_LTS=
#WEBI_CHANNEL=
#WEBI_EXT=
+146
View File
@@ -0,0 +1,146 @@
package main
import (
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
// TestBootstrapCurlPipe verifies the /{pkg} route returns the curl-pipe bootstrap.
func TestBootstrapCurlPipe(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/bat@stable")
if code != 200 {
t.Fatalf("status %d: %s", code, body[:min(len(body), 200)])
}
// Should contain the bootstrap env vars.
if !strings.Contains(body, "WEBI_PKG=") {
t.Error("missing WEBI_PKG= in bootstrap")
}
if !strings.Contains(body, "WEBI_HOST=") {
t.Error("missing WEBI_HOST= in bootstrap")
}
if !strings.Contains(body, "WEBI_CHECKSUM=") {
t.Error("missing WEBI_CHECKSUM= in bootstrap")
}
// Should NOT contain the full installer (install.sh content).
// The bootstrap just downloads and runs webi.
if strings.Contains(body, "pkg_install()") {
t.Error("bootstrap should not contain pkg_install — that's the full installer")
}
t.Logf("bootstrap size: %d bytes", len(body))
}
// TestInstallerFull verifies /api/installers/{pkg}.sh returns the full installer.
func TestInstallerFull(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
// Use a webi-style User-Agent so the server can detect platform.
code, body := getWithUA(t, ts, "/api/installers/bat@stable.sh", "aarch64/unknown Darwin/24.2.0 libc")
if code != 200 {
t.Fatalf("status %d: %s", code, body[:min(len(body), 500)])
}
// Should contain resolved release info.
if !strings.Contains(body, "WEBI_VERSION=") {
t.Error("missing WEBI_VERSION= in installer")
}
if !strings.Contains(body, "WEBI_PKG_URL=") {
t.Error("missing WEBI_PKG_URL= in installer")
}
if !strings.Contains(body, "PKG_NAME=") {
t.Error("missing PKG_NAME= in installer")
}
// Should contain the package's install.sh content (embedded).
if !strings.Contains(body, "pkg_") {
t.Error("installer should contain pkg_ functions from install.sh")
}
t.Logf("installer size: %d bytes", len(body))
}
// TestInstallerPowerShell verifies /api/installers/{pkg}.ps1 returns a PowerShell installer.
func TestInstallerPowerShell(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "node"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := getWithUA(t, ts, "/api/installers/node@stable.ps1", "AMD64/unknown Windows/10.0.19045 msvc")
if code != 200 {
t.Fatalf("status %d: %s", code, body[:min(len(body), 500)])
}
if !strings.Contains(body, "$Env:WEBI_VERSION") {
t.Error("missing $Env:WEBI_VERSION in PS1 installer")
}
if !strings.Contains(body, "$Env:WEBI_PKG_URL") {
t.Error("missing $Env:WEBI_PKG_URL in PS1 installer")
}
if !strings.Contains(body, "$Env:PKG_NAME") {
t.Error("missing $Env:PKG_NAME in PS1 installer")
}
t.Logf("PS1 installer size: %d bytes", len(body))
}
// TestInstallerSelfHosted verifies selfhosted packages get a script without resolution.
func TestInstallerSelfHosted(t *testing.T) {
_, ts := newTestServer(t)
// ssh-utils is selfhosted — has install.sh but no releases.conf.
code, body := getWithUA(t, ts, "/api/installers/ssh-utils.sh", "aarch64/unknown Darwin/24.2.0 libc")
if code == 404 {
t.Skip("ssh-utils not available as installer")
}
if code != 200 {
t.Skipf("status %d (selfhosted may not render without cache): %s", code, body[:min(len(body), 200)])
}
t.Logf("selfhosted installer size: %d bytes", len(body))
}
// TestBootstrapUnknownPackage verifies 404 for unknown packages.
func TestBootstrapUnknownPackage(t *testing.T) {
_, ts := newTestServer(t)
code, _ := get(t, ts, "/nonexistent-package-xyz")
if code != 404 {
t.Errorf("expected 404, got %d", code)
}
}
// getWithUA fetches a URL with a custom User-Agent header.
func getWithUA(t *testing.T, ts *httptest.Server, path, ua string) (int, string) {
t.Helper()
req, err := http.NewRequest("GET", ts.URL+path, nil)
if err != nil {
t.Fatalf("new request: %v", err)
}
req.Header.Set("User-Agent", ua)
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("GET %s: %v", path, err)
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
return resp.StatusCode, string(body)
}
+989
View File
@@ -0,0 +1,989 @@
// Command webid is the webi HTTP API server. It reads cached release
// data from the filesystem and serves release metadata, installer
// scripts, and bootstrap dispatches.
//
// It never fetches from upstream APIs — that's webicached's job.
// This server is stateless and fast: load from cache, resolve, render.
//
// Usage:
//
// go run ./cmd/webid
// go run ./cmd/webid -addr :3001 -cache ~/.cache/webi/legacy
package main
import (
"context"
"crypto/sha1"
"encoding/json"
"flag"
"fmt"
"io"
"log"
"net/http"
"os"
"os/signal"
"path/filepath"
"slices"
"strings"
"sync"
"time"
"github.com/webinstall/webi-installers/internal/buildmeta"
"github.com/webinstall/webi-installers/internal/lexver"
"github.com/webinstall/webi-installers/internal/render"
"github.com/webinstall/webi-installers/internal/resolve"
"github.com/webinstall/webi-installers/internal/resolver"
middleware "github.com/therootcompany/golib/http/middleware/v2"
"github.com/webinstall/webi-installers/internal/storage"
"github.com/webinstall/webi-installers/internal/storage/fsstore"
"github.com/webinstall/webi-installers/internal/uadetect"
)
var (
name = "webid"
version = "0.0.0-dev"
commit = "0000000"
date = "0001-01-01"
licenseYear = "2024"
licenseOwner = "AJ ONeal"
licenseType = "MPL-2.0"
)
func printVersion(w io.Writer) {
v := strings.TrimPrefix(version, "v")
_, _ = fmt.Fprintf(w, "%s v%s %s (%s)\n", name, v, commit[:7], date)
_, _ = fmt.Fprintf(w, "Copyright (C) %s %s\n", licenseYear, licenseOwner)
_, _ = fmt.Fprintf(w, "Licensed under %s\n", licenseType)
}
func main() {
addr := flag.String("addr", ":3001", "listen address")
cacheDir := flag.String("legacy", "~/.cache/webi/legacy", "legacy cache directory")
installersDir := flag.String("installers", ".", "installers repo root (for install.sh/ps1)")
if len(os.Args) > 1 {
switch os.Args[1] {
case "-V", "-version", "--version", "version":
printVersion(os.Stdout)
os.Exit(0)
case "help", "-help", "--help":
printVersion(os.Stdout)
fmt.Fprintln(os.Stdout, "")
flag.CommandLine.SetOutput(os.Stdout)
flag.Usage()
os.Exit(0)
}
}
flag.Parse()
cachePath := expandHome(*cacheDir)
fss, err := fsstore.New(cachePath)
if err != nil {
log.Fatalf("fsstore: %v", err)
}
var store storage.Store = fss
srv := &server{
store: store,
installersDir: *installersDir,
packages: make(map[string]*packageCache),
}
// Pre-load all cached packages.
srv.loadAll()
mux := http.NewServeMux()
mmux := middleware.WithMux(mux, requestLogger)
// Legacy API routes (Node.js compat).
mmux.HandleFunc("GET /api/releases/{rest...}", srv.handleReleasesAPI)
// New API routes (v1).
mmux.HandleFunc("GET /v1/releases/{rest...}", srv.handleV1Releases)
mmux.HandleFunc("GET /v1/resolve/{rest...}", srv.handleV1Resolve)
// Full installer script (package-install.tpl.sh + install.sh).
mmux.HandleFunc("GET /api/installers/{rest...}", srv.handleInstaller)
// Debug endpoint.
mmux.HandleFunc("GET /api/debug", srv.handleDebug)
// Health check (no logging — too noisy).
mux.HandleFunc("GET /api/health", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
fmt.Fprintln(w, "ok")
})
// Bootstrap route: /{package} and /{package}@{version}
// Detects UA and returns rendered installer script.
mmux.HandleFunc("GET /{pkgSpec}", srv.handleBootstrap)
httpSrv := &http.Server{
Addr: *addr,
Handler: mux,
ReadTimeout: 5 * time.Second,
WriteTimeout: 30 * time.Second,
IdleTimeout: 120 * time.Second,
}
// Graceful shutdown.
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
defer stop()
go func() {
log.Printf("webid listening on %s", *addr)
if err := httpSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Fatalf("listen: %v", err)
}
}()
<-ctx.Done()
log.Println("shutting down...")
shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
httpSrv.Shutdown(shutCtx)
}
// requestLogger is a middleware that logs each request with status and duration.
func requestLogger(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
rw := &statusWriter{ResponseWriter: w, code: http.StatusOK}
next.ServeHTTP(rw, r)
log.Printf("%s %s %d %s", r.Method, r.URL.Path, rw.code, time.Since(start))
})
}
// statusWriter wraps ResponseWriter to capture the HTTP status code.
type statusWriter struct {
http.ResponseWriter
code int
}
func (sw *statusWriter) WriteHeader(code int) {
sw.code = code
sw.ResponseWriter.WriteHeader(code)
}
// server holds the shared state for all HTTP handlers.
type server struct {
store storage.Store
installersDir string
mu sync.RWMutex
packages map[string]*packageCache
webiCksum string // cached sha1[:8] of webi.sh
}
// packageCache holds a loaded package's assets and catalog.
type packageCache struct {
assets []storage.Asset
dists []resolve.Dist
catalog resolve.Catalog
}
// loadAll pre-loads all packages from the store.
func (s *server) loadAll() {
ctx := context.Background()
pkgs, err := s.store.ListPackages(ctx)
if err != nil {
log.Printf("warn: list packages: %v", err)
return
}
count := 0
for _, pkg := range pkgs {
pd, err := s.store.Load(ctx, pkg)
if err != nil {
log.Printf("warn: load %s: %v", pkg, err)
continue
}
if pd == nil || len(pd.Assets) == 0 {
continue
}
pc := &packageCache{
assets: pd.Assets,
dists: assetsToDists(pd.Assets),
}
pc.catalog = resolve.Survey(pc.dists)
s.mu.Lock()
s.packages[pkg] = pc
s.mu.Unlock()
count++
}
log.Printf("loaded %d packages from store", count)
}
// getPackage returns the cached package data, or nil if not found.
func (s *server) getPackage(pkg string) *packageCache {
s.mu.RLock()
defer s.mu.RUnlock()
return s.packages[pkg]
}
// assetsToDists converts storage.Asset slice to resolve.Dist slice.
func assetsToDists(assets []storage.Asset) []resolve.Dist {
dists := make([]resolve.Dist, len(assets))
for i, a := range assets {
dists[i] = resolve.Dist{
Filename: a.Filename,
Version: a.Version,
LTS: a.LTS,
Channel: a.Channel,
Date: a.Date,
OS: a.OS,
Arch: a.Arch,
Libc: a.Libc,
Format: a.Format,
Download: a.Download,
Extra: a.Extra,
GitTag: a.GitTag,
GitCommitHash: a.GitCommitHash,
Variants: a.Variants,
}
}
return dists
}
// handleReleasesAPI serves /api/releases/{package}@{version}.{format}
func (s *server) handleReleasesAPI(w http.ResponseWriter, r *http.Request) {
rest := r.PathValue("rest")
// Parse: {package}@{version}.{json|tab} or {package}.{json|tab}
pkg, version, format, err := parseReleasePath(rest)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
pc := s.getPackage(pkg)
if pc == nil {
// Check if it's a selfhosted package.
if s.isSelfHosted(pkg) {
s.serveEmptyReleases(w, format)
return
}
http.Error(w, fmt.Sprintf("package %q not found", pkg), http.StatusNotFound)
return
}
// Parse query parameters.
q := r.URL.Query()
osStr := q.Get("os")
archStr := q.Get("arch")
libcStr := q.Get("libc")
ltsStr := q.Get("lts")
channelStr := q.Get("channel")
formatsStr := q.Get("formats")
limitStr := q.Get("limit")
// Normalize wildcard "-" to empty (means "any").
if osStr == "-" {
osStr = ""
}
if archStr == "-" {
archStr = ""
}
if libcStr == "-" {
libcStr = ""
}
// Map Node.js OS/arch names to our canonical names.
osStr = normalizeQueryOS(osStr)
archStr = normalizeQueryArch(archStr)
// Parse LTS.
lts := ltsStr == "true" || ltsStr == "1"
// Handle channel selectors in the version field: @stable, @lts, @beta, etc.
switch strings.ToLower(version) {
case "stable", "latest":
version = ""
if channelStr == "" {
channelStr = "stable"
}
case "lts":
version = ""
lts = true
case "beta", "pre", "preview":
version = ""
if channelStr == "" {
channelStr = "beta"
}
case "rc":
version = ""
if channelStr == "" {
channelStr = "rc"
}
case "alpha", "dev":
version = ""
if channelStr == "" {
channelStr = "alpha"
}
}
// Parse formats list.
var formats []string
if formatsStr != "" {
formats = strings.Split(formatsStr, ",")
}
// Parse limit.
limit := 100
if limitStr != "" {
fmt.Sscanf(limitStr, "%d", &limit)
}
// Filter matching releases, sort by specificity, then apply limit.
filtered := filterDists(pc.dists, osStr, archStr, libcStr, channelStr, version, formats, lts)
sortDistsDescending(filtered, osStr, archStr)
if len(filtered) > limit {
filtered = filtered[:limit]
}
switch format {
case "json":
s.serveJSON(w, r, pc, filtered)
case "tab":
s.serveTab(w, r, filtered)
default:
http.Error(w, "unsupported format: "+format, http.StatusBadRequest)
}
}
// normalizeQueryOS maps Node.js OS names to our canonical names.
func normalizeQueryOS(s string) string {
switch strings.ToLower(s) {
case "macos", "mac":
return "darwin"
case "win":
return "windows"
default:
return s
}
}
// normalizeQueryArch maps Node.js arch names to our canonical names.
func normalizeQueryArch(s string) string {
switch strings.ToLower(s) {
case "amd64":
return string(buildmeta.ArchAMD64) // "x86_64"
case "arm64":
return string(buildmeta.ArchARM64) // "aarch64"
case "armv7l":
return string(buildmeta.ArchARMv7)
case "armv6l":
return string(buildmeta.ArchARMv6)
case "x86", "i386", "i686":
return string(buildmeta.ArchX86)
default:
return s
}
}
// parseReleasePath parses "{pkg}@{version}.{format}" or "{pkg}.{format}".
func parseReleasePath(rest string) (pkg, version, format string, err error) {
if strings.HasSuffix(rest, ".json") {
format = "json"
rest = strings.TrimSuffix(rest, ".json")
} else if strings.HasSuffix(rest, ".tab") {
format = "tab"
rest = strings.TrimSuffix(rest, ".tab")
} else {
return "", "", "", fmt.Errorf("unsupported format (use .json or .tab)")
}
if idx := strings.IndexByte(rest, '@'); idx >= 0 {
pkg = rest[:idx]
version = rest[idx+1:]
} else {
pkg = rest
}
if pkg == "" {
return "", "", "", fmt.Errorf("package name required")
}
return pkg, version, format, nil
}
// filterDists filters dists by query parameters, returning all matches
// up to limit. This is for the API listing, not single-best resolution.
func filterDists(dists []resolve.Dist, osStr, archStr, libcStr, channel, version string, formats []string, lts bool) []resolve.Dist {
var result []resolve.Dist
archSet := make(map[string]bool)
if archStr != "" {
for _, a := range buildmeta.CompatArches(buildmeta.OS(osStr), buildmeta.Arch(archStr)) {
archSet[string(a)] = true
}
if len(archSet) == 0 {
archSet[archStr] = true
}
}
for _, d := range dists {
if osStr != "" && d.OS != osStr && d.OS != "*" && d.OS != "ANYOS" && d.OS != "" &&
!(d.OS == "posix_2017" && osStr != "windows") {
continue
}
if archStr != "" && !archSet[d.Arch] && d.Arch != "*" && d.Arch != "ANYARCH" && d.Arch != "" {
continue
}
if libcStr != "" && d.Libc != "none" && d.Libc != "" && d.Libc != libcStr {
continue
}
if lts && !d.LTS {
continue
}
if channel != "" && d.Channel != channel {
continue
}
if version != "" {
// Match with or without "v" prefix:
// query "0.25" should match version "v0.25.0".
v := strings.TrimPrefix(d.Version, "v")
vq := strings.TrimPrefix(version, "v")
if !strings.HasPrefix(v, vq) {
continue
}
}
if len(formats) > 0 {
matched := false
for _, f := range formats {
if strings.Contains(d.Format, f) {
matched = true
break
}
}
if !matched {
continue
}
}
result = append(result, d)
}
return result
}
// legacyRelease matches the Node.js JSON response format.
// Production returns a bare JSON array of these objects.
type legacyRelease struct {
Name string `json:"name"`
Version string `json:"version"`
GitTag string `json:"git_tag,omitempty"`
GitCommitHash string `json:"git_commit_hash,omitempty"`
LTS bool `json:"lts"`
Channel string `json:"channel"`
Date string `json:"date"`
OS string `json:"os"`
Arch string `json:"arch"`
Ext string `json:"ext"`
Download string `json:"download"`
Libc string `json:"libc"`
}
// legacyOS maps Go canonical OS names to Node.js legacy names.
func legacyOS(s string) string {
switch s {
case "darwin":
return "macos"
case "":
return "*"
default:
return s
}
}
// legacyArch maps Go canonical arch names to Node.js legacy names.
func legacyArch(s string) string {
switch s {
case "x86_64":
return "amd64"
case "aarch64":
return "arm64"
case "armv7":
return "armv7l"
case "armv6":
return "armv6l"
case "armv5":
return "arm"
case "":
return "*"
default:
return s
}
}
// legacyExt strips the leading "." from format strings.
func legacyExt(s string) string {
s = strings.TrimPrefix(s, ".")
if s == "" {
return "exe"
}
return s
}
// legacyVersion strips the leading "v" from version strings.
func legacyVersion(s string) string {
return strings.TrimPrefix(s, "v")
}
// legacyLibc returns "none" for empty libc values.
func legacyLibc(s string) string {
if s == "" {
return "none"
}
return s
}
func distsToLegacy(dists []resolve.Dist) []legacyRelease {
releases := make([]legacyRelease, len(dists))
for i, d := range dists {
releases[i] = legacyRelease{
Name: d.Filename,
Version: legacyVersion(d.Version),
GitTag: d.GitTag,
GitCommitHash: d.GitCommitHash,
LTS: d.LTS,
Channel: d.Channel,
Date: d.Date,
OS: legacyOS(d.OS),
Arch: legacyArch(d.Arch),
Ext: legacyExt(d.Format),
Download: d.Download,
Libc: legacyLibc(d.Libc),
}
}
return releases
}
func (s *server) serveJSON(w http.ResponseWriter, r *http.Request, pc *packageCache, filtered []resolve.Dist) {
// Production returns a bare JSON array, not wrapped in an object.
releases := distsToLegacy(filtered)
w.Header().Set("Content-Type", "application/json")
pretty := r.URL.Query().Get("pretty")
if pretty == "true" || pretty == "1" {
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
enc.Encode(releases)
} else {
json.NewEncoder(w).Encode(releases)
}
}
func (s *server) serveTab(w http.ResponseWriter, r *http.Request, filtered []resolve.Dist) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
// Production only shows header row with ?pretty=true.
pretty := r.URL.Query().Get("pretty")
if pretty != "" && pretty != "false" {
fmt.Fprintln(w, "VERSION\tLTS\tCHANNEL\tRELEASE_DATE\tOS\tARCH\tEXT\tHASH\tURL\t_\tLIBC")
}
// Tab format matches Node.js production:
// version \t lts \t channel \t date \t os \t arch \t ext \t hash \t download \t comment \t libc
for _, d := range filtered {
lts := "-"
if d.LTS {
lts = "lts"
}
channel := d.Channel
if channel == "" {
channel = "-"
}
date := d.Date
if date == "" {
date = "-"
}
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t-\t%s\t\t%s\n",
legacyVersion(d.Version),
lts,
channel,
date,
legacyOS(d.OS),
legacyArch(d.Arch),
legacyExt(d.Format),
d.Download,
legacyLibc(d.Libc),
)
}
}
// sortDistsDescending sorts dists newest-first by version.
func sortDistsDescending(dists []resolve.Dist, queryOS, queryArch string) {
slices.SortStableFunc(dists, func(a, b resolve.Dist) int {
va := lexver.Parse(strings.TrimPrefix(a.Version, "v"))
vb := lexver.Parse(strings.TrimPrefix(b.Version, "v"))
if cmp := lexver.Compare(vb, va); cmp != 0 {
return cmp
}
if cmp := osSpecificity(a.OS, queryOS) - osSpecificity(b.OS, queryOS); cmp != 0 {
return cmp
}
if cmp := archSpecificity(a.Arch, queryArch) - archSpecificity(b.Arch, queryArch); cmp != 0 {
return cmp
}
return libcRank(a.Libc) - libcRank(b.Libc)
})
}
func osSpecificity(distOS, queryOS string) int {
switch {
case distOS == queryOS:
return 0
case distOS == "posix_2017":
return 1
default:
return 2
}
}
func archSpecificity(distArch, queryArch string) int {
switch {
case distArch == queryArch:
return 0
case distArch == "" || distArch == "*":
return 2
default:
return 1
}
}
func libcRank(libc string) int {
switch libc {
case "none", "":
return 0
default:
return 1
}
}
// serveEmptyReleases returns an empty release list for selfhosted packages.
func (s *server) serveEmptyReleases(w http.ResponseWriter, format string) {
switch format {
case "json":
w.Header().Set("Content-Type", "application/json")
// Production returns an empty array.
json.NewEncoder(w).Encode([]legacyRelease{})
case "tab":
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
}
}
// isSelfHosted checks if a package has install.sh but no releases.conf.
func (s *server) isSelfHosted(pkg string) bool {
installPath := filepath.Join(s.installersDir, pkg, "install.sh")
if _, err := os.Stat(installPath); err != nil {
return false
}
confPath := filepath.Join(s.installersDir, pkg, "releases.conf")
if _, err := os.Stat(confPath); err == nil {
return false
}
return true
}
// handleDebug returns UA detection info for the requesting client.
func (s *server) handleDebug(w http.ResponseWriter, r *http.Request) {
result := uadetect.FromRequest(r)
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]string{
"user_agent": r.Header.Get("User-Agent"),
"os": string(result.OS),
"arch": string(result.Arch),
"libc": string(result.Libc),
})
}
// handleBootstrap serves /{package} and /{package}@{version}.
// This is the curl-pipe bootstrap: a minimal script that sets
// WEBI_PKG/WEBI_HOST/WEBI_CHECKSUM and downloads+runs webi.
func (s *server) handleBootstrap(w http.ResponseWriter, r *http.Request) {
pkgSpec := r.PathValue("pkgSpec")
// Parse package@version.
pkg, tag := pkgSpec, ""
if idx := strings.IndexByte(pkgSpec, '@'); idx >= 0 {
pkg = pkgSpec[:idx]
tag = pkgSpec[idx+1:]
}
if pkg == "" {
http.Error(w, "package name required", http.StatusBadRequest)
return
}
// Verify package exists.
if s.getPackage(pkg) == nil && !s.isSelfHosted(pkg) {
http.Error(w, fmt.Sprintf("package %q not found", pkg), http.StatusNotFound)
return
}
baseURL := baseURLFromRequest(r)
webiPkg := pkg
if tag != "" {
webiPkg = pkg + "@" + tag
}
// Read and inject the curl-pipe bootstrap template.
tplPath := filepath.Join(s.installersDir, "_webi", "curl-pipe-bootstrap.tpl.sh")
tpl, err := os.ReadFile(tplPath)
if err != nil {
log.Printf("bootstrap: read template: %v", err)
http.Error(w, "bootstrap template not found", http.StatusInternalServerError)
return
}
script := string(tpl)
script = render.InjectVar(script, "WEBI_PKG", webiPkg)
script = render.InjectVar(script, "WEBI_HOST", baseURL)
script = render.InjectVar(script, "WEBI_CHECKSUM", s.webiChecksum())
// text/html so browsers see the meta redirect to cheat sheet.
w.Header().Set("Content-Type", "text/html; charset=utf-8")
fmt.Fprint(w, script)
}
// handleInstaller serves /api/installers/{pkg}@{version}.sh
// This is the full installer script with release resolution and
// embedded install.sh.
func (s *server) handleInstaller(w http.ResponseWriter, r *http.Request) {
rest := r.PathValue("rest")
// Parse: {pkg}@{version}.sh or {pkg}.sh
ext := ""
if strings.HasSuffix(rest, ".sh") {
ext = "sh"
rest = strings.TrimSuffix(rest, ".sh")
} else if strings.HasSuffix(rest, ".ps1") {
ext = "ps1"
rest = strings.TrimSuffix(rest, ".ps1")
} else {
http.Error(w, "unsupported format (use .sh or .ps1)", http.StatusBadRequest)
return
}
pkg, tag := rest, ""
if idx := strings.IndexByte(rest, '@'); idx >= 0 {
pkg = rest[:idx]
tag = rest[idx+1:]
}
if pkg == "" {
http.Error(w, "package name required", http.StatusBadRequest)
return
}
// Detect platform from User-Agent.
ua := uadetect.FromRequest(r)
if ua.OS == "" {
http.Error(w, "could not detect OS from User-Agent", http.StatusBadRequest)
return
}
isSelfHosted := s.isSelfHosted(pkg)
pc := s.getPackage(pkg)
if pc == nil && !isSelfHosted {
http.Error(w, fmt.Sprintf("package %q not found", pkg), http.StatusNotFound)
return
}
baseURL := baseURLFromRequest(r)
p := render.Params{
Host: baseURL,
PkgName: pkg,
Tag: tag,
OS: string(ua.OS),
Arch: string(ua.Arch),
Libc: string(ua.Libc),
}
// Resolve the best release (if not selfhosted).
if pc != nil {
req := resolver.Request{
OS: string(ua.OS),
Arch: string(ua.Arch),
Libc: string(ua.Libc),
}
switch strings.ToLower(tag) {
case "stable", "latest", "":
// Default.
case "lts":
req.LTS = true
case "beta", "pre", "preview":
req.Channel = "beta"
case "rc":
req.Channel = "rc"
case "alpha", "dev":
req.Channel = "alpha"
default:
req.Version = tag
}
res, err := resolver.Resolve(pc.assets, req)
if err != nil {
p.Version = "0.0.0"
p.Channel = "error"
p.Ext = "err"
p.PkgURL = "https://example.com/doesntexist.ext"
p.PkgFile = "doesntexist.ext"
p.CSV = buildCSV(p)
} else {
v := strings.TrimPrefix(res.Version, "v")
parts := splitVersion(v)
p.Version = v
p.Major = parts[0]
p.Minor = parts[1]
p.Patch = parts[2]
p.Build = parts[3]
if res.Asset.GitTag != "" {
p.GitTag = res.Asset.GitTag
} else {
p.GitTag = "v" + v
}
p.GitBranch = p.GitTag
p.GitCommitHash = res.Asset.GitCommitHash
p.LTS = fmt.Sprintf("%v", res.Asset.LTS)
p.Channel = res.Asset.Channel
if p.Channel == "" {
p.Channel = "stable"
}
p.Ext = strings.TrimPrefix(res.Asset.Format, ".")
if p.Ext == "" {
p.Ext = "exe"
}
p.PkgURL = res.Asset.Download
p.PkgFile = res.Asset.Filename
p.CSV = buildCSV(p)
}
p.PkgStable = pc.catalog.Stable
p.PkgLatest = pc.catalog.Latest
p.PkgOSes = strings.Join(pc.catalog.OSes, " ")
p.PkgArches = strings.Join(pc.catalog.Arches, " ")
p.PkgLibcs = strings.Join(pc.catalog.Libcs, " ")
p.PkgFormats = strings.Join(pc.catalog.Formats, " ")
}
p.ReleasesURL = fmt.Sprintf("%s/api/releases/%s@%s.tab?os=%s&arch=%s&libc=%s&formats=tar&pretty=true",
baseURL, pkg, tag, p.OS, p.Arch, p.Libc)
var script string
var renderErr error
if ext == "ps1" {
tplPath := filepath.Join(s.installersDir, "_webi", "package-install.tpl.ps1")
script, renderErr = render.PowerShell(tplPath, s.installersDir, pkg, p)
} else {
tplPath := filepath.Join(s.installersDir, "_webi", "package-install.tpl.sh")
script, renderErr = render.Bash(tplPath, s.installersDir, pkg, p)
}
if renderErr != nil {
log.Printf("render %s: %v", pkg, renderErr)
http.Error(w, fmt.Sprintf("failed to render installer for %q: %v", pkg, renderErr), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
fmt.Fprint(w, script)
}
// baseURLFromRequest builds the base URL from the request.
func baseURLFromRequest(r *http.Request) string {
if r.TLS != nil || strings.Contains(r.Host, "webinstall") || strings.Contains(r.Host, "webi.") {
return "https://" + r.Host
}
return "http://" + r.Host
}
// webiChecksum returns the checksum of the webi.sh bootstrap script.
func (s *server) webiChecksum() string {
s.mu.RLock()
cksum := s.webiCksum
s.mu.RUnlock()
if cksum != "" {
return cksum
}
// Calculate checksum from webi.sh file.
webiPath := filepath.Join(s.installersDir, "webi", "webi.sh")
data, err := os.ReadFile(webiPath)
if err != nil {
return "00000000"
}
h := sha1.New()
h.Write(data)
cksum = fmt.Sprintf("%x", h.Sum(nil))[:8]
s.mu.Lock()
s.webiCksum = cksum
s.mu.Unlock()
return cksum
}
// buildCSV creates the WEBI_CSV line in the Node.js format.
func buildCSV(p render.Params) string {
return strings.Join([]string{
p.Version,
p.LTS,
p.Channel,
"", // date
p.OS,
p.Arch,
p.Ext,
"-",
p.PkgURL,
p.PkgFile,
"",
}, ",")
}
// splitVersion splits a version string into [major, minor, patch, build].
func splitVersion(v string) [4]string {
// Strip pre-release suffix for splitting.
base := v
build := ""
if idx := strings.IndexByte(v, '-'); idx >= 0 {
base = v[:idx]
build = v[idx+1:]
}
parts := strings.SplitN(base, ".", 4)
var result [4]string
for i := 0; i < len(parts) && i < 3; i++ {
result[i] = parts[i]
}
result[3] = build
return result
}
func expandHome(path string) string {
if !strings.HasPrefix(path, "~/") {
return path
}
home, err := os.UserHomeDir()
if err != nil {
return path
}
return filepath.Join(home, path[2:])
}
+298
View File
@@ -0,0 +1,298 @@
package main
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/webinstall/webi-installers/internal/resolve"
"github.com/webinstall/webi-installers/internal/storage"
"github.com/webinstall/webi-installers/internal/storage/fsstore"
)
// newTestServer creates a server backed by the real _cache directory
// and returns an httptest.Server with proper routing (so PathValue works).
func newTestServer(t *testing.T) (*server, *httptest.Server) {
t.Helper()
cacheDir := filepath.Join("..", "..", "_cache")
if _, err := os.Stat(cacheDir); err != nil {
t.Skipf("no cache dir at %s", cacheDir)
}
store, err := fsstore.New(cacheDir)
if err != nil {
t.Fatalf("fsstore: %v", err)
}
srv := &server{
store: store,
installersDir: filepath.Join("..", ".."),
packages: make(map[string]*packageCache),
}
// Load packages.
monthDir := time.Now().Format("2006-01")
dir := filepath.Join(store.Root(), monthDir)
entries, err := os.ReadDir(dir)
if err != nil {
t.Fatalf("readdir: %v", err)
}
for _, e := range entries {
if !strings.HasSuffix(e.Name(), ".json") {
continue
}
pkg := strings.TrimSuffix(e.Name(), ".json")
pd, err := store.Load(context.Background(), pkg)
if err != nil || pd == nil || len(pd.Assets) == 0 {
continue
}
pc := &packageCache{
assets: pd.Assets,
dists: assetsToDists(pd.Assets),
}
pc.catalog = resolve.Survey(pc.dists)
srv.packages[pkg] = pc
}
mux := http.NewServeMux()
mux.HandleFunc("GET /api/releases/{rest...}", srv.handleReleasesAPI)
mux.HandleFunc("GET /v1/releases/{rest...}", srv.handleV1Releases)
mux.HandleFunc("GET /v1/resolve/{rest...}", srv.handleV1Resolve)
mux.HandleFunc("GET /api/installers/{rest...}", srv.handleInstaller)
mux.HandleFunc("GET /api/debug", srv.handleDebug)
mux.HandleFunc("GET /{pkgSpec}", srv.handleBootstrap)
ts := httptest.NewServer(mux)
t.Cleanup(ts.Close)
return srv, ts
}
// get fetches a URL from the test server and returns the body.
func get(t *testing.T, ts *httptest.Server, path string) (int, string) {
t.Helper()
resp, err := http.Get(ts.URL + path)
if err != nil {
t.Fatalf("GET %s: %v", path, err)
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
return resp.StatusCode, string(body)
}
// TestLegacyJSONFormat verifies our JSON output matches the production format.
func TestLegacyJSONFormat(t *testing.T) {
srv, ts := newTestServer(t)
packages := []string{"bat", "node", "go", "jq"}
for _, pkg := range packages {
t.Run(pkg, func(t *testing.T) {
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/api/releases/"+pkg+".json?limit=5")
if code != http.StatusOK {
t.Fatalf("status %d: %s", code, body)
}
body = strings.TrimSpace(body)
// Must be a JSON array, not an object.
if !strings.HasPrefix(body, "[") {
t.Fatalf("expected JSON array, got: %.100s", body)
}
var releases []legacyRelease
if err := json.Unmarshal([]byte(body), &releases); err != nil {
t.Fatalf("decode: %v", err)
}
if len(releases) == 0 {
t.Fatal("no releases returned")
}
// Check field format conventions.
for i, r := range releases {
if strings.HasPrefix(r.Version, "v") {
t.Errorf("release[%d]: version %q should not have v prefix", i, r.Version)
}
if strings.HasPrefix(r.Ext, ".") {
t.Errorf("release[%d]: ext %q should not have . prefix", i, r.Ext)
}
if r.OS == "darwin" {
t.Errorf("release[%d]: os should be 'macos' not 'darwin'", i)
}
if r.Arch == "x86_64" {
t.Errorf("release[%d]: arch should be 'amd64' not 'x86_64'", i)
}
if r.Arch == "aarch64" {
t.Errorf("release[%d]: arch should be 'arm64' not 'aarch64'", i)
}
if r.Libc == "" {
t.Errorf("release[%d]: libc should be 'none' not empty", i)
}
if r.Download == "" {
t.Errorf("release[%d]: download URL is empty", i)
}
}
})
}
}
// TestLegacyTabFormat verifies our .tab output uses real TSV.
func TestLegacyTabFormat(t *testing.T) {
srv, ts := newTestServer(t)
packages := []string{"bat", "node", "go"}
for _, pkg := range packages {
t.Run(pkg, func(t *testing.T) {
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/api/releases/"+pkg+".tab?limit=3")
if code != http.StatusOK {
t.Fatalf("status %d: %s", code, body)
}
lines := strings.Split(strings.TrimSpace(body), "\n")
if len(lines) == 0 {
t.Fatal("no lines returned")
}
for i, line := range lines {
fields := strings.Split(line, "\t")
// Expect 11 tab-separated fields:
// version, lts, channel, date, os, arch, ext, hash, download, (empty), libc
if len(fields) != 11 {
t.Errorf("line[%d]: expected 11 tab fields, got %d: %q", i, len(fields), line)
continue
}
version := fields[0]
lts := fields[1]
ext := fields[6]
if strings.HasPrefix(version, "v") {
t.Errorf("line[%d]: version %q should not have v prefix", i, version)
}
if lts != "-" && lts != "lts" {
t.Errorf("line[%d]: lts should be '-' or 'lts', got %q", i, lts)
}
if strings.HasPrefix(ext, ".") {
t.Errorf("line[%d]: ext %q should not have . prefix", i, ext)
}
}
})
}
}
// TestLegacyJSONAgainstProduction compares our output against live production.
// Run with: WEBI_TEST_PROD=1 go test -run TestLegacyJSONAgainstProduction
func TestLegacyJSONAgainstProduction(t *testing.T) {
if os.Getenv("WEBI_TEST_PROD") == "" {
t.Skip("set WEBI_TEST_PROD=1 to compare against production")
}
srv, ts := newTestServer(t)
packages := []string{"bat", "node", "go", "jq", "rg"}
for _, pkg := range packages {
t.Run(pkg, func(t *testing.T) {
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
// Fetch from production.
prodURL := fmt.Sprintf("https://webinstall.dev/api/releases/%s.json?limit=3", pkg)
prodResp, err := http.Get(prodURL)
if err != nil {
t.Fatalf("fetch production: %v", err)
}
defer prodResp.Body.Close()
prodBody, _ := io.ReadAll(prodResp.Body)
var prodReleases []legacyRelease
if err := json.Unmarshal(prodBody, &prodReleases); err != nil {
t.Fatalf("decode production: %v\nbody: %.500s", err, string(prodBody))
}
// Fetch from local.
_, localBody := get(t, ts, "/api/releases/"+pkg+".json?limit=3")
var localReleases []legacyRelease
if err := json.Unmarshal([]byte(localBody), &localReleases); err != nil {
t.Fatalf("decode local: %v", err)
}
if len(prodReleases) == 0 || len(localReleases) == 0 {
t.Skip("empty releases")
}
// Compare the first release's format.
prod := prodReleases[0]
local := localReleases[0]
if strings.HasPrefix(local.Version, "v") != strings.HasPrefix(prod.Version, "v") {
t.Errorf("version prefix mismatch: prod=%q local=%q", prod.Version, local.Version)
}
if strings.HasPrefix(local.Ext, ".") != strings.HasPrefix(prod.Ext, ".") {
t.Errorf("ext prefix mismatch: prod=%q local=%q", prod.Ext, local.Ext)
}
if prod.OS == "macos" && local.OS == "darwin" {
t.Error("OS: prod uses 'macos', local uses 'darwin'")
}
if prod.Arch == "amd64" && local.Arch == "x86_64" {
t.Error("Arch: prod uses 'amd64', local uses 'x86_64'")
}
if prod.Arch == "arm64" && local.Arch == "aarch64" {
t.Error("Arch: prod uses 'arm64', local uses 'aarch64'")
}
t.Logf("prod[0]: version=%q os=%q arch=%q ext=%q libc=%q",
prod.Version, prod.OS, prod.Arch, prod.Ext, prod.Libc)
t.Logf("local[0]: version=%q os=%q arch=%q ext=%q libc=%q",
local.Version, local.OS, local.Arch, local.Ext, local.Libc)
})
}
}
// TestSortOrder verifies releases come back newest-first.
func TestSortOrder(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
_, body := get(t, ts, "/api/releases/"+pkg+".json?limit=20")
var releases []legacyRelease
if err := json.Unmarshal([]byte(body), &releases); err != nil {
t.Fatalf("decode: %v", err)
}
if len(releases) < 2 {
t.Skip("need at least 2 releases")
}
// First release should be newest (or equal) version.
first := releases[0].Date
last := releases[len(releases)-1].Date
if first < last {
t.Errorf("not newest-first: first=%q last=%q", first, last)
}
}
// Ensure imports are used.
var _ = storage.Asset{}
+459
View File
@@ -0,0 +1,459 @@
package main
import (
"bytes"
"encoding/csv"
"encoding/json"
"fmt"
"net/http"
"slices"
"strings"
"github.com/jszwec/csvutil"
"github.com/webinstall/webi-installers/internal/buildmeta"
"github.com/webinstall/webi-installers/internal/lexver"
"github.com/webinstall/webi-installers/internal/resolver"
"github.com/webinstall/webi-installers/internal/storage"
)
// v1Release is a single release in the new API TSV format.
// Field order matters for csvutil — it determines column order.
// Fields are designed to be easy to consume with cut/grep/sort.
type v1Release struct {
Version string `csv:"version"`
Channel string `csv:"channel"`
LTS string `csv:"lts"`
Date string `csv:"date"`
OS string `csv:"os"`
Arch string `csv:"arch"`
Libc string `csv:"libc"`
Format string `csv:"format"`
Variants string `csv:"variants"` // space-separated
Download string `csv:"download"`
Filename string `csv:"filename"`
}
// v1ResolveResult is the response for /v1/resolve/{pkg}.
type v1ResolveResult struct {
Version string `csv:"version" json:"version"`
Channel string `csv:"channel" json:"channel"`
LTS string `csv:"lts" json:"lts"`
Date string `csv:"date" json:"date"`
OS string `csv:"os" json:"os"`
Arch string `csv:"arch" json:"arch"`
Libc string `csv:"libc" json:"libc"`
Format string `csv:"format" json:"format"`
Variants string `csv:"variants" json:"variants"`
Download string `csv:"download" json:"download"`
Filename string `csv:"filename" json:"filename"`
Triplet string `csv:"triplet" json:"triplet"`
}
// handleV1Releases serves /v1/releases/{pkg}.tsv (or .json)
// with Go-native naming and TSV-first format.
//
// Query params:
//
// os — filter by OS (darwin, linux, windows)
// arch — filter by arch (aarch64, x86_64, armv7l)
// libc — filter by libc (gnu, musl, msvc)
// channel — release channel (stable, beta, rc, alpha)
// version — version prefix filter (e.g. "1.20")
// lts — if "true", only LTS releases
// format — filter by format (e.g. "tar.gz")
// variant — filter by variant (e.g. "rocm")
// limit — max results (default 1000)
func (s *server) handleV1Releases(w http.ResponseWriter, r *http.Request) {
rest := r.PathValue("rest")
pkg, version, format, err := parseReleasePath(rest)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
pc := s.getPackage(pkg)
if pc == nil {
if s.isSelfHosted(pkg) {
s.v1ServeEmpty(w, format)
return
}
http.Error(w, fmt.Sprintf("package %q not found", pkg), http.StatusNotFound)
return
}
q := r.URL.Query()
osStr := q.Get("os")
archStr := q.Get("arch")
libcStr := q.Get("libc")
channelStr := q.Get("channel")
ltsStr := q.Get("lts")
formatFilter := q.Get("format")
variantStr := q.Get("variant")
limitStr := q.Get("limit")
// Use version from URL path or query.
if version == "" {
version = q.Get("version")
}
// Handle channel selectors in version field.
switch strings.ToLower(version) {
case "stable", "latest":
version = ""
if channelStr == "" {
channelStr = "stable"
}
case "lts":
version = ""
ltsStr = "true"
case "beta", "pre", "preview":
version = ""
if channelStr == "" {
channelStr = "beta"
}
case "rc":
version = ""
if channelStr == "" {
channelStr = "rc"
}
case "alpha", "dev":
version = ""
if channelStr == "" {
channelStr = "alpha"
}
}
lts := ltsStr == "true" || ltsStr == "1"
limit := 1000
if limitStr != "" {
fmt.Sscanf(limitStr, "%d", &limit)
}
// Filter assets directly (not via resolve.Dist).
filtered := filterAssets(pc.assets, osStr, archStr, libcStr, channelStr, version, formatFilter, variantStr, lts, limit)
// Sort newest-first.
sortAssetsDescending(filtered)
switch format {
case "json":
s.v1ServeJSON(w, filtered)
case "tab":
s.v1ServeTSV(w, filtered)
default:
http.Error(w, "unsupported format: "+format+" (use .json or .tab)", http.StatusBadRequest)
}
}
// handleV1Resolve serves /v1/resolve/{pkg}.tsv (or .json)
// It resolves the single best asset for a given platform.
//
// Query params:
//
// os — target OS (required)
// arch — target arch (required)
// libc — target libc
// version — version prefix
// channel — release channel
// lts — if "true", only LTS
// format — preferred formats (comma-separated, in preference order)
// variant — preferred variant
func (s *server) handleV1Resolve(w http.ResponseWriter, r *http.Request) {
rest := r.PathValue("rest")
pkg, version, format, err := parseReleasePath(rest)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
pc := s.getPackage(pkg)
if pc == nil {
http.Error(w, fmt.Sprintf("package %q not found", pkg), http.StatusNotFound)
return
}
q := r.URL.Query()
osStr := q.Get("os")
archStr := q.Get("arch")
libcStr := q.Get("libc")
channelStr := q.Get("channel")
ltsStr := q.Get("lts")
formatsStr := q.Get("format")
variantStr := q.Get("variant")
if version == "" {
version = q.Get("version")
}
// Handle channel selectors in version field.
switch strings.ToLower(version) {
case "stable", "latest":
version = ""
if channelStr == "" {
channelStr = "stable"
}
case "lts":
version = ""
ltsStr = "true"
case "beta", "pre", "preview":
version = ""
if channelStr == "" {
channelStr = "beta"
}
case "rc":
version = ""
if channelStr == "" {
channelStr = "rc"
}
case "alpha", "dev":
version = ""
if channelStr == "" {
channelStr = "alpha"
}
}
lts := ltsStr == "true" || ltsStr == "1"
var formats []string
if formatsStr != "" {
formats = strings.Split(formatsStr, ",")
}
req := resolver.Request{
OS: osStr,
Arch: archStr,
Libc: libcStr,
Version: version,
Channel: channelStr,
LTS: lts,
Formats: formats,
Variant: variantStr,
}
res, err := resolver.Resolve(pc.assets, req)
if err != nil {
http.Error(w, fmt.Sprintf("no match for %s: %v", pkg, err), http.StatusNotFound)
return
}
result := assetToV1Resolve(res)
switch format {
case "json":
w.Header().Set("Content-Type", "application/json")
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
enc.Encode(result)
case "tab":
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
data, err := marshalTSV([]v1ResolveResult{result})
if err != nil {
http.Error(w, "encode error: "+err.Error(), http.StatusInternalServerError)
return
}
w.Write(data)
default:
http.Error(w, "unsupported format: "+format, http.StatusBadRequest)
}
}
func assetToV1Release(a storage.Asset) v1Release {
lts := "-"
if a.LTS {
lts = "lts"
}
channel := a.Channel
if channel == "" {
channel = "stable"
}
libc := a.Libc
if libc == "" {
libc = "-"
}
return v1Release{
Version: a.Version,
Channel: channel,
LTS: lts,
Date: a.Date,
OS: a.OS,
Arch: a.Arch,
Libc: libc,
Format: a.Format,
Variants: strings.Join(a.Variants, " "),
Download: a.Download,
Filename: a.Filename,
}
}
func assetToV1Resolve(res resolver.Result) v1ResolveResult {
a := res.Asset
lts := "-"
if a.LTS {
lts = "lts"
}
channel := a.Channel
if channel == "" {
channel = "stable"
}
libc := a.Libc
if libc == "" {
libc = "-"
}
return v1ResolveResult{
Version: a.Version,
Channel: channel,
LTS: lts,
Date: a.Date,
OS: a.OS,
Arch: a.Arch,
Libc: libc,
Format: a.Format,
Variants: strings.Join(a.Variants, " "),
Download: a.Download,
Filename: a.Filename,
Triplet: res.Triplet,
}
}
func (s *server) v1ServeTSV(w http.ResponseWriter, assets []storage.Asset) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
releases := make([]v1Release, len(assets))
for i, a := range assets {
releases[i] = assetToV1Release(a)
}
data, err := marshalTSV(releases)
if err != nil {
http.Error(w, "encode error: "+err.Error(), http.StatusInternalServerError)
return
}
w.Write(data)
}
func (s *server) v1ServeJSON(w http.ResponseWriter, assets []storage.Asset) {
w.Header().Set("Content-Type", "application/json")
releases := make([]v1Release, len(assets))
for i, a := range assets {
releases[i] = assetToV1Release(a)
}
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
enc.Encode(releases)
}
func (s *server) v1ServeEmpty(w http.ResponseWriter, format string) {
switch format {
case "json":
w.Header().Set("Content-Type", "application/json")
w.Write([]byte("[]\n"))
case "tab":
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
// Just the header.
data, _ := marshalTSV([]v1Release{})
w.Write(data)
}
}
// filterAssets filters storage.Asset slices directly.
func filterAssets(assets []storage.Asset, osStr, archStr, libcStr, channel, version, formatFilter, variant string, lts bool, limit int) []storage.Asset {
var result []storage.Asset
for _, a := range assets {
if osStr != "" && a.OS != osStr && a.OS != "ANYOS" && a.OS != "" {
continue
}
if archStr != "" && a.Arch != archStr && a.Arch != "ANYARCH" && a.Arch != "" {
continue
}
if libcStr != "" && a.Libc != "" && a.Libc != "none" && a.Libc != libcStr {
continue
}
if lts && !a.LTS {
continue
}
if channel != "" && a.Channel != channel {
continue
}
if version != "" {
v := strings.TrimPrefix(a.Version, "v")
vq := strings.TrimPrefix(version, "v")
if !strings.HasPrefix(v, vq) {
continue
}
}
if formatFilter != "" && !strings.Contains(a.Format, formatFilter) {
continue
}
if variant != "" {
if !hasVariant(a.Variants, variant) {
continue
}
}
result = append(result, a)
if len(result) >= limit {
break
}
}
return result
}
// sortAssetsDescending sorts assets newest-first by version.
func sortAssetsDescending(assets []storage.Asset) {
slices.SortStableFunc(assets, func(a, b storage.Asset) int {
va := lexver.Parse(strings.TrimPrefix(a.Version, "v"))
vb := lexver.Parse(strings.TrimPrefix(b.Version, "v"))
return lexver.Compare(vb, va) // descending
})
}
// hasVariant checks if the variant list contains the wanted variant.
// This is a copy of resolver.hasVariant since it's unexported.
func hasVariant(variants []string, want string) bool {
for _, v := range variants {
if v == want {
return true
}
}
return false
}
// marshalTSV encodes a slice of structs as tab-separated values with a header.
// Uses csvutil for struct-to-CSV mapping, with csv.Writer set to tab delimiter.
func marshalTSV[T any](records []T) ([]byte, error) {
var buf bytes.Buffer
w := csv.NewWriter(&buf)
w.Comma = '\t'
enc := csvutil.NewEncoder(w)
for _, r := range records {
if err := enc.Encode(r); err != nil {
return nil, err
}
}
w.Flush()
if err := w.Error(); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// normalizeV1Arch maps query arch names to canonical Go names.
func normalizeV1Arch(s string) string {
switch strings.ToLower(s) {
case "amd64":
return string(buildmeta.ArchAMD64) // "x86_64"
case "arm64":
return string(buildmeta.ArchARM64) // "aarch64"
default:
return s
}
}
+273
View File
@@ -0,0 +1,273 @@
package main
import (
"encoding/json"
"strings"
"testing"
)
// TestV1ReleasesTSV verifies the v1 releases endpoint returns proper TSV.
func TestV1ReleasesTSV(t *testing.T) {
srv, ts := newTestServer(t)
packages := []string{"bat", "node", "go"}
for _, pkg := range packages {
t.Run(pkg, func(t *testing.T) {
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/v1/releases/"+pkg+".tab?limit=5")
if code != 200 {
t.Fatalf("status %d: %s", code, body)
}
lines := strings.Split(strings.TrimSpace(body), "\n")
if len(lines) < 2 {
t.Fatal("expected header + data rows")
}
// First line should be header.
header := lines[0]
fields := strings.Split(header, "\t")
expectedHeaders := []string{
"version",
"channel",
"lts",
"date",
"os",
"arch",
"libc",
"format",
"variants",
"download",
"filename",
}
if len(fields) != len(expectedHeaders) {
t.Fatalf("expected %d columns, got %d: %q", len(expectedHeaders), len(fields), header)
}
for i, want := range expectedHeaders {
if fields[i] != want {
t.Errorf("column[%d]: want %q, got %q", i, want, fields[i])
}
}
// Data rows should have same number of fields.
for i, line := range lines[1:] {
dataFields := strings.Split(line, "\t")
if len(dataFields) != len(expectedHeaders) {
t.Errorf("row[%d]: expected %d fields, got %d: %q", i, len(expectedHeaders), len(dataFields), line)
}
}
})
}
}
// TestV1ReleasesJSON verifies the v1 releases JSON format.
func TestV1ReleasesJSON(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/v1/releases/"+pkg+".json?limit=3")
if code != 200 {
t.Fatalf("status %d: %s", code, body)
}
var releases []v1Release
if err := json.Unmarshal([]byte(body), &releases); err != nil {
t.Fatalf("decode: %v", err)
}
if len(releases) == 0 {
t.Fatal("no releases")
}
// v1 API uses Go-native naming — no mapping.
for i, r := range releases {
if r.Version == "" {
t.Errorf("release[%d]: empty version", i)
}
if r.Download == "" {
t.Errorf("release[%d]: empty download", i)
}
if r.Channel == "" {
t.Errorf("release[%d]: empty channel (should be 'stable' or similar)", i)
}
}
}
// TestV1Resolve verifies the v1 resolve endpoint.
func TestV1Resolve(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
tests := []struct {
name string
query string
wantOS string
}{
{
name: "linux amd64",
query: "?os=linux&arch=x86_64",
wantOS: "linux",
},
{
name: "darwin arm64",
query: "?os=darwin&arch=aarch64",
wantOS: "darwin",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
code, body := get(t, ts, "/v1/resolve/"+pkg+".json"+tt.query)
if code != 200 {
t.Fatalf("status %d: %s", code, body)
}
var result v1ResolveResult
if err := json.Unmarshal([]byte(body), &result); err != nil {
t.Fatalf("decode: %v", err)
}
if result.Version == "" {
t.Error("empty version")
}
if result.Download == "" {
t.Error("empty download")
}
if result.OS != tt.wantOS {
t.Errorf("os: want %q, got %q", tt.wantOS, result.OS)
}
if result.Triplet == "" {
t.Error("empty triplet")
}
t.Logf("resolved: %s %s %s %s → %s", result.Version, result.OS, result.Arch, result.Format, result.Download)
})
}
}
// TestV1ResolveTSV verifies the TSV format for resolve.
func TestV1ResolveTSV(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/v1/resolve/"+pkg+".tab?os=linux&arch=x86_64")
if code != 200 {
t.Fatalf("status %d: %s", code, body)
}
lines := strings.Split(strings.TrimSpace(body), "\n")
if len(lines) != 2 {
t.Fatalf("expected 2 lines (header + result), got %d", len(lines))
}
header := strings.Split(lines[0], "\t")
data := strings.Split(lines[1], "\t")
if len(header) != len(data) {
t.Fatalf("header has %d fields, data has %d", len(header), len(data))
}
// Should have a "triplet" column.
hasTriplet := false
for _, h := range header {
if h == "triplet" {
hasTriplet = true
}
}
if !hasTriplet {
t.Error("missing triplet column in header")
}
}
// TestV1ResolveJQ verifies jq resolves to binaries, not git.
func TestV1ResolveJQ(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "jq"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/v1/resolve/"+pkg+".json?os=darwin&arch=aarch64")
if code != 200 {
t.Fatalf("status %d: %s", code, body)
}
var result v1ResolveResult
if err := json.Unmarshal([]byte(body), &result); err != nil {
t.Fatalf("decode: %v", err)
}
if result.Format == "git" {
t.Errorf("resolved to git instead of binary: %+v", result)
}
if result.OS == "" {
t.Errorf("resolved to empty OS (git asset): %+v", result)
}
t.Logf("jq resolved: version=%s os=%s arch=%s format=%s → %s",
result.Version, result.OS, result.Arch, result.Format, result.Download)
}
// TestV1ReleasesFilterOS verifies OS filtering works.
func TestV1ReleasesFilterOS(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/v1/releases/"+pkg+".json?os=darwin&limit=10")
if code != 200 {
t.Fatalf("status %d: %s", code, body)
}
var releases []v1Release
if err := json.Unmarshal([]byte(body), &releases); err != nil {
t.Fatalf("decode: %v", err)
}
for i, r := range releases {
if r.OS != "darwin" && r.OS != "ANYOS" && r.OS != "" {
t.Errorf("release[%d]: os=%q, expected darwin", i, r.OS)
}
}
}
// TestV1NoQuotedFields verifies TSV output has no quoted fields.
func TestV1NoQuotedFields(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/v1/releases/"+pkg+".tab?limit=20")
if code != 200 {
t.Fatalf("status %d: %s", code, body)
}
lines := strings.Split(strings.TrimSpace(body), "\n")
for i, line := range lines {
if strings.Contains(line, "\"") {
t.Errorf("line[%d] contains quotes: %s", i, line)
}
}
}
+5 -2
View File
@@ -42,8 +42,9 @@ type Params struct {
Minor string
Patch string
Build string
GitTag string
GitBranch string
GitTag string
GitBranch string
GitCommitHash string
LTS string // "true" or "false"
Channel string
Ext string // archive extension (e.g. "tar.gz", "zip")
@@ -106,6 +107,7 @@ func Bash(tplPath, installersDir, pkgName string, p Params) (string, error) {
{"WEBI_BUILD", p.Build},
{"WEBI_GIT_BRANCH", p.GitBranch},
{"WEBI_GIT_TAG", p.GitTag},
{"WEBI_GIT_COMMIT_HASH", p.GitCommitHash},
{"WEBI_LTS", p.LTS},
{"WEBI_CHANNEL", p.Channel},
{"WEBI_EXT", p.Ext},
@@ -160,6 +162,7 @@ func PowerShell(tplPath, installersDir, pkgName string, p Params) (string, error
{"WEBI_HOST", p.Host},
{"WEBI_VERSION", p.Version},
{"WEBI_GIT_TAG", p.GitTag},
{"WEBI_GIT_COMMIT_HASH", p.GitCommitHash},
{"WEBI_PKG_URL", p.PkgURL},
{"WEBI_PKG_FILE", p.PkgFile},
{"WEBI_PKG_PATHNAME", p.PkgFile},
+6
View File
@@ -67,6 +67,12 @@ func (la LegacyAsset) ToAsset() Asset {
arch = "x86_64"
case "arm64":
arch = "aarch64"
case "armv7l":
arch = "armv7"
case "armv6l":
arch = "armv6"
case "arm":
arch = "armv5"
case "*":
arch = ""
}
+81
View File
@@ -0,0 +1,81 @@
#!/bin/sh
set -e
set -u
# Build and deploy webid to a target host
g_host="${1:-next.webi.sh}"
g_bin="webid"
g_out="agents/tmp/${g_bin}"
g_remote_bin="~/bin/${g_bin}"
case "${g_host}" in
beta.webi.sh) g_remote_conf="~/srv/beta.webinstall.dev/installers/" ;;
next.webi.sh) g_remote_conf="~/srv/next.webinstall.dev/installers/" ;;
*) g_remote_conf="~/srv/webid/installers/" ;;
esac
fn_build() {
b_version="$(git describe --tags --always 2> /dev/null || echo '0.0.0-dev')"
b_commit="$(git rev-parse --short HEAD)"
b_date="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
b_ldflags="-X main.version=${b_version} -X main.commit=${b_commit} -X main.date=${b_date}"
printf 'Building %s %s %s (%s)...\n' "${g_bin}" "${b_version}" "${b_commit}" "${b_date}"
GOOS=linux GOARCH=amd64 GOAMD64=v2 go build -ldflags "${b_ldflags}" -o "${g_out}" ./cmd/webid
printf 'Built: %s\n' "${g_out}"
}
fn_deploy() {
printf 'Stopping %s on %s...\n' "${g_bin}" "${g_host}"
ssh "${g_host}" "~/.local/bin/serviceman stop ${g_bin}" 2> /dev/null || true
printf 'Uploading binary...\n'
scp "${g_out}" "${g_host}:${g_remote_bin}"
printf 'Syncing install scripts and templates...\n'
rsync -av \
--exclude='_cache' --exclude='.git' --exclude='agents' \
--exclude='bin' --exclude='cmd' --exclude='internal' \
--exclude='docs' --exclude='scripts' --exclude='node_modules' \
--include='*/' --include='install.sh' --include='install.ps1' \
--include='_webi/*.tpl.sh' --include='_webi/*.tpl.ps1' \
--exclude='*' \
./ "${g_host}:${g_remote_conf}"
}
fn_start() {
printf 'Starting %s...\n' "${g_bin}"
ssh "${g_host}" "~/.local/bin/serviceman start ${g_bin}" || {
printf 'Service not configured. Run serviceman add on the host:\n'
printf ' serviceman add --name %s \\\n' "${g_bin}"
printf ' --workdir %s -- \\\n' "${g_remote_conf}"
printf ' %s \\\n' "${g_remote_bin}"
printf ' --addr :3082 \\\n'
printf ' --legacy ~/.cache/webi/legacy \\\n'
printf ' --installers %s\n' "${g_remote_conf}"
exit 1
}
}
fn_verify() {
printf 'Waiting 3s for startup...\n'
sleep 3
printf 'Checking version...\n'
ssh "${g_host}" "${g_remote_bin} -V"
printf 'Checking health...\n'
ssh "${g_host}" "curl -s http://localhost:3082/api/releases/bat.json | head -c 100"
printf '\n'
printf 'Checking logs...\n'
ssh "${g_host}" "sudo journalctl -u ${g_bin} --no-pager -n 5"
}
fn_build
fn_deploy
fn_start
fn_verify
printf '\nDone. %s deployed to %s.\n' "${g_bin}" "${g_host}"