From ac59e4728e241f0853694fba197add7b4f79f96b Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Sat, 16 May 2026 21:35:55 -0600 Subject: [PATCH] feat(webid): add HTTP API server (releases, resolve, installer scripts) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Serves the HTTP API for webinstall.dev: - GET /api/releases/{pkg}.json — classified release list from fsstore - GET /v1/resolve/{pkg} — resolve OS/arch/version to a specific asset - GET /api/installers/{pkg}.sh — render installer shell script - GET /api/installers/{pkg}.ps1 — render installer PowerShell script Git-clone packages include git_tag and git_commit_hash in responses. UA detection infers OS/arch from User-Agent when not query-specified. --- _webi/package-install.tpl.ps1 | 1 + _webi/package-install.tpl.sh | 1 + cmd/webid/bootstrap_test.go | 146 +++++ cmd/webid/main.go | 989 ++++++++++++++++++++++++++++++++++ cmd/webid/main_test.go | 298 ++++++++++ cmd/webid/v1api.go | 459 ++++++++++++++++ cmd/webid/v1api_test.go | 273 ++++++++++ internal/render/render.go | 7 +- internal/storage/legacy.go | 6 + scripts/deploy-webid.sh | 81 +++ 10 files changed, 2259 insertions(+), 2 deletions(-) create mode 100644 cmd/webid/bootstrap_test.go create mode 100644 cmd/webid/main.go create mode 100644 cmd/webid/main_test.go create mode 100644 cmd/webid/v1api.go create mode 100644 cmd/webid/v1api_test.go create mode 100755 scripts/deploy-webid.sh diff --git a/_webi/package-install.tpl.ps1 b/_webi/package-install.tpl.ps1 index 813745e..9647dde 100644 --- a/_webi/package-install.tpl.ps1 +++ b/_webi/package-install.tpl.ps1 @@ -17,6 +17,7 @@ $Env:WEBI_HOST = 'https://webinstall.dev' #$Env:PKG_NAME = node #$Env:WEBI_VERSION = v12.16.2 #$Env:WEBI_GIT_TAG = 12.16.2 +#$Env:WEBI_GIT_COMMIT_HASH = #$Env:WEBI_PKG_URL = "https://.../node-....zip" #$Env:WEBI_PKG_FILE = "node-v12.16.2-win-x64.zip" #$Env:WEBI_PKG_PATHNAME = "node-v12.16.2-win-x64.zip" diff --git a/_webi/package-install.tpl.sh b/_webi/package-install.tpl.sh index 9bf0d43..5f9eb43 100644 --- a/_webi/package-install.tpl.sh +++ b/_webi/package-install.tpl.sh @@ -15,6 +15,7 @@ __bootstrap_webi() { # TODO not sure if BUILD is the best name for this #WEBI_BUILD= #WEBI_GIT_TAG= + #WEBI_GIT_COMMIT_HASH= #WEBI_LTS= #WEBI_CHANNEL= #WEBI_EXT= diff --git a/cmd/webid/bootstrap_test.go b/cmd/webid/bootstrap_test.go new file mode 100644 index 0000000..4c5d6c3 --- /dev/null +++ b/cmd/webid/bootstrap_test.go @@ -0,0 +1,146 @@ +package main + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// TestBootstrapCurlPipe verifies the /{pkg} route returns the curl-pipe bootstrap. +func TestBootstrapCurlPipe(t *testing.T) { + srv, ts := newTestServer(t) + + pkg := "bat" + if srv.getPackage(pkg) == nil { + t.Skipf("package %s not in cache", pkg) + } + + code, body := get(t, ts, "/bat@stable") + if code != 200 { + t.Fatalf("status %d: %s", code, body[:min(len(body), 200)]) + } + + // Should contain the bootstrap env vars. + if !strings.Contains(body, "WEBI_PKG=") { + t.Error("missing WEBI_PKG= in bootstrap") + } + if !strings.Contains(body, "WEBI_HOST=") { + t.Error("missing WEBI_HOST= in bootstrap") + } + if !strings.Contains(body, "WEBI_CHECKSUM=") { + t.Error("missing WEBI_CHECKSUM= in bootstrap") + } + // Should NOT contain the full installer (install.sh content). + // The bootstrap just downloads and runs webi. + if strings.Contains(body, "pkg_install()") { + t.Error("bootstrap should not contain pkg_install — that's the full installer") + } + + t.Logf("bootstrap size: %d bytes", len(body)) +} + +// TestInstallerFull verifies /api/installers/{pkg}.sh returns the full installer. +func TestInstallerFull(t *testing.T) { + srv, ts := newTestServer(t) + + pkg := "bat" + if srv.getPackage(pkg) == nil { + t.Skipf("package %s not in cache", pkg) + } + + // Use a webi-style User-Agent so the server can detect platform. + code, body := getWithUA(t, ts, "/api/installers/bat@stable.sh", "aarch64/unknown Darwin/24.2.0 libc") + if code != 200 { + t.Fatalf("status %d: %s", code, body[:min(len(body), 500)]) + } + + // Should contain resolved release info. + if !strings.Contains(body, "WEBI_VERSION=") { + t.Error("missing WEBI_VERSION= in installer") + } + if !strings.Contains(body, "WEBI_PKG_URL=") { + t.Error("missing WEBI_PKG_URL= in installer") + } + if !strings.Contains(body, "PKG_NAME=") { + t.Error("missing PKG_NAME= in installer") + } + + // Should contain the package's install.sh content (embedded). + if !strings.Contains(body, "pkg_") { + t.Error("installer should contain pkg_ functions from install.sh") + } + + t.Logf("installer size: %d bytes", len(body)) +} + +// TestInstallerPowerShell verifies /api/installers/{pkg}.ps1 returns a PowerShell installer. +func TestInstallerPowerShell(t *testing.T) { + srv, ts := newTestServer(t) + + pkg := "node" + if srv.getPackage(pkg) == nil { + t.Skipf("package %s not in cache", pkg) + } + + code, body := getWithUA(t, ts, "/api/installers/node@stable.ps1", "AMD64/unknown Windows/10.0.19045 msvc") + if code != 200 { + t.Fatalf("status %d: %s", code, body[:min(len(body), 500)]) + } + + if !strings.Contains(body, "$Env:WEBI_VERSION") { + t.Error("missing $Env:WEBI_VERSION in PS1 installer") + } + if !strings.Contains(body, "$Env:WEBI_PKG_URL") { + t.Error("missing $Env:WEBI_PKG_URL in PS1 installer") + } + if !strings.Contains(body, "$Env:PKG_NAME") { + t.Error("missing $Env:PKG_NAME in PS1 installer") + } + + t.Logf("PS1 installer size: %d bytes", len(body)) +} + +// TestInstallerSelfHosted verifies selfhosted packages get a script without resolution. +func TestInstallerSelfHosted(t *testing.T) { + _, ts := newTestServer(t) + + // ssh-utils is selfhosted — has install.sh but no releases.conf. + code, body := getWithUA(t, ts, "/api/installers/ssh-utils.sh", "aarch64/unknown Darwin/24.2.0 libc") + if code == 404 { + t.Skip("ssh-utils not available as installer") + } + if code != 200 { + t.Skipf("status %d (selfhosted may not render without cache): %s", code, body[:min(len(body), 200)]) + } + + t.Logf("selfhosted installer size: %d bytes", len(body)) +} + +// TestBootstrapUnknownPackage verifies 404 for unknown packages. +func TestBootstrapUnknownPackage(t *testing.T) { + _, ts := newTestServer(t) + + code, _ := get(t, ts, "/nonexistent-package-xyz") + if code != 404 { + t.Errorf("expected 404, got %d", code) + } +} + +// getWithUA fetches a URL with a custom User-Agent header. +func getWithUA(t *testing.T, ts *httptest.Server, path, ua string) (int, string) { + t.Helper() + req, err := http.NewRequest("GET", ts.URL+path, nil) + if err != nil { + t.Fatalf("new request: %v", err) + } + req.Header.Set("User-Agent", ua) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("GET %s: %v", path, err) + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + return resp.StatusCode, string(body) +} diff --git a/cmd/webid/main.go b/cmd/webid/main.go new file mode 100644 index 0000000..4c5de38 --- /dev/null +++ b/cmd/webid/main.go @@ -0,0 +1,989 @@ +// Command webid is the webi HTTP API server. It reads cached release +// data from the filesystem and serves release metadata, installer +// scripts, and bootstrap dispatches. +// +// It never fetches from upstream APIs — that's webicached's job. +// This server is stateless and fast: load from cache, resolve, render. +// +// Usage: +// +// go run ./cmd/webid +// go run ./cmd/webid -addr :3001 -cache ~/.cache/webi/legacy +package main + +import ( + "context" + "crypto/sha1" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "os/signal" + "path/filepath" + "slices" + "strings" + "sync" + "time" + + "github.com/webinstall/webi-installers/internal/buildmeta" + "github.com/webinstall/webi-installers/internal/lexver" + "github.com/webinstall/webi-installers/internal/render" + "github.com/webinstall/webi-installers/internal/resolve" + "github.com/webinstall/webi-installers/internal/resolver" + middleware "github.com/therootcompany/golib/http/middleware/v2" + + "github.com/webinstall/webi-installers/internal/storage" + "github.com/webinstall/webi-installers/internal/storage/fsstore" + "github.com/webinstall/webi-installers/internal/uadetect" +) + +var ( + name = "webid" + version = "0.0.0-dev" + commit = "0000000" + date = "0001-01-01" + licenseYear = "2024" + licenseOwner = "AJ ONeal" + licenseType = "MPL-2.0" +) + +func printVersion(w io.Writer) { + v := strings.TrimPrefix(version, "v") + _, _ = fmt.Fprintf(w, "%s v%s %s (%s)\n", name, v, commit[:7], date) + _, _ = fmt.Fprintf(w, "Copyright (C) %s %s\n", licenseYear, licenseOwner) + _, _ = fmt.Fprintf(w, "Licensed under %s\n", licenseType) +} + +func main() { + addr := flag.String("addr", ":3001", "listen address") + cacheDir := flag.String("legacy", "~/.cache/webi/legacy", "legacy cache directory") + installersDir := flag.String("installers", ".", "installers repo root (for install.sh/ps1)") + + if len(os.Args) > 1 { + switch os.Args[1] { + case "-V", "-version", "--version", "version": + printVersion(os.Stdout) + os.Exit(0) + case "help", "-help", "--help": + printVersion(os.Stdout) + fmt.Fprintln(os.Stdout, "") + flag.CommandLine.SetOutput(os.Stdout) + flag.Usage() + os.Exit(0) + } + } + + flag.Parse() + + cachePath := expandHome(*cacheDir) + + fss, err := fsstore.New(cachePath) + if err != nil { + log.Fatalf("fsstore: %v", err) + } + var store storage.Store = fss + + srv := &server{ + store: store, + installersDir: *installersDir, + packages: make(map[string]*packageCache), + } + + // Pre-load all cached packages. + srv.loadAll() + + mux := http.NewServeMux() + mmux := middleware.WithMux(mux, requestLogger) + + // Legacy API routes (Node.js compat). + mmux.HandleFunc("GET /api/releases/{rest...}", srv.handleReleasesAPI) + + // New API routes (v1). + mmux.HandleFunc("GET /v1/releases/{rest...}", srv.handleV1Releases) + mmux.HandleFunc("GET /v1/resolve/{rest...}", srv.handleV1Resolve) + + // Full installer script (package-install.tpl.sh + install.sh). + mmux.HandleFunc("GET /api/installers/{rest...}", srv.handleInstaller) + + // Debug endpoint. + mmux.HandleFunc("GET /api/debug", srv.handleDebug) + + // Health check (no logging — too noisy). + mux.HandleFunc("GET /api/health", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + fmt.Fprintln(w, "ok") + }) + + // Bootstrap route: /{package} and /{package}@{version} + // Detects UA and returns rendered installer script. + mmux.HandleFunc("GET /{pkgSpec}", srv.handleBootstrap) + + httpSrv := &http.Server{ + Addr: *addr, + Handler: mux, + ReadTimeout: 5 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 120 * time.Second, + } + + // Graceful shutdown. + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + + go func() { + log.Printf("webid listening on %s", *addr) + if err := httpSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatalf("listen: %v", err) + } + }() + + <-ctx.Done() + log.Println("shutting down...") + + shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + httpSrv.Shutdown(shutCtx) +} + +// requestLogger is a middleware that logs each request with status and duration. +func requestLogger(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + rw := &statusWriter{ResponseWriter: w, code: http.StatusOK} + next.ServeHTTP(rw, r) + log.Printf("%s %s %d %s", r.Method, r.URL.Path, rw.code, time.Since(start)) + }) +} + +// statusWriter wraps ResponseWriter to capture the HTTP status code. +type statusWriter struct { + http.ResponseWriter + code int +} + +func (sw *statusWriter) WriteHeader(code int) { + sw.code = code + sw.ResponseWriter.WriteHeader(code) +} + +// server holds the shared state for all HTTP handlers. +type server struct { + store storage.Store + installersDir string + + mu sync.RWMutex + packages map[string]*packageCache + webiCksum string // cached sha1[:8] of webi.sh +} + +// packageCache holds a loaded package's assets and catalog. +type packageCache struct { + assets []storage.Asset + dists []resolve.Dist + catalog resolve.Catalog +} + +// loadAll pre-loads all packages from the store. +func (s *server) loadAll() { + ctx := context.Background() + + pkgs, err := s.store.ListPackages(ctx) + if err != nil { + log.Printf("warn: list packages: %v", err) + return + } + + count := 0 + for _, pkg := range pkgs { + pd, err := s.store.Load(ctx, pkg) + if err != nil { + log.Printf("warn: load %s: %v", pkg, err) + continue + } + if pd == nil || len(pd.Assets) == 0 { + continue + } + + pc := &packageCache{ + assets: pd.Assets, + dists: assetsToDists(pd.Assets), + } + pc.catalog = resolve.Survey(pc.dists) + + s.mu.Lock() + s.packages[pkg] = pc + s.mu.Unlock() + count++ + } + log.Printf("loaded %d packages from store", count) +} + +// getPackage returns the cached package data, or nil if not found. +func (s *server) getPackage(pkg string) *packageCache { + s.mu.RLock() + defer s.mu.RUnlock() + return s.packages[pkg] +} + +// assetsToDists converts storage.Asset slice to resolve.Dist slice. +func assetsToDists(assets []storage.Asset) []resolve.Dist { + dists := make([]resolve.Dist, len(assets)) + for i, a := range assets { + dists[i] = resolve.Dist{ + Filename: a.Filename, + Version: a.Version, + LTS: a.LTS, + Channel: a.Channel, + Date: a.Date, + OS: a.OS, + Arch: a.Arch, + Libc: a.Libc, + Format: a.Format, + Download: a.Download, + Extra: a.Extra, + GitTag: a.GitTag, + GitCommitHash: a.GitCommitHash, + Variants: a.Variants, + } + } + return dists +} + +// handleReleasesAPI serves /api/releases/{package}@{version}.{format} +func (s *server) handleReleasesAPI(w http.ResponseWriter, r *http.Request) { + rest := r.PathValue("rest") + + // Parse: {package}@{version}.{json|tab} or {package}.{json|tab} + pkg, version, format, err := parseReleasePath(rest) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + pc := s.getPackage(pkg) + if pc == nil { + // Check if it's a selfhosted package. + if s.isSelfHosted(pkg) { + s.serveEmptyReleases(w, format) + return + } + http.Error(w, fmt.Sprintf("package %q not found", pkg), http.StatusNotFound) + return + } + + // Parse query parameters. + q := r.URL.Query() + osStr := q.Get("os") + archStr := q.Get("arch") + libcStr := q.Get("libc") + ltsStr := q.Get("lts") + channelStr := q.Get("channel") + formatsStr := q.Get("formats") + limitStr := q.Get("limit") + + // Normalize wildcard "-" to empty (means "any"). + if osStr == "-" { + osStr = "" + } + if archStr == "-" { + archStr = "" + } + if libcStr == "-" { + libcStr = "" + } + + // Map Node.js OS/arch names to our canonical names. + osStr = normalizeQueryOS(osStr) + archStr = normalizeQueryArch(archStr) + + // Parse LTS. + lts := ltsStr == "true" || ltsStr == "1" + + // Handle channel selectors in the version field: @stable, @lts, @beta, etc. + switch strings.ToLower(version) { + case "stable", "latest": + version = "" + if channelStr == "" { + channelStr = "stable" + } + case "lts": + version = "" + lts = true + case "beta", "pre", "preview": + version = "" + if channelStr == "" { + channelStr = "beta" + } + case "rc": + version = "" + if channelStr == "" { + channelStr = "rc" + } + case "alpha", "dev": + version = "" + if channelStr == "" { + channelStr = "alpha" + } + } + + // Parse formats list. + var formats []string + if formatsStr != "" { + formats = strings.Split(formatsStr, ",") + } + + // Parse limit. + limit := 100 + if limitStr != "" { + fmt.Sscanf(limitStr, "%d", &limit) + } + + // Filter matching releases, sort by specificity, then apply limit. + filtered := filterDists(pc.dists, osStr, archStr, libcStr, channelStr, version, formats, lts) + sortDistsDescending(filtered, osStr, archStr) + if len(filtered) > limit { + filtered = filtered[:limit] + } + + switch format { + case "json": + s.serveJSON(w, r, pc, filtered) + case "tab": + s.serveTab(w, r, filtered) + default: + http.Error(w, "unsupported format: "+format, http.StatusBadRequest) + } +} + +// normalizeQueryOS maps Node.js OS names to our canonical names. +func normalizeQueryOS(s string) string { + switch strings.ToLower(s) { + case "macos", "mac": + return "darwin" + case "win": + return "windows" + default: + return s + } +} + +// normalizeQueryArch maps Node.js arch names to our canonical names. +func normalizeQueryArch(s string) string { + switch strings.ToLower(s) { + case "amd64": + return string(buildmeta.ArchAMD64) // "x86_64" + case "arm64": + return string(buildmeta.ArchARM64) // "aarch64" + case "armv7l": + return string(buildmeta.ArchARMv7) + case "armv6l": + return string(buildmeta.ArchARMv6) + case "x86", "i386", "i686": + return string(buildmeta.ArchX86) + default: + return s + } +} + +// parseReleasePath parses "{pkg}@{version}.{format}" or "{pkg}.{format}". +func parseReleasePath(rest string) (pkg, version, format string, err error) { + if strings.HasSuffix(rest, ".json") { + format = "json" + rest = strings.TrimSuffix(rest, ".json") + } else if strings.HasSuffix(rest, ".tab") { + format = "tab" + rest = strings.TrimSuffix(rest, ".tab") + } else { + return "", "", "", fmt.Errorf("unsupported format (use .json or .tab)") + } + + if idx := strings.IndexByte(rest, '@'); idx >= 0 { + pkg = rest[:idx] + version = rest[idx+1:] + } else { + pkg = rest + } + + if pkg == "" { + return "", "", "", fmt.Errorf("package name required") + } + + return pkg, version, format, nil +} + +// filterDists filters dists by query parameters, returning all matches +// up to limit. This is for the API listing, not single-best resolution. +func filterDists(dists []resolve.Dist, osStr, archStr, libcStr, channel, version string, formats []string, lts bool) []resolve.Dist { + var result []resolve.Dist + + archSet := make(map[string]bool) + if archStr != "" { + for _, a := range buildmeta.CompatArches(buildmeta.OS(osStr), buildmeta.Arch(archStr)) { + archSet[string(a)] = true + } + if len(archSet) == 0 { + archSet[archStr] = true + } + } + + for _, d := range dists { + if osStr != "" && d.OS != osStr && d.OS != "*" && d.OS != "ANYOS" && d.OS != "" && + !(d.OS == "posix_2017" && osStr != "windows") { + continue + } + + if archStr != "" && !archSet[d.Arch] && d.Arch != "*" && d.Arch != "ANYARCH" && d.Arch != "" { + continue + } + + if libcStr != "" && d.Libc != "none" && d.Libc != "" && d.Libc != libcStr { + continue + } + + if lts && !d.LTS { + continue + } + + if channel != "" && d.Channel != channel { + continue + } + + if version != "" { + // Match with or without "v" prefix: + // query "0.25" should match version "v0.25.0". + v := strings.TrimPrefix(d.Version, "v") + vq := strings.TrimPrefix(version, "v") + if !strings.HasPrefix(v, vq) { + continue + } + } + + if len(formats) > 0 { + matched := false + for _, f := range formats { + if strings.Contains(d.Format, f) { + matched = true + break + } + } + if !matched { + continue + } + } + + result = append(result, d) + } + + return result +} + +// legacyRelease matches the Node.js JSON response format. +// Production returns a bare JSON array of these objects. +type legacyRelease struct { + Name string `json:"name"` + Version string `json:"version"` + GitTag string `json:"git_tag,omitempty"` + GitCommitHash string `json:"git_commit_hash,omitempty"` + LTS bool `json:"lts"` + Channel string `json:"channel"` + Date string `json:"date"` + OS string `json:"os"` + Arch string `json:"arch"` + Ext string `json:"ext"` + Download string `json:"download"` + Libc string `json:"libc"` +} + +// legacyOS maps Go canonical OS names to Node.js legacy names. +func legacyOS(s string) string { + switch s { + case "darwin": + return "macos" + case "": + return "*" + default: + return s + } +} + +// legacyArch maps Go canonical arch names to Node.js legacy names. +func legacyArch(s string) string { + switch s { + case "x86_64": + return "amd64" + case "aarch64": + return "arm64" + case "armv7": + return "armv7l" + case "armv6": + return "armv6l" + case "armv5": + return "arm" + case "": + return "*" + default: + return s + } +} + +// legacyExt strips the leading "." from format strings. +func legacyExt(s string) string { + s = strings.TrimPrefix(s, ".") + if s == "" { + return "exe" + } + return s +} + +// legacyVersion strips the leading "v" from version strings. +func legacyVersion(s string) string { + return strings.TrimPrefix(s, "v") +} + +// legacyLibc returns "none" for empty libc values. +func legacyLibc(s string) string { + if s == "" { + return "none" + } + return s +} + +func distsToLegacy(dists []resolve.Dist) []legacyRelease { + releases := make([]legacyRelease, len(dists)) + for i, d := range dists { + releases[i] = legacyRelease{ + Name: d.Filename, + Version: legacyVersion(d.Version), + GitTag: d.GitTag, + GitCommitHash: d.GitCommitHash, + LTS: d.LTS, + Channel: d.Channel, + Date: d.Date, + OS: legacyOS(d.OS), + Arch: legacyArch(d.Arch), + Ext: legacyExt(d.Format), + Download: d.Download, + Libc: legacyLibc(d.Libc), + } + } + return releases +} + +func (s *server) serveJSON(w http.ResponseWriter, r *http.Request, pc *packageCache, filtered []resolve.Dist) { + // Production returns a bare JSON array, not wrapped in an object. + releases := distsToLegacy(filtered) + + w.Header().Set("Content-Type", "application/json") + + pretty := r.URL.Query().Get("pretty") + if pretty == "true" || pretty == "1" { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + enc.Encode(releases) + } else { + json.NewEncoder(w).Encode(releases) + } +} + +func (s *server) serveTab(w http.ResponseWriter, r *http.Request, filtered []resolve.Dist) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + + // Production only shows header row with ?pretty=true. + pretty := r.URL.Query().Get("pretty") + if pretty != "" && pretty != "false" { + fmt.Fprintln(w, "VERSION\tLTS\tCHANNEL\tRELEASE_DATE\tOS\tARCH\tEXT\tHASH\tURL\t_\tLIBC") + } + + // Tab format matches Node.js production: + // version \t lts \t channel \t date \t os \t arch \t ext \t hash \t download \t comment \t libc + for _, d := range filtered { + lts := "-" + if d.LTS { + lts = "lts" + } + channel := d.Channel + if channel == "" { + channel = "-" + } + date := d.Date + if date == "" { + date = "-" + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t-\t%s\t\t%s\n", + legacyVersion(d.Version), + lts, + channel, + date, + legacyOS(d.OS), + legacyArch(d.Arch), + legacyExt(d.Format), + d.Download, + legacyLibc(d.Libc), + ) + } +} + +// sortDistsDescending sorts dists newest-first by version. +func sortDistsDescending(dists []resolve.Dist, queryOS, queryArch string) { + slices.SortStableFunc(dists, func(a, b resolve.Dist) int { + va := lexver.Parse(strings.TrimPrefix(a.Version, "v")) + vb := lexver.Parse(strings.TrimPrefix(b.Version, "v")) + if cmp := lexver.Compare(vb, va); cmp != 0 { + return cmp + } + if cmp := osSpecificity(a.OS, queryOS) - osSpecificity(b.OS, queryOS); cmp != 0 { + return cmp + } + if cmp := archSpecificity(a.Arch, queryArch) - archSpecificity(b.Arch, queryArch); cmp != 0 { + return cmp + } + return libcRank(a.Libc) - libcRank(b.Libc) + }) +} + +func osSpecificity(distOS, queryOS string) int { + switch { + case distOS == queryOS: + return 0 + case distOS == "posix_2017": + return 1 + default: + return 2 + } +} + +func archSpecificity(distArch, queryArch string) int { + switch { + case distArch == queryArch: + return 0 + case distArch == "" || distArch == "*": + return 2 + default: + return 1 + } +} + +func libcRank(libc string) int { + switch libc { + case "none", "": + return 0 + default: + return 1 + } +} + +// serveEmptyReleases returns an empty release list for selfhosted packages. +func (s *server) serveEmptyReleases(w http.ResponseWriter, format string) { + switch format { + case "json": + w.Header().Set("Content-Type", "application/json") + // Production returns an empty array. + json.NewEncoder(w).Encode([]legacyRelease{}) + case "tab": + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + } +} + +// isSelfHosted checks if a package has install.sh but no releases.conf. +func (s *server) isSelfHosted(pkg string) bool { + installPath := filepath.Join(s.installersDir, pkg, "install.sh") + if _, err := os.Stat(installPath); err != nil { + return false + } + confPath := filepath.Join(s.installersDir, pkg, "releases.conf") + if _, err := os.Stat(confPath); err == nil { + return false + } + return true +} + +// handleDebug returns UA detection info for the requesting client. +func (s *server) handleDebug(w http.ResponseWriter, r *http.Request) { + result := uadetect.FromRequest(r) + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]string{ + "user_agent": r.Header.Get("User-Agent"), + "os": string(result.OS), + "arch": string(result.Arch), + "libc": string(result.Libc), + }) +} + +// handleBootstrap serves /{package} and /{package}@{version}. +// This is the curl-pipe bootstrap: a minimal script that sets +// WEBI_PKG/WEBI_HOST/WEBI_CHECKSUM and downloads+runs webi. +func (s *server) handleBootstrap(w http.ResponseWriter, r *http.Request) { + pkgSpec := r.PathValue("pkgSpec") + + // Parse package@version. + pkg, tag := pkgSpec, "" + if idx := strings.IndexByte(pkgSpec, '@'); idx >= 0 { + pkg = pkgSpec[:idx] + tag = pkgSpec[idx+1:] + } + + if pkg == "" { + http.Error(w, "package name required", http.StatusBadRequest) + return + } + + // Verify package exists. + if s.getPackage(pkg) == nil && !s.isSelfHosted(pkg) { + http.Error(w, fmt.Sprintf("package %q not found", pkg), http.StatusNotFound) + return + } + + baseURL := baseURLFromRequest(r) + webiPkg := pkg + if tag != "" { + webiPkg = pkg + "@" + tag + } + + // Read and inject the curl-pipe bootstrap template. + tplPath := filepath.Join(s.installersDir, "_webi", "curl-pipe-bootstrap.tpl.sh") + tpl, err := os.ReadFile(tplPath) + if err != nil { + log.Printf("bootstrap: read template: %v", err) + http.Error(w, "bootstrap template not found", http.StatusInternalServerError) + return + } + + script := string(tpl) + script = render.InjectVar(script, "WEBI_PKG", webiPkg) + script = render.InjectVar(script, "WEBI_HOST", baseURL) + script = render.InjectVar(script, "WEBI_CHECKSUM", s.webiChecksum()) + + // text/html so browsers see the meta redirect to cheat sheet. + w.Header().Set("Content-Type", "text/html; charset=utf-8") + fmt.Fprint(w, script) +} + +// handleInstaller serves /api/installers/{pkg}@{version}.sh +// This is the full installer script with release resolution and +// embedded install.sh. +func (s *server) handleInstaller(w http.ResponseWriter, r *http.Request) { + rest := r.PathValue("rest") + + // Parse: {pkg}@{version}.sh or {pkg}.sh + ext := "" + if strings.HasSuffix(rest, ".sh") { + ext = "sh" + rest = strings.TrimSuffix(rest, ".sh") + } else if strings.HasSuffix(rest, ".ps1") { + ext = "ps1" + rest = strings.TrimSuffix(rest, ".ps1") + } else { + http.Error(w, "unsupported format (use .sh or .ps1)", http.StatusBadRequest) + return + } + + pkg, tag := rest, "" + if idx := strings.IndexByte(rest, '@'); idx >= 0 { + pkg = rest[:idx] + tag = rest[idx+1:] + } + + if pkg == "" { + http.Error(w, "package name required", http.StatusBadRequest) + return + } + // Detect platform from User-Agent. + ua := uadetect.FromRequest(r) + if ua.OS == "" { + http.Error(w, "could not detect OS from User-Agent", http.StatusBadRequest) + return + } + + isSelfHosted := s.isSelfHosted(pkg) + pc := s.getPackage(pkg) + + if pc == nil && !isSelfHosted { + http.Error(w, fmt.Sprintf("package %q not found", pkg), http.StatusNotFound) + return + } + + baseURL := baseURLFromRequest(r) + + p := render.Params{ + Host: baseURL, + PkgName: pkg, + Tag: tag, + OS: string(ua.OS), + Arch: string(ua.Arch), + Libc: string(ua.Libc), + } + + // Resolve the best release (if not selfhosted). + if pc != nil { + req := resolver.Request{ + OS: string(ua.OS), + Arch: string(ua.Arch), + Libc: string(ua.Libc), + } + + switch strings.ToLower(tag) { + case "stable", "latest", "": + // Default. + case "lts": + req.LTS = true + case "beta", "pre", "preview": + req.Channel = "beta" + case "rc": + req.Channel = "rc" + case "alpha", "dev": + req.Channel = "alpha" + default: + req.Version = tag + } + + res, err := resolver.Resolve(pc.assets, req) + if err != nil { + p.Version = "0.0.0" + p.Channel = "error" + p.Ext = "err" + p.PkgURL = "https://example.com/doesntexist.ext" + p.PkgFile = "doesntexist.ext" + p.CSV = buildCSV(p) + } else { + v := strings.TrimPrefix(res.Version, "v") + parts := splitVersion(v) + p.Version = v + p.Major = parts[0] + p.Minor = parts[1] + p.Patch = parts[2] + p.Build = parts[3] + if res.Asset.GitTag != "" { + p.GitTag = res.Asset.GitTag + } else { + p.GitTag = "v" + v + } + p.GitBranch = p.GitTag + p.GitCommitHash = res.Asset.GitCommitHash + p.LTS = fmt.Sprintf("%v", res.Asset.LTS) + p.Channel = res.Asset.Channel + if p.Channel == "" { + p.Channel = "stable" + } + p.Ext = strings.TrimPrefix(res.Asset.Format, ".") + if p.Ext == "" { + p.Ext = "exe" + } + p.PkgURL = res.Asset.Download + p.PkgFile = res.Asset.Filename + p.CSV = buildCSV(p) + } + + p.PkgStable = pc.catalog.Stable + p.PkgLatest = pc.catalog.Latest + p.PkgOSes = strings.Join(pc.catalog.OSes, " ") + p.PkgArches = strings.Join(pc.catalog.Arches, " ") + p.PkgLibcs = strings.Join(pc.catalog.Libcs, " ") + p.PkgFormats = strings.Join(pc.catalog.Formats, " ") + } + + p.ReleasesURL = fmt.Sprintf("%s/api/releases/%s@%s.tab?os=%s&arch=%s&libc=%s&formats=tar&pretty=true", + baseURL, pkg, tag, p.OS, p.Arch, p.Libc) + + var script string + var renderErr error + if ext == "ps1" { + tplPath := filepath.Join(s.installersDir, "_webi", "package-install.tpl.ps1") + script, renderErr = render.PowerShell(tplPath, s.installersDir, pkg, p) + } else { + tplPath := filepath.Join(s.installersDir, "_webi", "package-install.tpl.sh") + script, renderErr = render.Bash(tplPath, s.installersDir, pkg, p) + } + if renderErr != nil { + log.Printf("render %s: %v", pkg, renderErr) + http.Error(w, fmt.Sprintf("failed to render installer for %q: %v", pkg, renderErr), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + fmt.Fprint(w, script) +} + +// baseURLFromRequest builds the base URL from the request. +func baseURLFromRequest(r *http.Request) string { + if r.TLS != nil || strings.Contains(r.Host, "webinstall") || strings.Contains(r.Host, "webi.") { + return "https://" + r.Host + } + return "http://" + r.Host +} + +// webiChecksum returns the checksum of the webi.sh bootstrap script. +func (s *server) webiChecksum() string { + s.mu.RLock() + cksum := s.webiCksum + s.mu.RUnlock() + if cksum != "" { + return cksum + } + + // Calculate checksum from webi.sh file. + webiPath := filepath.Join(s.installersDir, "webi", "webi.sh") + data, err := os.ReadFile(webiPath) + if err != nil { + return "00000000" + } + + h := sha1.New() + h.Write(data) + cksum = fmt.Sprintf("%x", h.Sum(nil))[:8] + + s.mu.Lock() + s.webiCksum = cksum + s.mu.Unlock() + return cksum +} + +// buildCSV creates the WEBI_CSV line in the Node.js format. +func buildCSV(p render.Params) string { + return strings.Join([]string{ + p.Version, + p.LTS, + p.Channel, + "", // date + p.OS, + p.Arch, + p.Ext, + "-", + p.PkgURL, + p.PkgFile, + "", + }, ",") +} + +// splitVersion splits a version string into [major, minor, patch, build]. +func splitVersion(v string) [4]string { + // Strip pre-release suffix for splitting. + base := v + build := "" + if idx := strings.IndexByte(v, '-'); idx >= 0 { + base = v[:idx] + build = v[idx+1:] + } + + parts := strings.SplitN(base, ".", 4) + var result [4]string + for i := 0; i < len(parts) && i < 3; i++ { + result[i] = parts[i] + } + result[3] = build + return result +} + +func expandHome(path string) string { + if !strings.HasPrefix(path, "~/") { + return path + } + home, err := os.UserHomeDir() + if err != nil { + return path + } + return filepath.Join(home, path[2:]) +} diff --git a/cmd/webid/main_test.go b/cmd/webid/main_test.go new file mode 100644 index 0000000..e2f94c9 --- /dev/null +++ b/cmd/webid/main_test.go @@ -0,0 +1,298 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/webinstall/webi-installers/internal/resolve" + "github.com/webinstall/webi-installers/internal/storage" + "github.com/webinstall/webi-installers/internal/storage/fsstore" +) + +// newTestServer creates a server backed by the real _cache directory +// and returns an httptest.Server with proper routing (so PathValue works). +func newTestServer(t *testing.T) (*server, *httptest.Server) { + t.Helper() + + cacheDir := filepath.Join("..", "..", "_cache") + if _, err := os.Stat(cacheDir); err != nil { + t.Skipf("no cache dir at %s", cacheDir) + } + + store, err := fsstore.New(cacheDir) + if err != nil { + t.Fatalf("fsstore: %v", err) + } + + srv := &server{ + store: store, + installersDir: filepath.Join("..", ".."), + packages: make(map[string]*packageCache), + } + + // Load packages. + monthDir := time.Now().Format("2006-01") + dir := filepath.Join(store.Root(), monthDir) + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("readdir: %v", err) + } + for _, e := range entries { + if !strings.HasSuffix(e.Name(), ".json") { + continue + } + pkg := strings.TrimSuffix(e.Name(), ".json") + pd, err := store.Load(context.Background(), pkg) + if err != nil || pd == nil || len(pd.Assets) == 0 { + continue + } + pc := &packageCache{ + assets: pd.Assets, + dists: assetsToDists(pd.Assets), + } + pc.catalog = resolve.Survey(pc.dists) + srv.packages[pkg] = pc + } + + mux := http.NewServeMux() + mux.HandleFunc("GET /api/releases/{rest...}", srv.handleReleasesAPI) + mux.HandleFunc("GET /v1/releases/{rest...}", srv.handleV1Releases) + mux.HandleFunc("GET /v1/resolve/{rest...}", srv.handleV1Resolve) + mux.HandleFunc("GET /api/installers/{rest...}", srv.handleInstaller) + mux.HandleFunc("GET /api/debug", srv.handleDebug) + mux.HandleFunc("GET /{pkgSpec}", srv.handleBootstrap) + + ts := httptest.NewServer(mux) + t.Cleanup(ts.Close) + + return srv, ts +} + +// get fetches a URL from the test server and returns the body. +func get(t *testing.T, ts *httptest.Server, path string) (int, string) { + t.Helper() + resp, err := http.Get(ts.URL + path) + if err != nil { + t.Fatalf("GET %s: %v", path, err) + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + return resp.StatusCode, string(body) +} + +// TestLegacyJSONFormat verifies our JSON output matches the production format. +func TestLegacyJSONFormat(t *testing.T) { + srv, ts := newTestServer(t) + + packages := []string{"bat", "node", "go", "jq"} + for _, pkg := range packages { + t.Run(pkg, func(t *testing.T) { + if srv.getPackage(pkg) == nil { + t.Skipf("package %s not in cache", pkg) + } + + code, body := get(t, ts, "/api/releases/"+pkg+".json?limit=5") + if code != http.StatusOK { + t.Fatalf("status %d: %s", code, body) + } + + body = strings.TrimSpace(body) + + // Must be a JSON array, not an object. + if !strings.HasPrefix(body, "[") { + t.Fatalf("expected JSON array, got: %.100s", body) + } + + var releases []legacyRelease + if err := json.Unmarshal([]byte(body), &releases); err != nil { + t.Fatalf("decode: %v", err) + } + if len(releases) == 0 { + t.Fatal("no releases returned") + } + + // Check field format conventions. + for i, r := range releases { + if strings.HasPrefix(r.Version, "v") { + t.Errorf("release[%d]: version %q should not have v prefix", i, r.Version) + } + if strings.HasPrefix(r.Ext, ".") { + t.Errorf("release[%d]: ext %q should not have . prefix", i, r.Ext) + } + if r.OS == "darwin" { + t.Errorf("release[%d]: os should be 'macos' not 'darwin'", i) + } + if r.Arch == "x86_64" { + t.Errorf("release[%d]: arch should be 'amd64' not 'x86_64'", i) + } + if r.Arch == "aarch64" { + t.Errorf("release[%d]: arch should be 'arm64' not 'aarch64'", i) + } + if r.Libc == "" { + t.Errorf("release[%d]: libc should be 'none' not empty", i) + } + if r.Download == "" { + t.Errorf("release[%d]: download URL is empty", i) + } + } + }) + } +} + +// TestLegacyTabFormat verifies our .tab output uses real TSV. +func TestLegacyTabFormat(t *testing.T) { + srv, ts := newTestServer(t) + + packages := []string{"bat", "node", "go"} + for _, pkg := range packages { + t.Run(pkg, func(t *testing.T) { + if srv.getPackage(pkg) == nil { + t.Skipf("package %s not in cache", pkg) + } + + code, body := get(t, ts, "/api/releases/"+pkg+".tab?limit=3") + if code != http.StatusOK { + t.Fatalf("status %d: %s", code, body) + } + + lines := strings.Split(strings.TrimSpace(body), "\n") + if len(lines) == 0 { + t.Fatal("no lines returned") + } + + for i, line := range lines { + fields := strings.Split(line, "\t") + // Expect 11 tab-separated fields: + // version, lts, channel, date, os, arch, ext, hash, download, (empty), libc + if len(fields) != 11 { + t.Errorf("line[%d]: expected 11 tab fields, got %d: %q", i, len(fields), line) + continue + } + + version := fields[0] + lts := fields[1] + ext := fields[6] + + if strings.HasPrefix(version, "v") { + t.Errorf("line[%d]: version %q should not have v prefix", i, version) + } + if lts != "-" && lts != "lts" { + t.Errorf("line[%d]: lts should be '-' or 'lts', got %q", i, lts) + } + if strings.HasPrefix(ext, ".") { + t.Errorf("line[%d]: ext %q should not have . prefix", i, ext) + } + } + }) + } +} + +// TestLegacyJSONAgainstProduction compares our output against live production. +// Run with: WEBI_TEST_PROD=1 go test -run TestLegacyJSONAgainstProduction +func TestLegacyJSONAgainstProduction(t *testing.T) { + if os.Getenv("WEBI_TEST_PROD") == "" { + t.Skip("set WEBI_TEST_PROD=1 to compare against production") + } + + srv, ts := newTestServer(t) + + packages := []string{"bat", "node", "go", "jq", "rg"} + for _, pkg := range packages { + t.Run(pkg, func(t *testing.T) { + if srv.getPackage(pkg) == nil { + t.Skipf("package %s not in cache", pkg) + } + + // Fetch from production. + prodURL := fmt.Sprintf("https://webinstall.dev/api/releases/%s.json?limit=3", pkg) + prodResp, err := http.Get(prodURL) + if err != nil { + t.Fatalf("fetch production: %v", err) + } + defer prodResp.Body.Close() + prodBody, _ := io.ReadAll(prodResp.Body) + + var prodReleases []legacyRelease + if err := json.Unmarshal(prodBody, &prodReleases); err != nil { + t.Fatalf("decode production: %v\nbody: %.500s", err, string(prodBody)) + } + + // Fetch from local. + _, localBody := get(t, ts, "/api/releases/"+pkg+".json?limit=3") + + var localReleases []legacyRelease + if err := json.Unmarshal([]byte(localBody), &localReleases); err != nil { + t.Fatalf("decode local: %v", err) + } + + if len(prodReleases) == 0 || len(localReleases) == 0 { + t.Skip("empty releases") + } + + // Compare the first release's format. + prod := prodReleases[0] + local := localReleases[0] + + if strings.HasPrefix(local.Version, "v") != strings.HasPrefix(prod.Version, "v") { + t.Errorf("version prefix mismatch: prod=%q local=%q", prod.Version, local.Version) + } + if strings.HasPrefix(local.Ext, ".") != strings.HasPrefix(prod.Ext, ".") { + t.Errorf("ext prefix mismatch: prod=%q local=%q", prod.Ext, local.Ext) + } + if prod.OS == "macos" && local.OS == "darwin" { + t.Error("OS: prod uses 'macos', local uses 'darwin'") + } + if prod.Arch == "amd64" && local.Arch == "x86_64" { + t.Error("Arch: prod uses 'amd64', local uses 'x86_64'") + } + if prod.Arch == "arm64" && local.Arch == "aarch64" { + t.Error("Arch: prod uses 'arm64', local uses 'aarch64'") + } + + t.Logf("prod[0]: version=%q os=%q arch=%q ext=%q libc=%q", + prod.Version, prod.OS, prod.Arch, prod.Ext, prod.Libc) + t.Logf("local[0]: version=%q os=%q arch=%q ext=%q libc=%q", + local.Version, local.OS, local.Arch, local.Ext, local.Libc) + }) + } +} + +// TestSortOrder verifies releases come back newest-first. +func TestSortOrder(t *testing.T) { + srv, ts := newTestServer(t) + + pkg := "bat" + if srv.getPackage(pkg) == nil { + t.Skipf("package %s not in cache", pkg) + } + + _, body := get(t, ts, "/api/releases/"+pkg+".json?limit=20") + + var releases []legacyRelease + if err := json.Unmarshal([]byte(body), &releases); err != nil { + t.Fatalf("decode: %v", err) + } + + if len(releases) < 2 { + t.Skip("need at least 2 releases") + } + + // First release should be newest (or equal) version. + first := releases[0].Date + last := releases[len(releases)-1].Date + if first < last { + t.Errorf("not newest-first: first=%q last=%q", first, last) + } +} + +// Ensure imports are used. +var _ = storage.Asset{} diff --git a/cmd/webid/v1api.go b/cmd/webid/v1api.go new file mode 100644 index 0000000..03b1f1c --- /dev/null +++ b/cmd/webid/v1api.go @@ -0,0 +1,459 @@ +package main + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "fmt" + "net/http" + "slices" + "strings" + + "github.com/jszwec/csvutil" + + "github.com/webinstall/webi-installers/internal/buildmeta" + "github.com/webinstall/webi-installers/internal/lexver" + "github.com/webinstall/webi-installers/internal/resolver" + "github.com/webinstall/webi-installers/internal/storage" +) + +// v1Release is a single release in the new API TSV format. +// Field order matters for csvutil — it determines column order. +// Fields are designed to be easy to consume with cut/grep/sort. +type v1Release struct { + Version string `csv:"version"` + Channel string `csv:"channel"` + LTS string `csv:"lts"` + Date string `csv:"date"` + OS string `csv:"os"` + Arch string `csv:"arch"` + Libc string `csv:"libc"` + Format string `csv:"format"` + Variants string `csv:"variants"` // space-separated + Download string `csv:"download"` + Filename string `csv:"filename"` +} + +// v1ResolveResult is the response for /v1/resolve/{pkg}. +type v1ResolveResult struct { + Version string `csv:"version" json:"version"` + Channel string `csv:"channel" json:"channel"` + LTS string `csv:"lts" json:"lts"` + Date string `csv:"date" json:"date"` + OS string `csv:"os" json:"os"` + Arch string `csv:"arch" json:"arch"` + Libc string `csv:"libc" json:"libc"` + Format string `csv:"format" json:"format"` + Variants string `csv:"variants" json:"variants"` + Download string `csv:"download" json:"download"` + Filename string `csv:"filename" json:"filename"` + Triplet string `csv:"triplet" json:"triplet"` +} + +// handleV1Releases serves /v1/releases/{pkg}.tsv (or .json) +// with Go-native naming and TSV-first format. +// +// Query params: +// +// os — filter by OS (darwin, linux, windows) +// arch — filter by arch (aarch64, x86_64, armv7l) +// libc — filter by libc (gnu, musl, msvc) +// channel — release channel (stable, beta, rc, alpha) +// version — version prefix filter (e.g. "1.20") +// lts — if "true", only LTS releases +// format — filter by format (e.g. "tar.gz") +// variant — filter by variant (e.g. "rocm") +// limit — max results (default 1000) +func (s *server) handleV1Releases(w http.ResponseWriter, r *http.Request) { + rest := r.PathValue("rest") + + pkg, version, format, err := parseReleasePath(rest) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + pc := s.getPackage(pkg) + if pc == nil { + if s.isSelfHosted(pkg) { + s.v1ServeEmpty(w, format) + return + } + http.Error(w, fmt.Sprintf("package %q not found", pkg), http.StatusNotFound) + return + } + + q := r.URL.Query() + osStr := q.Get("os") + archStr := q.Get("arch") + libcStr := q.Get("libc") + channelStr := q.Get("channel") + ltsStr := q.Get("lts") + formatFilter := q.Get("format") + variantStr := q.Get("variant") + limitStr := q.Get("limit") + + // Use version from URL path or query. + if version == "" { + version = q.Get("version") + } + + // Handle channel selectors in version field. + switch strings.ToLower(version) { + case "stable", "latest": + version = "" + if channelStr == "" { + channelStr = "stable" + } + case "lts": + version = "" + ltsStr = "true" + case "beta", "pre", "preview": + version = "" + if channelStr == "" { + channelStr = "beta" + } + case "rc": + version = "" + if channelStr == "" { + channelStr = "rc" + } + case "alpha", "dev": + version = "" + if channelStr == "" { + channelStr = "alpha" + } + } + + lts := ltsStr == "true" || ltsStr == "1" + + limit := 1000 + if limitStr != "" { + fmt.Sscanf(limitStr, "%d", &limit) + } + + // Filter assets directly (not via resolve.Dist). + filtered := filterAssets(pc.assets, osStr, archStr, libcStr, channelStr, version, formatFilter, variantStr, lts, limit) + + // Sort newest-first. + sortAssetsDescending(filtered) + + switch format { + case "json": + s.v1ServeJSON(w, filtered) + case "tab": + s.v1ServeTSV(w, filtered) + default: + http.Error(w, "unsupported format: "+format+" (use .json or .tab)", http.StatusBadRequest) + } +} + +// handleV1Resolve serves /v1/resolve/{pkg}.tsv (or .json) +// It resolves the single best asset for a given platform. +// +// Query params: +// +// os — target OS (required) +// arch — target arch (required) +// libc — target libc +// version — version prefix +// channel — release channel +// lts — if "true", only LTS +// format — preferred formats (comma-separated, in preference order) +// variant — preferred variant +func (s *server) handleV1Resolve(w http.ResponseWriter, r *http.Request) { + rest := r.PathValue("rest") + + pkg, version, format, err := parseReleasePath(rest) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + pc := s.getPackage(pkg) + if pc == nil { + http.Error(w, fmt.Sprintf("package %q not found", pkg), http.StatusNotFound) + return + } + + q := r.URL.Query() + osStr := q.Get("os") + archStr := q.Get("arch") + libcStr := q.Get("libc") + channelStr := q.Get("channel") + ltsStr := q.Get("lts") + formatsStr := q.Get("format") + variantStr := q.Get("variant") + + if version == "" { + version = q.Get("version") + } + + // Handle channel selectors in version field. + switch strings.ToLower(version) { + case "stable", "latest": + version = "" + if channelStr == "" { + channelStr = "stable" + } + case "lts": + version = "" + ltsStr = "true" + case "beta", "pre", "preview": + version = "" + if channelStr == "" { + channelStr = "beta" + } + case "rc": + version = "" + if channelStr == "" { + channelStr = "rc" + } + case "alpha", "dev": + version = "" + if channelStr == "" { + channelStr = "alpha" + } + } + + lts := ltsStr == "true" || ltsStr == "1" + + var formats []string + if formatsStr != "" { + formats = strings.Split(formatsStr, ",") + } + + req := resolver.Request{ + OS: osStr, + Arch: archStr, + Libc: libcStr, + Version: version, + Channel: channelStr, + LTS: lts, + Formats: formats, + Variant: variantStr, + } + + res, err := resolver.Resolve(pc.assets, req) + if err != nil { + http.Error(w, fmt.Sprintf("no match for %s: %v", pkg, err), http.StatusNotFound) + return + } + + result := assetToV1Resolve(res) + + switch format { + case "json": + w.Header().Set("Content-Type", "application/json") + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + enc.Encode(result) + case "tab": + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + data, err := marshalTSV([]v1ResolveResult{result}) + if err != nil { + http.Error(w, "encode error: "+err.Error(), http.StatusInternalServerError) + return + } + w.Write(data) + default: + http.Error(w, "unsupported format: "+format, http.StatusBadRequest) + } +} + +func assetToV1Release(a storage.Asset) v1Release { + lts := "-" + if a.LTS { + lts = "lts" + } + channel := a.Channel + if channel == "" { + channel = "stable" + } + libc := a.Libc + if libc == "" { + libc = "-" + } + return v1Release{ + Version: a.Version, + Channel: channel, + LTS: lts, + Date: a.Date, + OS: a.OS, + Arch: a.Arch, + Libc: libc, + Format: a.Format, + Variants: strings.Join(a.Variants, " "), + Download: a.Download, + Filename: a.Filename, + } +} + +func assetToV1Resolve(res resolver.Result) v1ResolveResult { + a := res.Asset + lts := "-" + if a.LTS { + lts = "lts" + } + channel := a.Channel + if channel == "" { + channel = "stable" + } + libc := a.Libc + if libc == "" { + libc = "-" + } + return v1ResolveResult{ + Version: a.Version, + Channel: channel, + LTS: lts, + Date: a.Date, + OS: a.OS, + Arch: a.Arch, + Libc: libc, + Format: a.Format, + Variants: strings.Join(a.Variants, " "), + Download: a.Download, + Filename: a.Filename, + Triplet: res.Triplet, + } +} + +func (s *server) v1ServeTSV(w http.ResponseWriter, assets []storage.Asset) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + + releases := make([]v1Release, len(assets)) + for i, a := range assets { + releases[i] = assetToV1Release(a) + } + + data, err := marshalTSV(releases) + if err != nil { + http.Error(w, "encode error: "+err.Error(), http.StatusInternalServerError) + return + } + w.Write(data) +} + +func (s *server) v1ServeJSON(w http.ResponseWriter, assets []storage.Asset) { + w.Header().Set("Content-Type", "application/json") + + releases := make([]v1Release, len(assets)) + for i, a := range assets { + releases[i] = assetToV1Release(a) + } + + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + enc.Encode(releases) +} + +func (s *server) v1ServeEmpty(w http.ResponseWriter, format string) { + switch format { + case "json": + w.Header().Set("Content-Type", "application/json") + w.Write([]byte("[]\n")) + case "tab": + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + // Just the header. + data, _ := marshalTSV([]v1Release{}) + w.Write(data) + } +} + +// filterAssets filters storage.Asset slices directly. +func filterAssets(assets []storage.Asset, osStr, archStr, libcStr, channel, version, formatFilter, variant string, lts bool, limit int) []storage.Asset { + var result []storage.Asset + + for _, a := range assets { + if osStr != "" && a.OS != osStr && a.OS != "ANYOS" && a.OS != "" { + continue + } + if archStr != "" && a.Arch != archStr && a.Arch != "ANYARCH" && a.Arch != "" { + continue + } + if libcStr != "" && a.Libc != "" && a.Libc != "none" && a.Libc != libcStr { + continue + } + if lts && !a.LTS { + continue + } + if channel != "" && a.Channel != channel { + continue + } + if version != "" { + v := strings.TrimPrefix(a.Version, "v") + vq := strings.TrimPrefix(version, "v") + if !strings.HasPrefix(v, vq) { + continue + } + } + if formatFilter != "" && !strings.Contains(a.Format, formatFilter) { + continue + } + if variant != "" { + if !hasVariant(a.Variants, variant) { + continue + } + } + + result = append(result, a) + if len(result) >= limit { + break + } + } + + return result +} + +// sortAssetsDescending sorts assets newest-first by version. +func sortAssetsDescending(assets []storage.Asset) { + slices.SortStableFunc(assets, func(a, b storage.Asset) int { + va := lexver.Parse(strings.TrimPrefix(a.Version, "v")) + vb := lexver.Parse(strings.TrimPrefix(b.Version, "v")) + return lexver.Compare(vb, va) // descending + }) +} + +// hasVariant checks if the variant list contains the wanted variant. +// This is a copy of resolver.hasVariant since it's unexported. +func hasVariant(variants []string, want string) bool { + for _, v := range variants { + if v == want { + return true + } + } + return false +} + +// marshalTSV encodes a slice of structs as tab-separated values with a header. +// Uses csvutil for struct-to-CSV mapping, with csv.Writer set to tab delimiter. +func marshalTSV[T any](records []T) ([]byte, error) { + var buf bytes.Buffer + w := csv.NewWriter(&buf) + w.Comma = '\t' + + enc := csvutil.NewEncoder(w) + for _, r := range records { + if err := enc.Encode(r); err != nil { + return nil, err + } + } + w.Flush() + if err := w.Error(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// normalizeV1Arch maps query arch names to canonical Go names. +func normalizeV1Arch(s string) string { + switch strings.ToLower(s) { + case "amd64": + return string(buildmeta.ArchAMD64) // "x86_64" + case "arm64": + return string(buildmeta.ArchARM64) // "aarch64" + default: + return s + } +} diff --git a/cmd/webid/v1api_test.go b/cmd/webid/v1api_test.go new file mode 100644 index 0000000..70ba4e1 --- /dev/null +++ b/cmd/webid/v1api_test.go @@ -0,0 +1,273 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" +) + +// TestV1ReleasesTSV verifies the v1 releases endpoint returns proper TSV. +func TestV1ReleasesTSV(t *testing.T) { + srv, ts := newTestServer(t) + + packages := []string{"bat", "node", "go"} + for _, pkg := range packages { + t.Run(pkg, func(t *testing.T) { + if srv.getPackage(pkg) == nil { + t.Skipf("package %s not in cache", pkg) + } + + code, body := get(t, ts, "/v1/releases/"+pkg+".tab?limit=5") + if code != 200 { + t.Fatalf("status %d: %s", code, body) + } + + lines := strings.Split(strings.TrimSpace(body), "\n") + if len(lines) < 2 { + t.Fatal("expected header + data rows") + } + + // First line should be header. + header := lines[0] + fields := strings.Split(header, "\t") + expectedHeaders := []string{ + "version", + "channel", + "lts", + "date", + "os", + "arch", + "libc", + "format", + "variants", + "download", + "filename", + } + if len(fields) != len(expectedHeaders) { + t.Fatalf("expected %d columns, got %d: %q", len(expectedHeaders), len(fields), header) + } + for i, want := range expectedHeaders { + if fields[i] != want { + t.Errorf("column[%d]: want %q, got %q", i, want, fields[i]) + } + } + + // Data rows should have same number of fields. + for i, line := range lines[1:] { + dataFields := strings.Split(line, "\t") + if len(dataFields) != len(expectedHeaders) { + t.Errorf("row[%d]: expected %d fields, got %d: %q", i, len(expectedHeaders), len(dataFields), line) + } + } + }) + } +} + +// TestV1ReleasesJSON verifies the v1 releases JSON format. +func TestV1ReleasesJSON(t *testing.T) { + srv, ts := newTestServer(t) + + pkg := "bat" + if srv.getPackage(pkg) == nil { + t.Skipf("package %s not in cache", pkg) + } + + code, body := get(t, ts, "/v1/releases/"+pkg+".json?limit=3") + if code != 200 { + t.Fatalf("status %d: %s", code, body) + } + + var releases []v1Release + if err := json.Unmarshal([]byte(body), &releases); err != nil { + t.Fatalf("decode: %v", err) + } + + if len(releases) == 0 { + t.Fatal("no releases") + } + + // v1 API uses Go-native naming — no mapping. + for i, r := range releases { + if r.Version == "" { + t.Errorf("release[%d]: empty version", i) + } + if r.Download == "" { + t.Errorf("release[%d]: empty download", i) + } + if r.Channel == "" { + t.Errorf("release[%d]: empty channel (should be 'stable' or similar)", i) + } + } +} + +// TestV1Resolve verifies the v1 resolve endpoint. +func TestV1Resolve(t *testing.T) { + srv, ts := newTestServer(t) + + pkg := "bat" + if srv.getPackage(pkg) == nil { + t.Skipf("package %s not in cache", pkg) + } + + tests := []struct { + name string + query string + wantOS string + }{ + { + name: "linux amd64", + query: "?os=linux&arch=x86_64", + wantOS: "linux", + }, + { + name: "darwin arm64", + query: "?os=darwin&arch=aarch64", + wantOS: "darwin", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + code, body := get(t, ts, "/v1/resolve/"+pkg+".json"+tt.query) + if code != 200 { + t.Fatalf("status %d: %s", code, body) + } + + var result v1ResolveResult + if err := json.Unmarshal([]byte(body), &result); err != nil { + t.Fatalf("decode: %v", err) + } + + if result.Version == "" { + t.Error("empty version") + } + if result.Download == "" { + t.Error("empty download") + } + if result.OS != tt.wantOS { + t.Errorf("os: want %q, got %q", tt.wantOS, result.OS) + } + if result.Triplet == "" { + t.Error("empty triplet") + } + + t.Logf("resolved: %s %s %s %s → %s", result.Version, result.OS, result.Arch, result.Format, result.Download) + }) + } +} + +// TestV1ResolveTSV verifies the TSV format for resolve. +func TestV1ResolveTSV(t *testing.T) { + srv, ts := newTestServer(t) + + pkg := "bat" + if srv.getPackage(pkg) == nil { + t.Skipf("package %s not in cache", pkg) + } + + code, body := get(t, ts, "/v1/resolve/"+pkg+".tab?os=linux&arch=x86_64") + if code != 200 { + t.Fatalf("status %d: %s", code, body) + } + + lines := strings.Split(strings.TrimSpace(body), "\n") + if len(lines) != 2 { + t.Fatalf("expected 2 lines (header + result), got %d", len(lines)) + } + + header := strings.Split(lines[0], "\t") + data := strings.Split(lines[1], "\t") + + if len(header) != len(data) { + t.Fatalf("header has %d fields, data has %d", len(header), len(data)) + } + + // Should have a "triplet" column. + hasTriplet := false + for _, h := range header { + if h == "triplet" { + hasTriplet = true + } + } + if !hasTriplet { + t.Error("missing triplet column in header") + } +} + +// TestV1ResolveJQ verifies jq resolves to binaries, not git. +func TestV1ResolveJQ(t *testing.T) { + srv, ts := newTestServer(t) + + pkg := "jq" + if srv.getPackage(pkg) == nil { + t.Skipf("package %s not in cache", pkg) + } + + code, body := get(t, ts, "/v1/resolve/"+pkg+".json?os=darwin&arch=aarch64") + if code != 200 { + t.Fatalf("status %d: %s", code, body) + } + + var result v1ResolveResult + if err := json.Unmarshal([]byte(body), &result); err != nil { + t.Fatalf("decode: %v", err) + } + + if result.Format == "git" { + t.Errorf("resolved to git instead of binary: %+v", result) + } + if result.OS == "" { + t.Errorf("resolved to empty OS (git asset): %+v", result) + } + + t.Logf("jq resolved: version=%s os=%s arch=%s format=%s → %s", + result.Version, result.OS, result.Arch, result.Format, result.Download) +} + +// TestV1ReleasesFilterOS verifies OS filtering works. +func TestV1ReleasesFilterOS(t *testing.T) { + srv, ts := newTestServer(t) + + pkg := "bat" + if srv.getPackage(pkg) == nil { + t.Skipf("package %s not in cache", pkg) + } + + code, body := get(t, ts, "/v1/releases/"+pkg+".json?os=darwin&limit=10") + if code != 200 { + t.Fatalf("status %d: %s", code, body) + } + + var releases []v1Release + if err := json.Unmarshal([]byte(body), &releases); err != nil { + t.Fatalf("decode: %v", err) + } + + for i, r := range releases { + if r.OS != "darwin" && r.OS != "ANYOS" && r.OS != "" { + t.Errorf("release[%d]: os=%q, expected darwin", i, r.OS) + } + } +} + +// TestV1NoQuotedFields verifies TSV output has no quoted fields. +func TestV1NoQuotedFields(t *testing.T) { + srv, ts := newTestServer(t) + + pkg := "bat" + if srv.getPackage(pkg) == nil { + t.Skipf("package %s not in cache", pkg) + } + + code, body := get(t, ts, "/v1/releases/"+pkg+".tab?limit=20") + if code != 200 { + t.Fatalf("status %d: %s", code, body) + } + + lines := strings.Split(strings.TrimSpace(body), "\n") + for i, line := range lines { + if strings.Contains(line, "\"") { + t.Errorf("line[%d] contains quotes: %s", i, line) + } + } +} diff --git a/internal/render/render.go b/internal/render/render.go index 7a1701b..69d04e5 100644 --- a/internal/render/render.go +++ b/internal/render/render.go @@ -42,8 +42,9 @@ type Params struct { Minor string Patch string Build string - GitTag string - GitBranch string + GitTag string + GitBranch string + GitCommitHash string LTS string // "true" or "false" Channel string Ext string // archive extension (e.g. "tar.gz", "zip") @@ -106,6 +107,7 @@ func Bash(tplPath, installersDir, pkgName string, p Params) (string, error) { {"WEBI_BUILD", p.Build}, {"WEBI_GIT_BRANCH", p.GitBranch}, {"WEBI_GIT_TAG", p.GitTag}, + {"WEBI_GIT_COMMIT_HASH", p.GitCommitHash}, {"WEBI_LTS", p.LTS}, {"WEBI_CHANNEL", p.Channel}, {"WEBI_EXT", p.Ext}, @@ -160,6 +162,7 @@ func PowerShell(tplPath, installersDir, pkgName string, p Params) (string, error {"WEBI_HOST", p.Host}, {"WEBI_VERSION", p.Version}, {"WEBI_GIT_TAG", p.GitTag}, + {"WEBI_GIT_COMMIT_HASH", p.GitCommitHash}, {"WEBI_PKG_URL", p.PkgURL}, {"WEBI_PKG_FILE", p.PkgFile}, {"WEBI_PKG_PATHNAME", p.PkgFile}, diff --git a/internal/storage/legacy.go b/internal/storage/legacy.go index d60b182..8cf772d 100644 --- a/internal/storage/legacy.go +++ b/internal/storage/legacy.go @@ -67,6 +67,12 @@ func (la LegacyAsset) ToAsset() Asset { arch = "x86_64" case "arm64": arch = "aarch64" + case "armv7l": + arch = "armv7" + case "armv6l": + arch = "armv6" + case "arm": + arch = "armv5" case "*": arch = "" } diff --git a/scripts/deploy-webid.sh b/scripts/deploy-webid.sh new file mode 100755 index 0000000..f8f49c5 --- /dev/null +++ b/scripts/deploy-webid.sh @@ -0,0 +1,81 @@ +#!/bin/sh +set -e +set -u + +# Build and deploy webid to a target host + +g_host="${1:-next.webi.sh}" +g_bin="webid" +g_out="agents/tmp/${g_bin}" +g_remote_bin="~/bin/${g_bin}" + +case "${g_host}" in + beta.webi.sh) g_remote_conf="~/srv/beta.webinstall.dev/installers/" ;; + next.webi.sh) g_remote_conf="~/srv/next.webinstall.dev/installers/" ;; + *) g_remote_conf="~/srv/webid/installers/" ;; +esac + +fn_build() { + b_version="$(git describe --tags --always 2> /dev/null || echo '0.0.0-dev')" + b_commit="$(git rev-parse --short HEAD)" + b_date="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + b_ldflags="-X main.version=${b_version} -X main.commit=${b_commit} -X main.date=${b_date}" + + printf 'Building %s %s %s (%s)...\n' "${g_bin}" "${b_version}" "${b_commit}" "${b_date}" + GOOS=linux GOARCH=amd64 GOAMD64=v2 go build -ldflags "${b_ldflags}" -o "${g_out}" ./cmd/webid + printf 'Built: %s\n' "${g_out}" +} + +fn_deploy() { + printf 'Stopping %s on %s...\n' "${g_bin}" "${g_host}" + ssh "${g_host}" "~/.local/bin/serviceman stop ${g_bin}" 2> /dev/null || true + + printf 'Uploading binary...\n' + scp "${g_out}" "${g_host}:${g_remote_bin}" + + printf 'Syncing install scripts and templates...\n' + rsync -av \ + --exclude='_cache' --exclude='.git' --exclude='agents' \ + --exclude='bin' --exclude='cmd' --exclude='internal' \ + --exclude='docs' --exclude='scripts' --exclude='node_modules' \ + --include='*/' --include='install.sh' --include='install.ps1' \ + --include='_webi/*.tpl.sh' --include='_webi/*.tpl.ps1' \ + --exclude='*' \ + ./ "${g_host}:${g_remote_conf}" +} + +fn_start() { + printf 'Starting %s...\n' "${g_bin}" + ssh "${g_host}" "~/.local/bin/serviceman start ${g_bin}" || { + printf 'Service not configured. Run serviceman add on the host:\n' + printf ' serviceman add --name %s \\\n' "${g_bin}" + printf ' --workdir %s -- \\\n' "${g_remote_conf}" + printf ' %s \\\n' "${g_remote_bin}" + printf ' --addr :3082 \\\n' + printf ' --legacy ~/.cache/webi/legacy \\\n' + printf ' --installers %s\n' "${g_remote_conf}" + exit 1 + } +} + +fn_verify() { + printf 'Waiting 3s for startup...\n' + sleep 3 + + printf 'Checking version...\n' + ssh "${g_host}" "${g_remote_bin} -V" + + printf 'Checking health...\n' + ssh "${g_host}" "curl -s http://localhost:3082/api/releases/bat.json | head -c 100" + printf '\n' + + printf 'Checking logs...\n' + ssh "${g_host}" "sudo journalctl -u ${g_bin} --no-pager -n 5" +} + +fn_build +fn_deploy +fn_start +fn_verify + +printf '\nDone. %s deployed to %s.\n' "${g_bin}" "${g_host}"