Compare commits

..

7 Commits

Author SHA1 Message Date
AJ ONeal
d67ac8d9f2 feat(webicached,webid): add pgstore backend and --pg flag
Adds an optional PostgreSQL storage backend for webicached and webid.
When --pg is given a DSN, the daemon writes/reads classified assets to/from
Postgres instead of the legacy fsstore. Brings in pgx/v5 as a direct dependency.
2026-05-16 21:58:59 -06:00
AJ ONeal
b79c5529e1 chore(deps): upgrade pgx v5.8.0 → v5.9.2 2026-05-16 21:58:49 -06:00
AJ ONeal
ac59e4728e feat(webid): add HTTP API server (releases, resolve, installer scripts)
Serves the HTTP API for webinstall.dev:
- GET /api/releases/{pkg}.json — classified release list from fsstore
- GET /v1/resolve/{pkg} — resolve OS/arch/version to a specific asset
- GET /api/installers/{pkg}.sh — render installer shell script
- GET /api/installers/{pkg}.ps1 — render installer PowerShell script

Git-clone packages include git_tag and git_commit_hash in responses.
UA detection infers OS/arch from User-Agent when not query-specified.
2026-05-16 21:53:05 -06:00
AJ ONeal
59b2956d60 feat(webid): add UA detection and platform resolve packages 2026-05-16 21:53:05 -06:00
AJ ONeal
75bd1a3cf9 feat(resolver): add platform/version resolver for webid 2026-05-16 21:53:05 -06:00
AJ ONeal
89e12d22f3 feat(render): add installer script renderer for webid 2026-05-16 21:53:05 -06:00
AJ ONeal
23064a6db7 fix(webicached): don't treat 0-asset packages as perpetually stale
Packages that produce no classifiable assets (e.g. mariadb-galera with
the galera asset_filter) were being refetched every batch because
!hasAssets marked them stale regardless of timestamp. The hasAssets
condition was intended for the startup case (classified from empty
rawcache), but those packages are already caught by t.IsZero() on first
run. Respect the timestamp for 0-asset results as for any other package.
2026-05-16 21:53:01 -06:00
56 changed files with 4897 additions and 1042 deletions

7
.gitignore vendored
View File

@@ -18,21 +18,15 @@ install-*.ps1
*.env
.env
!example.env
LOCAL.md
# caches
_cache/
node_modules/
# temporary & backup files
agents/
.*.sw*
*.bak
*.bak.*
tmp
*.tmp
tmp.*
*.tmp.*
# agent session files
agents/
@@ -44,4 +38,3 @@ desktop.ini
LIVE_cache
/webid
bin/
.claude/

View File

@@ -17,6 +17,7 @@ $Env:WEBI_HOST = 'https://webinstall.dev'
#$Env:PKG_NAME = node
#$Env:WEBI_VERSION = v12.16.2
#$Env:WEBI_GIT_TAG = 12.16.2
#$Env:WEBI_GIT_COMMIT_HASH =
#$Env:WEBI_PKG_URL = "https://.../node-....zip"
#$Env:WEBI_PKG_FILE = "node-v12.16.2-win-x64.zip"
#$Env:WEBI_PKG_PATHNAME = "node-v12.16.2-win-x64.zip"

View File

@@ -15,6 +15,7 @@ __bootstrap_webi() {
# TODO not sure if BUILD is the best name for this
#WEBI_BUILD=
#WEBI_GIT_TAG=
#WEBI_GIT_COMMIT_HASH=
#WEBI_LTS=
#WEBI_CHANNEL=
#WEBI_EXT=

View File

@@ -71,14 +71,6 @@ Builds.getPackage({ name: projName }).then(async function (/*projInfo*/) {
var nodeOs = os.platform();
var nodeOsRelease = os.release();
var nodeArch = os.arch();
// To make arch names compatible across all helpers
if (nodeArch === 'x64') {
nodeArch = 'amd64';
} else if (nodeArch === 'arm64') {
nodeArch = 'arm64';
}
var nodeLibc = 'libc';
if (process.platform === 'linux') {
nodeLibc = 'gnu';

View File

@@ -1,128 +0,0 @@
---
title: basecamp
homepage: https://github.com/basecamp/basecamp-cli
tagline: |
basecamp: CLI for Basecamp 3 — manage projects, todos, messages, cards, and more from the terminal.
---
To update or switch versions, run `webi basecamp@stable` (or `@v0.7`,
`@beta`, etc).
### Files
These are the files / directories that are created and/or modified with this
install:
```text
~/.config/envman/PATH.env
~/.local/bin/basecamp
~/.local/opt/basecamp-VERSION/bin/basecamp
~/.local/opt/basecamp-VERSION/completions/
```
## Cheat Sheet
> `basecamp` is the official CLI for Basecamp 3. It provides full API coverage
> for projects, todos, messages, cards, schedule, files, and more — all from the
> command line.
### How to authenticate
```sh
basecamp auth login
```
For headless environments (CI, remote servers):
```sh
basecamp auth login --device-code
```
Check auth status:
```sh
basecamp auth status
```
### How to list projects and todos
```sh
basecamp projects list --md
basecamp todos list --assignee me --in PROJECT_ID --md
```
Cross-project view of your assigned work:
```sh
basecamp assignments --md
```
### How to create and complete todos
```sh
basecamp todo "Write release notes" --in PROJECT_ID --list TODOLIST_ID --assignee me --due tomorrow
basecamp done TODO_ID
```
### How to post a message or comment
```sh
basecamp message "Sprint Update" "Shipped v2.1 to production." --in PROJECT_ID
basecamp comment RECORDING_ID "Looks good." --in PROJECT_ID
```
### How to move cards through a workflow
```sh
basecamp cards columns --in PROJECT_ID --md
basecamp cards move CARD_ID --to COLUMN_ID --in PROJECT_ID
```
### How to set up per-project defaults
Create `.basecamp/config.json` in your repo (commit it):
```json
{
"project_id": "12345",
"todolist_id": "67890"
}
```
Then trust it once:
```sh
basecamp config trust
```
After that, you can omit `--in` for most commands in that repo.
### Shell completions
Completions for bash, fish, and zsh ship with the installer. Find them at:
```text
~/.local/opt/basecamp-VERSION/completions/
```
Bash:
```sh
echo "source ~/.local/opt/basecamp-VERSION/completions/basecamp.bash" >> ~/.bashrc
```
Fish:
```sh
ln -s ~/.local/opt/basecamp-VERSION/completions/basecamp.fish ~/.config/fish/completions/
```
Zsh:
```sh
echo "fpath+=( ~/.local/opt/basecamp-VERSION/completions )" >> ~/.zshrc
```

View File

@@ -1,46 +0,0 @@
#!/usr/bin/env pwsh
$pkg_cmd_name = "basecamp"
$pkg_dst_cmd = "$Env:USERPROFILE\.local\bin\basecamp.exe"
$pkg_dst = "$pkg_dst_cmd"
$pkg_src_cmd = "$Env:USERPROFILE\.local\opt\basecamp-v$Env:WEBI_VERSION\bin\basecamp.exe"
$pkg_src_bin = "$Env:USERPROFILE\.local\opt\basecamp-v$Env:WEBI_VERSION\bin"
$pkg_src_dir = "$Env:USERPROFILE\.local\opt\basecamp-v$Env:WEBI_VERSION"
$pkg_src = "$pkg_src_cmd"
New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
$pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
Write-Output "Downloading basecamp from $Env:WEBI_PKG_URL to $pkg_download"
& curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
& Move-Item "$pkg_download.part" "$pkg_download"
}
if (!(Test-Path -Path "$pkg_src_cmd")) {
Write-Output "Installing basecamp"
Push-Location .local\tmp
Remove-Item -Path ".\basecamp-*" -Recurse -ErrorAction Ignore
Remove-Item -Path ".\basecamp.exe" -Recurse -ErrorAction Ignore
Write-Output "Unpacking $pkg_download"
& tar xf "$pkg_download"
New-Item "$pkg_src_bin" -ItemType Directory -Force | Out-Null
Move-Item -Path ".\basecamp.exe" -Destination "$pkg_src_bin"
New-Item "$pkg_src_dir\completions" -ItemType Directory -Force | Out-Null
if (Test-Path -Path ".\completions") {
Copy-Item -Path ".\completions\*" -Destination "$pkg_src_dir\completions" -Recurse
}
Pop-Location
}
Write-Output "Copying into '$pkg_dst_cmd' from '$pkg_src_cmd'"
Remove-Item -Path "$pkg_dst_cmd" -Recurse -ErrorAction Ignore | Out-Null
Copy-Item -Path "$pkg_src" -Destination "$pkg_dst" -Recurse

View File

@@ -1,44 +0,0 @@
#!/bin/sh
# shellcheck disable=SC2034
set -e
set -u
__init_basecamp() {
pkg_cmd_name="basecamp"
pkg_src_dir="$HOME/.local/opt/basecamp-v$WEBI_VERSION"
pkg_src_cmd="$pkg_src_dir/bin/basecamp"
pkg_src="$pkg_src_cmd"
pkg_dst_cmd="$HOME/.local/bin/basecamp"
pkg_dst="$pkg_dst_cmd"
pkg_install() {
mkdir -p "$(dirname "$pkg_src_cmd")"
mkdir -p "$pkg_src_dir/completions"
if test -f ./basecamp; then
mv ./basecamp "$pkg_src_cmd"
elif test -e ./basecamp-*/basecamp; then
mv ./basecamp-*/basecamp "$pkg_src_cmd"
elif test -e ./basecamp-*; then
mv ./basecamp-* "$pkg_src_cmd"
else
echo >&2 "failed to find 'basecamp' executable"
return 1
fi
if test -d ./completions; then
cp -a ./completions/. "$pkg_src_dir/completions/"
fi
}
pkg_get_current_version() {
basecamp --version 2> /dev/null |
head -n 1 |
cut -d' ' -f3
}
}
__init_basecamp

View File

@@ -1,2 +0,0 @@
github_releases = basecamp/basecamp-cli
exclude = .bundle .txt

View File

@@ -4,143 +4,73 @@ set -e
set -u
_install_brew() {
# Straight from https://brew.sh
#/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
# Straight from https://brew.sh
#/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
if test "Darwin" = "$(uname -s)"; then
needs_xcode="$(/usr/bin/xcode-select -p > /dev/null 2> /dev/null || echo "true")"
if test -n "${needs_xcode}"; then
echo ""
echo ""
echo "ERROR: Run this command to install XCode Command Line Tools first:"
echo ""
echo " xcode-select --install"
echo ""
echo "After the install, close this terminal, open a new one, and try again."
echo ""
fi
else
if ! command -v gcc > /dev/null; then
echo >&2 "Warning: to install 'gcc' et al on Linux use the built-in package manager."
echo >&2 " For example, try: sudo apt install -y build-essential"
fi
if ! command -v git > /dev/null; then
echo >&2 "Error: to install 'git' on Linux use the built-in package manager."
echo >&2 " For example, try: sudo apt install -y git"
exit 1
fi
fi
if test "Darwin" = "$(uname -s)"; then
needs_xcode="$(/usr/bin/xcode-select -p > /dev/null 2> /dev/null || echo "true")"
if test -n "${needs_xcode}"; then
echo ""
echo ""
echo "ERROR: Run this command to install XCode Command Line Tools first:"
echo ""
echo " xcode-select --install"
echo ""
echo "After the install, close this terminal, open a new one, and try again."
echo ""
fi
else
if ! command -v gcc > /dev/null; then
echo >&2 "Warning: to install 'gcc' et al on Linux use the built-in package manager."
echo >&2 " For example, try: sudo apt install -y build-essential"
fi
if ! command -v git > /dev/null; then
echo >&2 "Error: to install 'git' on Linux use the built-in package manager."
echo >&2 " For example, try: sudo apt install -y git"
exit 1
fi
fi
# From Straight from https://brew.sh
if ! test -d "$HOME/.local/opt/brew"; then
echo "Installing to '$HOME/.local/opt/brew'"
echo ""
echo "If you prefer to have brew installed to '/usr/local' cancel now and do the following:"
echo " rm -rf '$HOME/.local/opt/brew'"
# shellcheck disable=2016
echo ' /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"'
echo ""
sleep 3
#mkdir -p "$HOME/.local/opt/brew"
#curl -fsSL https://github.com/Homebrew/brew/tarball/main |
# tar xz --strip-components 1 -C "$HOME/.local/opt/brew"
git clone --depth 3 https://github.com/Homebrew/brew "$HOME/.local/opt/brew"
fi
# From Straight from https://brew.sh
if ! test -d "$HOME/.local/opt/brew"; then
echo "Installing to '$HOME/.local/opt/brew'"
echo ""
echo "If you prefer to have brew installed to '/usr/local' cancel now and do the following:"
echo " rm -rf '$HOME/.local/opt/brew'"
# shellcheck disable=2016
echo ' /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"'
echo ""
sleep 3
git clone --depth=1 https://github.com/Homebrew/brew "$HOME/.local/opt/brew"
fi
my_shellenv="$("$HOME/.local/opt/brew/bin/brew" shellenv)"
eval "${my_shellenv}"
chmod -R go-w "$(brew --prefix)/share/zsh" 2> /dev/null || true
rm -rf "$HOME/.local/bin/brew-update-service-install"
webi_download \
"$WEBI_HOST/packages/brew/brew-update-service-install" \
"$HOME/.local/bin/brew-update-service-install" \
brew-update-service-install
chmod a+x "$HOME/.local/bin/brew-update-service-install"
rm -rf "$HOME/.local/bin/brew-update-service-install"
webi_download \
"$WEBI_HOST/packages/brew/brew-update-service-install" \
"$HOME/.local/bin/brew-update-service-install" \
brew-update-service-install
chmod a+x "$HOME/.local/bin/brew-update-service-install"
webi_path_add "$HOME/.local/opt/brew/bin"
export PATH="$HOME/.local/opt/brew/bin:$PATH"
webi_path_add "$HOME/.local/opt/brew/bin"
webi_path_add "$HOME/.local/opt/brew/sbin"
echo "Updating brew..."
brew update
fn_brew_shell_integrate_bash
fn_brew_shell_integrate_zsh
fn_brew_shell_integrate_fish
webi_path_add "$HOME/.local/opt/brew/sbin"
export PATH="$HOME/.local/opt/brew/sbin:$PATH"
echo "Installed 'brew' to '$HOME/.local/opt/brew'"
echo ""
echo "If you prefer to have brew installed to '/usr/local' do the following:"
echo " mv '$HOME/.local/opt/brew' '$HOME/.local/opt/brew.$(date '+%F_%H-%M-%S').bak'"
# shellcheck disable=2016
echo ' /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"'
echo ""
echo "Installed 'brew' to '$HOME/.local/opt/brew'"
echo ""
echo "If you prefer to have brew installed to '/usr/local' do the following:"
echo " mv '$HOME/.local/opt/brew' '$HOME/.local/opt/brew.$(date '+%F_%H-%M-%S').bak'"
# shellcheck disable=2016
echo ' /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"'
echo ""
echo "To register 'brew update' as a hourly system service:"
echo " brew-update-service-install"
echo ""
}
fn_brew_shell_integrate_bash() {
if ! command -v bash > /dev/null; then
return 0
fi
if ! test -e ~/.bashrc && ! test -e ~/.bash_history; then
return 0
fi
touch -a ~/.bashrc
if grep -q 'brew shellenv' ~/.bashrc; then
return 0
fi
echo >&2 " Edit ~/.bashrc to init brew"
# shellcheck disable=SC2016
{
echo ''
echo '# Generated by Webi. Do not edit.'
echo 'eval "$('"$HOME/.local/opt/brew/bin/brew"' shellenv)"'
} >> ~/.bashrc
}
fn_brew_shell_integrate_zsh() {
if ! command -v zsh > /dev/null; then
return 0
fi
if ! test -e ~/.zshrc &&
! test -e ~/.zsh_sessions &&
! test -e ~/.zsh_history; then
return 0
fi
touch -a ~/.zshrc
if grep -q 'brew shellenv' ~/.zshrc; then
return 0
fi
echo >&2 " Edit ~/.zshrc to init brew"
# shellcheck disable=SC2016
{
echo ''
echo '# Generated by Webi. Do not edit.'
echo 'eval "$('"$HOME/.local/opt/brew/bin/brew"' shellenv)"'
} >> ~/.zshrc
}
fn_brew_shell_integrate_fish() {
if ! command -v fish > /dev/null; then
return 0
fi
mkdir -p ~/.config/fish
touch -a ~/.config/fish/config.fish
if grep -q 'brew shellenv' ~/.config/fish/config.fish; then
return 0
fi
echo >&2 " Edit ~/.config/fish/config.fish to init brew"
{
echo ''
echo '# Generated by Webi. Do not edit.'
echo "$HOME/.local/opt/brew/bin/brew shellenv | source"
} >> ~/.config/fish/config.fish
echo "To register 'brew update' as a hourly system service:"
echo " brew-update-service-install"
echo ""
}
_install_brew

View File

@@ -1,109 +0,0 @@
---
title: btop
homepage: https://github.com/aristocratos/btop
tagline: |
btop: a beautiful, interactive resource monitor
description: |
btop++ is a fast, feature-rich terminal resource monitor written in C++.
It shows real-time usage and stats for CPU, memory, disks, network, and
processes — with full mouse support, customizable themes, and an
easy-to-use menu system. The spiritual successor to bashtop and bpytop.
---
To update or switch versions, run `webi btop@stable` (or `@v1.4`, `@beta`, etc).
### Files
These are the files / directories that are created and/or modified with this
install:
```
~/.config/envman/PATH.env
~/.local/bin/btop
~/.local/opt/btop/
~/.local/opt/btop-<VERSION>/
```
## Cheat Sheet
![](https://static.linuxblog.io/wp-content/uploads/2021/11/btop.png)
> btop gives you a gorgeous, interactive view of what your system is doing —
> CPU cores, RAM, swap, disk I/O, network throughput, and a filterable process
> list — all in one terminal window.
### Launch btop
```sh
btop
```
### Navigation
| Key | Action |
| -------------- | ----------------------------------- |
| `Arrow keys` | Move selection in process list |
| `Enter` | Show detailed stats for process |
| `F` | Filter / search processes |
| `K` | Send signal (kill, SIGTERM, etc.) |
| `R` | Renice (change process priority) |
| `T` | Toggle tree / flat process view |
| `M` | Change sort field |
| `ESC` | Open settings menu |
| `Q` | Quit |
Mouse support is fully enabled by default — scroll and click anywhere in the UI.
### Change the color theme
Press `ESC` to open the menu, navigate to **Options → Color theme**, and pick
from the built-in themes (Default, TTY, Dracula, Gruvbox, and more).
Custom themes can be placed in:
```
~/.config/btop/themes/
```
### Adjust update interval
In the Options menu, set **Update interval** (in milliseconds). The default is
`2000` (2 seconds). Lower values give a more live feel; higher values reduce CPU
overhead from btop itself.
### Config file location
btop's settings are saved automatically at:
```
~/.config/btop/btop.conf
```
You can edit this file directly to set options like `update_ms`, `color_theme`,
`proc_sorting`, or `net_iface`.
### Run btop with a specific network interface shown
```sh
btop --utf-foce # force UTF-8 box drawing
btop --debug # verbose debug output to btop.log
```
Network interface selection is done interactively inside btop via the network
panel — press `B` / `N` to cycle interfaces.
### GPU monitoring (Linux x86\_64)
On Linux, btop supports Nvidia, AMD, and Intel GPUs out of the box provided the
correct drivers are installed. If wattage or GPU stats are missing, you may need
to grant extended capabilities:
```sh
# Run once after install (requires sudo)
sudo setcap cap_perfmon,cap_sys_ptrace+ep ~/.local/bin/btop
```
### See also
- [btop releases](https://github.com/aristocratos/btop/releases)
- [Theme gallery](https://github.com/aristocratos/btop/tree/main/themes)

View File

@@ -1,58 +0,0 @@
#!/bin/sh
__init_btop() {
set -e
set -u
################
# Install btop #
################
# Every package should define these 6 variables
pkg_cmd_name="btop"
pkg_dst_cmd="$HOME/.local/bin/btop"
pkg_dst="$pkg_dst_cmd"
pkg_src_cmd="$HOME/.local/opt/btop-v$WEBI_VERSION/bin/btop"
pkg_src_dir="$HOME/.local/opt/btop-v$WEBI_VERSION"
pkg_src="$pkg_src_cmd"
# pkg_install must be defined by every package
pkg_install() {
# ~/.local/opt/btop-v1.4.6/bin
mkdir -p "$(dirname "${pkg_src_cmd}")"
# mv ./btop/bin/btop ~/.local/opt/btop-v1.4.6/bin/btop
mv ./btop/bin/btop "${pkg_src_cmd}"
}
# pkg_get_current_version is recommended, but not required
pkg_get_current_version() {
# 'btop --version' has output in this format:
# btop 1.4.6 (rev abcdef0123)
# This trims it down to just the version number:
# 1.4.6
btop --version 2> /dev/null |
head -n 1 |
cut -d ' ' -f 2
}
}
fn_btop_brew_install() {
if ! command -v brew > /dev/null; then
"$HOME/.local/bin/webi" brew
export PATH="$HOME/.local/opt/brew/bin:$HOME/.local/opt/brew/sbin:$PATH"
fi
export HOMEBREW_NO_AUTO_UPDATE=1
export HOMEBREW_NO_ENV_HINTS=1
brew install btop
}
my_os=$(uname -s)
if test "Darwin" = "${my_os}" && test "$WEBI_CHANNEL" = "error"; then
fn_btop_brew_install
return 0
fi
__init_btop

View File

@@ -1,2 +0,0 @@
github_releases = aristocratos/btop
exclude = -m68k -bigsur -monterey -ventura -macos

View File

@@ -55,6 +55,7 @@ import (
"github.com/webinstall/webi-installers/internal/releases/zigdist"
"github.com/webinstall/webi-installers/internal/storage"
"github.com/webinstall/webi-installers/internal/storage/fsstore"
"github.com/webinstall/webi-installers/internal/storage/pgstore"
)
var (
@@ -78,6 +79,7 @@ type MainConfig struct {
envFile string
confDir string
cacheDir string
pgDSN string
rawDir string
token string
once bool
@@ -156,11 +158,20 @@ func main() {
cfg.token = os.Getenv("GITHUB_TOKEN")
}
fss, err := fsstore.New(cfg.cacheDir)
if err != nil {
log.Fatalf("fsstore: %v", err)
var store storage.Store
if cfg.pgDSN != "" {
pg, err := pgstore.New(context.Background(), cfg.pgDSN)
if err != nil {
log.Fatalf("pgstore: %v", err)
}
store = pg
} else {
fss, err := fsstore.New(cfg.cacheDir)
if err != nil {
log.Fatalf("fsstore: %v", err)
}
store = fss
}
var store storage.Store = fss
var auth *githubish.Auth
if cfg.token != "" {
@@ -298,6 +309,7 @@ func registerFlags(fs *flag.FlagSet, cfg *MainConfig) {
fs.StringVar(&cfg.envFile, "envfile", "", "path to .env file to load before running")
fs.StringVar(&cfg.confDir, "conf", ".", "root directory containing {pkg}/releases.conf files")
fs.StringVar(&cfg.cacheDir, "legacy", "~/.cache/webi/legacy", "legacy cache directory (fsstore root)")
fs.StringVar(&cfg.pgDSN, "pg", "", "PostgreSQL DSN (enables pgstore; mutually exclusive with -legacy)")
fs.StringVar(&cfg.rawDir, "raw", "~/.cache/webi/raw", "raw cache directory for upstream responses")
fs.StringVar(&cfg.token, "token", "", "GitHub API token (or set $GITHUB_TOKEN)")
fs.BoolVar(&cfg.once, "once", false, "run once then exit (no periodic refresh)")

146
cmd/webid/bootstrap_test.go Normal file
View File

@@ -0,0 +1,146 @@
package main
import (
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
// TestBootstrapCurlPipe verifies the /{pkg} route returns the curl-pipe bootstrap.
func TestBootstrapCurlPipe(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/bat@stable")
if code != 200 {
t.Fatalf("status %d: %s", code, body[:min(len(body), 200)])
}
// Should contain the bootstrap env vars.
if !strings.Contains(body, "WEBI_PKG=") {
t.Error("missing WEBI_PKG= in bootstrap")
}
if !strings.Contains(body, "WEBI_HOST=") {
t.Error("missing WEBI_HOST= in bootstrap")
}
if !strings.Contains(body, "WEBI_CHECKSUM=") {
t.Error("missing WEBI_CHECKSUM= in bootstrap")
}
// Should NOT contain the full installer (install.sh content).
// The bootstrap just downloads and runs webi.
if strings.Contains(body, "pkg_install()") {
t.Error("bootstrap should not contain pkg_install — that's the full installer")
}
t.Logf("bootstrap size: %d bytes", len(body))
}
// TestInstallerFull verifies /api/installers/{pkg}.sh returns the full installer.
func TestInstallerFull(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
// Use a webi-style User-Agent so the server can detect platform.
code, body := getWithUA(t, ts, "/api/installers/bat@stable.sh", "aarch64/unknown Darwin/24.2.0 libc")
if code != 200 {
t.Fatalf("status %d: %s", code, body[:min(len(body), 500)])
}
// Should contain resolved release info.
if !strings.Contains(body, "WEBI_VERSION=") {
t.Error("missing WEBI_VERSION= in installer")
}
if !strings.Contains(body, "WEBI_PKG_URL=") {
t.Error("missing WEBI_PKG_URL= in installer")
}
if !strings.Contains(body, "PKG_NAME=") {
t.Error("missing PKG_NAME= in installer")
}
// Should contain the package's install.sh content (embedded).
if !strings.Contains(body, "pkg_") {
t.Error("installer should contain pkg_ functions from install.sh")
}
t.Logf("installer size: %d bytes", len(body))
}
// TestInstallerPowerShell verifies /api/installers/{pkg}.ps1 returns a PowerShell installer.
func TestInstallerPowerShell(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "node"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := getWithUA(t, ts, "/api/installers/node@stable.ps1", "AMD64/unknown Windows/10.0.19045 msvc")
if code != 200 {
t.Fatalf("status %d: %s", code, body[:min(len(body), 500)])
}
if !strings.Contains(body, "$Env:WEBI_VERSION") {
t.Error("missing $Env:WEBI_VERSION in PS1 installer")
}
if !strings.Contains(body, "$Env:WEBI_PKG_URL") {
t.Error("missing $Env:WEBI_PKG_URL in PS1 installer")
}
if !strings.Contains(body, "$Env:PKG_NAME") {
t.Error("missing $Env:PKG_NAME in PS1 installer")
}
t.Logf("PS1 installer size: %d bytes", len(body))
}
// TestInstallerSelfHosted verifies selfhosted packages get a script without resolution.
func TestInstallerSelfHosted(t *testing.T) {
_, ts := newTestServer(t)
// ssh-utils is selfhosted — has install.sh but no releases.conf.
code, body := getWithUA(t, ts, "/api/installers/ssh-utils.sh", "aarch64/unknown Darwin/24.2.0 libc")
if code == 404 {
t.Skip("ssh-utils not available as installer")
}
if code != 200 {
t.Skipf("status %d (selfhosted may not render without cache): %s", code, body[:min(len(body), 200)])
}
t.Logf("selfhosted installer size: %d bytes", len(body))
}
// TestBootstrapUnknownPackage verifies 404 for unknown packages.
func TestBootstrapUnknownPackage(t *testing.T) {
_, ts := newTestServer(t)
code, _ := get(t, ts, "/nonexistent-package-xyz")
if code != 404 {
t.Errorf("expected 404, got %d", code)
}
}
// getWithUA fetches a URL with a custom User-Agent header.
func getWithUA(t *testing.T, ts *httptest.Server, path, ua string) (int, string) {
t.Helper()
req, err := http.NewRequest("GET", ts.URL+path, nil)
if err != nil {
t.Fatalf("new request: %v", err)
}
req.Header.Set("User-Agent", ua)
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("GET %s: %v", path, err)
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
return resp.StatusCode, string(body)
}

1000
cmd/webid/main.go Normal file

File diff suppressed because it is too large Load Diff

298
cmd/webid/main_test.go Normal file
View File

@@ -0,0 +1,298 @@
package main
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/webinstall/webi-installers/internal/resolve"
"github.com/webinstall/webi-installers/internal/storage"
"github.com/webinstall/webi-installers/internal/storage/fsstore"
)
// newTestServer creates a server backed by the real _cache directory
// and returns an httptest.Server with proper routing (so PathValue works).
func newTestServer(t *testing.T) (*server, *httptest.Server) {
t.Helper()
cacheDir := filepath.Join("..", "..", "_cache")
if _, err := os.Stat(cacheDir); err != nil {
t.Skipf("no cache dir at %s", cacheDir)
}
store, err := fsstore.New(cacheDir)
if err != nil {
t.Fatalf("fsstore: %v", err)
}
srv := &server{
store: store,
installersDir: filepath.Join("..", ".."),
packages: make(map[string]*packageCache),
}
// Load packages.
monthDir := time.Now().Format("2006-01")
dir := filepath.Join(store.Root(), monthDir)
entries, err := os.ReadDir(dir)
if err != nil {
t.Fatalf("readdir: %v", err)
}
for _, e := range entries {
if !strings.HasSuffix(e.Name(), ".json") {
continue
}
pkg := strings.TrimSuffix(e.Name(), ".json")
pd, err := store.Load(context.Background(), pkg)
if err != nil || pd == nil || len(pd.Assets) == 0 {
continue
}
pc := &packageCache{
assets: pd.Assets,
dists: assetsToDists(pd.Assets),
}
pc.catalog = resolve.Survey(pc.dists)
srv.packages[pkg] = pc
}
mux := http.NewServeMux()
mux.HandleFunc("GET /api/releases/{rest...}", srv.handleReleasesAPI)
mux.HandleFunc("GET /v1/releases/{rest...}", srv.handleV1Releases)
mux.HandleFunc("GET /v1/resolve/{rest...}", srv.handleV1Resolve)
mux.HandleFunc("GET /api/installers/{rest...}", srv.handleInstaller)
mux.HandleFunc("GET /api/debug", srv.handleDebug)
mux.HandleFunc("GET /{pkgSpec}", srv.handleBootstrap)
ts := httptest.NewServer(mux)
t.Cleanup(ts.Close)
return srv, ts
}
// get fetches a URL from the test server and returns the body.
func get(t *testing.T, ts *httptest.Server, path string) (int, string) {
t.Helper()
resp, err := http.Get(ts.URL + path)
if err != nil {
t.Fatalf("GET %s: %v", path, err)
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
return resp.StatusCode, string(body)
}
// TestLegacyJSONFormat verifies our JSON output matches the production format.
func TestLegacyJSONFormat(t *testing.T) {
srv, ts := newTestServer(t)
packages := []string{"bat", "node", "go", "jq"}
for _, pkg := range packages {
t.Run(pkg, func(t *testing.T) {
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/api/releases/"+pkg+".json?limit=5")
if code != http.StatusOK {
t.Fatalf("status %d: %s", code, body)
}
body = strings.TrimSpace(body)
// Must be a JSON array, not an object.
if !strings.HasPrefix(body, "[") {
t.Fatalf("expected JSON array, got: %.100s", body)
}
var releases []legacyRelease
if err := json.Unmarshal([]byte(body), &releases); err != nil {
t.Fatalf("decode: %v", err)
}
if len(releases) == 0 {
t.Fatal("no releases returned")
}
// Check field format conventions.
for i, r := range releases {
if strings.HasPrefix(r.Version, "v") {
t.Errorf("release[%d]: version %q should not have v prefix", i, r.Version)
}
if strings.HasPrefix(r.Ext, ".") {
t.Errorf("release[%d]: ext %q should not have . prefix", i, r.Ext)
}
if r.OS == "darwin" {
t.Errorf("release[%d]: os should be 'macos' not 'darwin'", i)
}
if r.Arch == "x86_64" {
t.Errorf("release[%d]: arch should be 'amd64' not 'x86_64'", i)
}
if r.Arch == "aarch64" {
t.Errorf("release[%d]: arch should be 'arm64' not 'aarch64'", i)
}
if r.Libc == "" {
t.Errorf("release[%d]: libc should be 'none' not empty", i)
}
if r.Download == "" {
t.Errorf("release[%d]: download URL is empty", i)
}
}
})
}
}
// TestLegacyTabFormat verifies our .tab output uses real TSV.
func TestLegacyTabFormat(t *testing.T) {
srv, ts := newTestServer(t)
packages := []string{"bat", "node", "go"}
for _, pkg := range packages {
t.Run(pkg, func(t *testing.T) {
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/api/releases/"+pkg+".tab?limit=3")
if code != http.StatusOK {
t.Fatalf("status %d: %s", code, body)
}
lines := strings.Split(strings.TrimSpace(body), "\n")
if len(lines) == 0 {
t.Fatal("no lines returned")
}
for i, line := range lines {
fields := strings.Split(line, "\t")
// Expect 11 tab-separated fields:
// version, lts, channel, date, os, arch, ext, hash, download, (empty), libc
if len(fields) != 11 {
t.Errorf("line[%d]: expected 11 tab fields, got %d: %q", i, len(fields), line)
continue
}
version := fields[0]
lts := fields[1]
ext := fields[6]
if strings.HasPrefix(version, "v") {
t.Errorf("line[%d]: version %q should not have v prefix", i, version)
}
if lts != "-" && lts != "lts" {
t.Errorf("line[%d]: lts should be '-' or 'lts', got %q", i, lts)
}
if strings.HasPrefix(ext, ".") {
t.Errorf("line[%d]: ext %q should not have . prefix", i, ext)
}
}
})
}
}
// TestLegacyJSONAgainstProduction compares our output against live production.
// Run with: WEBI_TEST_PROD=1 go test -run TestLegacyJSONAgainstProduction
func TestLegacyJSONAgainstProduction(t *testing.T) {
if os.Getenv("WEBI_TEST_PROD") == "" {
t.Skip("set WEBI_TEST_PROD=1 to compare against production")
}
srv, ts := newTestServer(t)
packages := []string{"bat", "node", "go", "jq", "rg"}
for _, pkg := range packages {
t.Run(pkg, func(t *testing.T) {
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
// Fetch from production.
prodURL := fmt.Sprintf("https://webinstall.dev/api/releases/%s.json?limit=3", pkg)
prodResp, err := http.Get(prodURL)
if err != nil {
t.Fatalf("fetch production: %v", err)
}
defer prodResp.Body.Close()
prodBody, _ := io.ReadAll(prodResp.Body)
var prodReleases []legacyRelease
if err := json.Unmarshal(prodBody, &prodReleases); err != nil {
t.Fatalf("decode production: %v\nbody: %.500s", err, string(prodBody))
}
// Fetch from local.
_, localBody := get(t, ts, "/api/releases/"+pkg+".json?limit=3")
var localReleases []legacyRelease
if err := json.Unmarshal([]byte(localBody), &localReleases); err != nil {
t.Fatalf("decode local: %v", err)
}
if len(prodReleases) == 0 || len(localReleases) == 0 {
t.Skip("empty releases")
}
// Compare the first release's format.
prod := prodReleases[0]
local := localReleases[0]
if strings.HasPrefix(local.Version, "v") != strings.HasPrefix(prod.Version, "v") {
t.Errorf("version prefix mismatch: prod=%q local=%q", prod.Version, local.Version)
}
if strings.HasPrefix(local.Ext, ".") != strings.HasPrefix(prod.Ext, ".") {
t.Errorf("ext prefix mismatch: prod=%q local=%q", prod.Ext, local.Ext)
}
if prod.OS == "macos" && local.OS == "darwin" {
t.Error("OS: prod uses 'macos', local uses 'darwin'")
}
if prod.Arch == "amd64" && local.Arch == "x86_64" {
t.Error("Arch: prod uses 'amd64', local uses 'x86_64'")
}
if prod.Arch == "arm64" && local.Arch == "aarch64" {
t.Error("Arch: prod uses 'arm64', local uses 'aarch64'")
}
t.Logf("prod[0]: version=%q os=%q arch=%q ext=%q libc=%q",
prod.Version, prod.OS, prod.Arch, prod.Ext, prod.Libc)
t.Logf("local[0]: version=%q os=%q arch=%q ext=%q libc=%q",
local.Version, local.OS, local.Arch, local.Ext, local.Libc)
})
}
}
// TestSortOrder verifies releases come back newest-first.
func TestSortOrder(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
_, body := get(t, ts, "/api/releases/"+pkg+".json?limit=20")
var releases []legacyRelease
if err := json.Unmarshal([]byte(body), &releases); err != nil {
t.Fatalf("decode: %v", err)
}
if len(releases) < 2 {
t.Skip("need at least 2 releases")
}
// First release should be newest (or equal) version.
first := releases[0].Date
last := releases[len(releases)-1].Date
if first < last {
t.Errorf("not newest-first: first=%q last=%q", first, last)
}
}
// Ensure imports are used.
var _ = storage.Asset{}

459
cmd/webid/v1api.go Normal file
View File

@@ -0,0 +1,459 @@
package main
import (
"bytes"
"encoding/csv"
"encoding/json"
"fmt"
"net/http"
"slices"
"strings"
"github.com/jszwec/csvutil"
"github.com/webinstall/webi-installers/internal/buildmeta"
"github.com/webinstall/webi-installers/internal/lexver"
"github.com/webinstall/webi-installers/internal/resolver"
"github.com/webinstall/webi-installers/internal/storage"
)
// v1Release is a single release in the new API TSV format.
// Field order matters for csvutil — it determines column order.
// Fields are designed to be easy to consume with cut/grep/sort.
type v1Release struct {
Version string `csv:"version"`
Channel string `csv:"channel"`
LTS string `csv:"lts"`
Date string `csv:"date"`
OS string `csv:"os"`
Arch string `csv:"arch"`
Libc string `csv:"libc"`
Format string `csv:"format"`
Variants string `csv:"variants"` // space-separated
Download string `csv:"download"`
Filename string `csv:"filename"`
}
// v1ResolveResult is the response for /v1/resolve/{pkg}.
type v1ResolveResult struct {
Version string `csv:"version" json:"version"`
Channel string `csv:"channel" json:"channel"`
LTS string `csv:"lts" json:"lts"`
Date string `csv:"date" json:"date"`
OS string `csv:"os" json:"os"`
Arch string `csv:"arch" json:"arch"`
Libc string `csv:"libc" json:"libc"`
Format string `csv:"format" json:"format"`
Variants string `csv:"variants" json:"variants"`
Download string `csv:"download" json:"download"`
Filename string `csv:"filename" json:"filename"`
Triplet string `csv:"triplet" json:"triplet"`
}
// handleV1Releases serves /v1/releases/{pkg}.tsv (or .json)
// with Go-native naming and TSV-first format.
//
// Query params:
//
// os — filter by OS (darwin, linux, windows)
// arch — filter by arch (aarch64, x86_64, armv7l)
// libc — filter by libc (gnu, musl, msvc)
// channel — release channel (stable, beta, rc, alpha)
// version — version prefix filter (e.g. "1.20")
// lts — if "true", only LTS releases
// format — filter by format (e.g. "tar.gz")
// variant — filter by variant (e.g. "rocm")
// limit — max results (default 1000)
func (s *server) handleV1Releases(w http.ResponseWriter, r *http.Request) {
rest := r.PathValue("rest")
pkg, version, format, err := parseReleasePath(rest)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
pc := s.getPackage(pkg)
if pc == nil {
if s.isSelfHosted(pkg) {
s.v1ServeEmpty(w, format)
return
}
http.Error(w, fmt.Sprintf("package %q not found", pkg), http.StatusNotFound)
return
}
q := r.URL.Query()
osStr := q.Get("os")
archStr := q.Get("arch")
libcStr := q.Get("libc")
channelStr := q.Get("channel")
ltsStr := q.Get("lts")
formatFilter := q.Get("format")
variantStr := q.Get("variant")
limitStr := q.Get("limit")
// Use version from URL path or query.
if version == "" {
version = q.Get("version")
}
// Handle channel selectors in version field.
switch strings.ToLower(version) {
case "stable", "latest":
version = ""
if channelStr == "" {
channelStr = "stable"
}
case "lts":
version = ""
ltsStr = "true"
case "beta", "pre", "preview":
version = ""
if channelStr == "" {
channelStr = "beta"
}
case "rc":
version = ""
if channelStr == "" {
channelStr = "rc"
}
case "alpha", "dev":
version = ""
if channelStr == "" {
channelStr = "alpha"
}
}
lts := ltsStr == "true" || ltsStr == "1"
limit := 1000
if limitStr != "" {
fmt.Sscanf(limitStr, "%d", &limit)
}
// Filter assets directly (not via resolve.Dist).
filtered := filterAssets(pc.assets, osStr, archStr, libcStr, channelStr, version, formatFilter, variantStr, lts, limit)
// Sort newest-first.
sortAssetsDescending(filtered)
switch format {
case "json":
s.v1ServeJSON(w, filtered)
case "tab":
s.v1ServeTSV(w, filtered)
default:
http.Error(w, "unsupported format: "+format+" (use .json or .tab)", http.StatusBadRequest)
}
}
// handleV1Resolve serves /v1/resolve/{pkg}.tsv (or .json)
// It resolves the single best asset for a given platform.
//
// Query params:
//
// os — target OS (required)
// arch — target arch (required)
// libc — target libc
// version — version prefix
// channel — release channel
// lts — if "true", only LTS
// format — preferred formats (comma-separated, in preference order)
// variant — preferred variant
func (s *server) handleV1Resolve(w http.ResponseWriter, r *http.Request) {
rest := r.PathValue("rest")
pkg, version, format, err := parseReleasePath(rest)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
pc := s.getPackage(pkg)
if pc == nil {
http.Error(w, fmt.Sprintf("package %q not found", pkg), http.StatusNotFound)
return
}
q := r.URL.Query()
osStr := q.Get("os")
archStr := q.Get("arch")
libcStr := q.Get("libc")
channelStr := q.Get("channel")
ltsStr := q.Get("lts")
formatsStr := q.Get("format")
variantStr := q.Get("variant")
if version == "" {
version = q.Get("version")
}
// Handle channel selectors in version field.
switch strings.ToLower(version) {
case "stable", "latest":
version = ""
if channelStr == "" {
channelStr = "stable"
}
case "lts":
version = ""
ltsStr = "true"
case "beta", "pre", "preview":
version = ""
if channelStr == "" {
channelStr = "beta"
}
case "rc":
version = ""
if channelStr == "" {
channelStr = "rc"
}
case "alpha", "dev":
version = ""
if channelStr == "" {
channelStr = "alpha"
}
}
lts := ltsStr == "true" || ltsStr == "1"
var formats []string
if formatsStr != "" {
formats = strings.Split(formatsStr, ",")
}
req := resolver.Request{
OS: osStr,
Arch: archStr,
Libc: libcStr,
Version: version,
Channel: channelStr,
LTS: lts,
Formats: formats,
Variant: variantStr,
}
res, err := resolver.Resolve(pc.assets, req)
if err != nil {
http.Error(w, fmt.Sprintf("no match for %s: %v", pkg, err), http.StatusNotFound)
return
}
result := assetToV1Resolve(res)
switch format {
case "json":
w.Header().Set("Content-Type", "application/json")
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
enc.Encode(result)
case "tab":
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
data, err := marshalTSV([]v1ResolveResult{result})
if err != nil {
http.Error(w, "encode error: "+err.Error(), http.StatusInternalServerError)
return
}
w.Write(data)
default:
http.Error(w, "unsupported format: "+format, http.StatusBadRequest)
}
}
func assetToV1Release(a storage.Asset) v1Release {
lts := "-"
if a.LTS {
lts = "lts"
}
channel := a.Channel
if channel == "" {
channel = "stable"
}
libc := a.Libc
if libc == "" {
libc = "-"
}
return v1Release{
Version: a.Version,
Channel: channel,
LTS: lts,
Date: a.Date,
OS: a.OS,
Arch: a.Arch,
Libc: libc,
Format: a.Format,
Variants: strings.Join(a.Variants, " "),
Download: a.Download,
Filename: a.Filename,
}
}
func assetToV1Resolve(res resolver.Result) v1ResolveResult {
a := res.Asset
lts := "-"
if a.LTS {
lts = "lts"
}
channel := a.Channel
if channel == "" {
channel = "stable"
}
libc := a.Libc
if libc == "" {
libc = "-"
}
return v1ResolveResult{
Version: a.Version,
Channel: channel,
LTS: lts,
Date: a.Date,
OS: a.OS,
Arch: a.Arch,
Libc: libc,
Format: a.Format,
Variants: strings.Join(a.Variants, " "),
Download: a.Download,
Filename: a.Filename,
Triplet: res.Triplet,
}
}
func (s *server) v1ServeTSV(w http.ResponseWriter, assets []storage.Asset) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
releases := make([]v1Release, len(assets))
for i, a := range assets {
releases[i] = assetToV1Release(a)
}
data, err := marshalTSV(releases)
if err != nil {
http.Error(w, "encode error: "+err.Error(), http.StatusInternalServerError)
return
}
w.Write(data)
}
func (s *server) v1ServeJSON(w http.ResponseWriter, assets []storage.Asset) {
w.Header().Set("Content-Type", "application/json")
releases := make([]v1Release, len(assets))
for i, a := range assets {
releases[i] = assetToV1Release(a)
}
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
enc.Encode(releases)
}
func (s *server) v1ServeEmpty(w http.ResponseWriter, format string) {
switch format {
case "json":
w.Header().Set("Content-Type", "application/json")
w.Write([]byte("[]\n"))
case "tab":
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
// Just the header.
data, _ := marshalTSV([]v1Release{})
w.Write(data)
}
}
// filterAssets filters storage.Asset slices directly.
func filterAssets(assets []storage.Asset, osStr, archStr, libcStr, channel, version, formatFilter, variant string, lts bool, limit int) []storage.Asset {
var result []storage.Asset
for _, a := range assets {
if osStr != "" && a.OS != osStr && a.OS != "ANYOS" && a.OS != "" {
continue
}
if archStr != "" && a.Arch != archStr && a.Arch != "ANYARCH" && a.Arch != "" {
continue
}
if libcStr != "" && a.Libc != "" && a.Libc != "none" && a.Libc != libcStr {
continue
}
if lts && !a.LTS {
continue
}
if channel != "" && a.Channel != channel {
continue
}
if version != "" {
v := strings.TrimPrefix(a.Version, "v")
vq := strings.TrimPrefix(version, "v")
if !strings.HasPrefix(v, vq) {
continue
}
}
if formatFilter != "" && !strings.Contains(a.Format, formatFilter) {
continue
}
if variant != "" {
if !hasVariant(a.Variants, variant) {
continue
}
}
result = append(result, a)
if len(result) >= limit {
break
}
}
return result
}
// sortAssetsDescending sorts assets newest-first by version.
func sortAssetsDescending(assets []storage.Asset) {
slices.SortStableFunc(assets, func(a, b storage.Asset) int {
va := lexver.Parse(strings.TrimPrefix(a.Version, "v"))
vb := lexver.Parse(strings.TrimPrefix(b.Version, "v"))
return lexver.Compare(vb, va) // descending
})
}
// hasVariant checks if the variant list contains the wanted variant.
// This is a copy of resolver.hasVariant since it's unexported.
func hasVariant(variants []string, want string) bool {
for _, v := range variants {
if v == want {
return true
}
}
return false
}
// marshalTSV encodes a slice of structs as tab-separated values with a header.
// Uses csvutil for struct-to-CSV mapping, with csv.Writer set to tab delimiter.
func marshalTSV[T any](records []T) ([]byte, error) {
var buf bytes.Buffer
w := csv.NewWriter(&buf)
w.Comma = '\t'
enc := csvutil.NewEncoder(w)
for _, r := range records {
if err := enc.Encode(r); err != nil {
return nil, err
}
}
w.Flush()
if err := w.Error(); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// normalizeV1Arch maps query arch names to canonical Go names.
func normalizeV1Arch(s string) string {
switch strings.ToLower(s) {
case "amd64":
return string(buildmeta.ArchAMD64) // "x86_64"
case "arm64":
return string(buildmeta.ArchARM64) // "aarch64"
default:
return s
}
}

273
cmd/webid/v1api_test.go Normal file
View File

@@ -0,0 +1,273 @@
package main
import (
"encoding/json"
"strings"
"testing"
)
// TestV1ReleasesTSV verifies the v1 releases endpoint returns proper TSV.
func TestV1ReleasesTSV(t *testing.T) {
srv, ts := newTestServer(t)
packages := []string{"bat", "node", "go"}
for _, pkg := range packages {
t.Run(pkg, func(t *testing.T) {
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/v1/releases/"+pkg+".tab?limit=5")
if code != 200 {
t.Fatalf("status %d: %s", code, body)
}
lines := strings.Split(strings.TrimSpace(body), "\n")
if len(lines) < 2 {
t.Fatal("expected header + data rows")
}
// First line should be header.
header := lines[0]
fields := strings.Split(header, "\t")
expectedHeaders := []string{
"version",
"channel",
"lts",
"date",
"os",
"arch",
"libc",
"format",
"variants",
"download",
"filename",
}
if len(fields) != len(expectedHeaders) {
t.Fatalf("expected %d columns, got %d: %q", len(expectedHeaders), len(fields), header)
}
for i, want := range expectedHeaders {
if fields[i] != want {
t.Errorf("column[%d]: want %q, got %q", i, want, fields[i])
}
}
// Data rows should have same number of fields.
for i, line := range lines[1:] {
dataFields := strings.Split(line, "\t")
if len(dataFields) != len(expectedHeaders) {
t.Errorf("row[%d]: expected %d fields, got %d: %q", i, len(expectedHeaders), len(dataFields), line)
}
}
})
}
}
// TestV1ReleasesJSON verifies the v1 releases JSON format.
func TestV1ReleasesJSON(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/v1/releases/"+pkg+".json?limit=3")
if code != 200 {
t.Fatalf("status %d: %s", code, body)
}
var releases []v1Release
if err := json.Unmarshal([]byte(body), &releases); err != nil {
t.Fatalf("decode: %v", err)
}
if len(releases) == 0 {
t.Fatal("no releases")
}
// v1 API uses Go-native naming — no mapping.
for i, r := range releases {
if r.Version == "" {
t.Errorf("release[%d]: empty version", i)
}
if r.Download == "" {
t.Errorf("release[%d]: empty download", i)
}
if r.Channel == "" {
t.Errorf("release[%d]: empty channel (should be 'stable' or similar)", i)
}
}
}
// TestV1Resolve verifies the v1 resolve endpoint.
func TestV1Resolve(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
tests := []struct {
name string
query string
wantOS string
}{
{
name: "linux amd64",
query: "?os=linux&arch=x86_64",
wantOS: "linux",
},
{
name: "darwin arm64",
query: "?os=darwin&arch=aarch64",
wantOS: "darwin",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
code, body := get(t, ts, "/v1/resolve/"+pkg+".json"+tt.query)
if code != 200 {
t.Fatalf("status %d: %s", code, body)
}
var result v1ResolveResult
if err := json.Unmarshal([]byte(body), &result); err != nil {
t.Fatalf("decode: %v", err)
}
if result.Version == "" {
t.Error("empty version")
}
if result.Download == "" {
t.Error("empty download")
}
if result.OS != tt.wantOS {
t.Errorf("os: want %q, got %q", tt.wantOS, result.OS)
}
if result.Triplet == "" {
t.Error("empty triplet")
}
t.Logf("resolved: %s %s %s %s → %s", result.Version, result.OS, result.Arch, result.Format, result.Download)
})
}
}
// TestV1ResolveTSV verifies the TSV format for resolve.
func TestV1ResolveTSV(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/v1/resolve/"+pkg+".tab?os=linux&arch=x86_64")
if code != 200 {
t.Fatalf("status %d: %s", code, body)
}
lines := strings.Split(strings.TrimSpace(body), "\n")
if len(lines) != 2 {
t.Fatalf("expected 2 lines (header + result), got %d", len(lines))
}
header := strings.Split(lines[0], "\t")
data := strings.Split(lines[1], "\t")
if len(header) != len(data) {
t.Fatalf("header has %d fields, data has %d", len(header), len(data))
}
// Should have a "triplet" column.
hasTriplet := false
for _, h := range header {
if h == "triplet" {
hasTriplet = true
}
}
if !hasTriplet {
t.Error("missing triplet column in header")
}
}
// TestV1ResolveJQ verifies jq resolves to binaries, not git.
func TestV1ResolveJQ(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "jq"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/v1/resolve/"+pkg+".json?os=darwin&arch=aarch64")
if code != 200 {
t.Fatalf("status %d: %s", code, body)
}
var result v1ResolveResult
if err := json.Unmarshal([]byte(body), &result); err != nil {
t.Fatalf("decode: %v", err)
}
if result.Format == "git" {
t.Errorf("resolved to git instead of binary: %+v", result)
}
if result.OS == "" {
t.Errorf("resolved to empty OS (git asset): %+v", result)
}
t.Logf("jq resolved: version=%s os=%s arch=%s format=%s → %s",
result.Version, result.OS, result.Arch, result.Format, result.Download)
}
// TestV1ReleasesFilterOS verifies OS filtering works.
func TestV1ReleasesFilterOS(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/v1/releases/"+pkg+".json?os=darwin&limit=10")
if code != 200 {
t.Fatalf("status %d: %s", code, body)
}
var releases []v1Release
if err := json.Unmarshal([]byte(body), &releases); err != nil {
t.Fatalf("decode: %v", err)
}
for i, r := range releases {
if r.OS != "darwin" && r.OS != "ANYOS" && r.OS != "" {
t.Errorf("release[%d]: os=%q, expected darwin", i, r.OS)
}
}
}
// TestV1NoQuotedFields verifies TSV output has no quoted fields.
func TestV1NoQuotedFields(t *testing.T) {
srv, ts := newTestServer(t)
pkg := "bat"
if srv.getPackage(pkg) == nil {
t.Skipf("package %s not in cache", pkg)
}
code, body := get(t, ts, "/v1/releases/"+pkg+".tab?limit=20")
if code != 200 {
t.Fatalf("status %d: %s", code, body)
}
lines := strings.Split(strings.TrimSpace(body), "\n")
for i, line := range lines {
if strings.Contains(line, "\"") {
t.Errorf("line[%d] contains quotes: %s", i, line)
}
}
}

View File

@@ -3,19 +3,23 @@ set -e
set -u
if command -v fish > /dev/null; then
if ! test -e ~/.config/fish/config.fish; then
mkdir -p ~/.config/fish
touch ~/.config/fish/config.fish
chmod 0600 ~/.config/fish/config.fish
fi
if [ ! -e ~/.config/fish/config.fish ]; then
mkdir -p ~/.config/fish
touch ~/.config/fish/config.fish
chmod 0600 ~/.config/fish/config.fish
fi
fi
if [ "Darwin" != "$(uname -s)" ]; then
echo "No fish installer for Linux yet. Try this instead:"
echo " sudo apt install -y fish"
exit 1
fi
################
# Install fish #
################
my_os=$(uname -s)
# Every package should define these 6 variables
# shellcheck disable=2034
pkg_cmd_name="fish"
@@ -30,109 +34,70 @@ pkg_src_dir="$HOME/.local/opt/fish-v$WEBI_VERSION"
# shellcheck disable=2034
pkg_src="$pkg_src_cmd"
if test "Darwin" = "${my_os}"; then
pkg_src_cmd="/Applications/fish.app/Contents/Resources/base/usr/local/bin/fish"
# shellcheck disable=2034
pkg_src="${pkg_src_cmd}"
fi
_linux_post_install() {
if ! test -e "$HOME/.local/bin/fish"; then
return 0
fi
echo ""
echo "To set fish as your default shell, run:"
echo " chsh -s $HOME/.local/bin/fish"
echo ""
}
# pkg_install must be defined by every package
_macos_post_install() {
if ! test -e "$HOME/.local/bin/fish"; then
return 0
fi
if ! [ -e "$HOME/.local/bin/fish" ]; then
return 0
fi
echo ""
echo "Trying to set fish as the default shell..."
echo ""
# stop the caching of preferences
killall cfprefsd
echo ""
echo "Trying to set fish as the default shell..."
echo ""
# stop the caching of preferences
killall cfprefsd
# Set default Terminal.app shell to fish
defaults write com.apple.Terminal "Shell" -string "$HOME/.local/bin/fish"
echo "To set 'fish' as the default Terminal.app shell:"
echo " Terminal > Preferences > General > Shells open with:"
echo " $HOME/.local/bin/fish"
echo ""
# Set default Terminal.app shell to fish
defaults write com.apple.Terminal "Shell" -string "$HOME/.local/bin/fish"
echo "To set 'fish' as the default Terminal.app shell:"
echo " Terminal > Preferences > General > Shells open with:"
echo " $HOME/.local/bin/fish"
echo ""
# Set default iTerm2 shell to fish
if test -e "$HOME/Library/Preferences/com.googlecode.iterm2.plist"; then
/usr/libexec/PlistBuddy \
-c "SET ':New Bookmarks:0:Custom Command' 'Custom Shell'" \
"$HOME/Library/Preferences/com.googlecode.iterm2.plist"
/usr/libexec/PlistBuddy \
-c "SET ':New Bookmarks:0:Command' $HOME/.local/bin/fish" \
"$HOME/Library/Preferences/com.googlecode.iterm2.plist"
echo "To set 'fish' as the default iTerm2 shell:"
echo " iTerm2 > Preferences > Profiles > General > Command >"
echo " Custom Shell: $HOME/.local/bin/fish"
echo ""
fi
# Set default iTerm2 shell to fish
if [ -e "$HOME/Library/Preferences/com.googlecode.iterm2.plist" ]; then
/usr/libexec/PlistBuddy \
-c "SET ':New Bookmarks:0:Custom Command' 'Custom Shell'" \
"$HOME/Library/Preferences/com.googlecode.iterm2.plist"
/usr/libexec/PlistBuddy \
-c "SET ':New Bookmarks:0:Command' $HOME/.local/bin/fish" \
"$HOME/Library/Preferences/com.googlecode.iterm2.plist"
echo "To set 'fish' as the default iTerm2 shell:"
echo " iTerm2 > Preferences > Profiles > General > Command >"
echo " Custom Shell: $HOME/.local/bin/fish"
echo ""
fi
killall cfprefsd
killall cfprefsd
}
# always try to reset the default shells
if test "Darwin" = "${my_os}"; then
_macos_post_install
fi
_macos_post_install
pkg_install() {
if test "Darwin" = "${my_os}"; then
rm -rf "/Applications/fish-v${WEBI_VERSION}.app"
mv -f fish*.app "/Applications/fish-v${WEBI_VERSION}.app"
rm -rf /Applications/fish.app
mv "/Applications/fish-v${WEBI_VERSION}.app" "/Applications/fish.app"
return 0
fi
mv fish.app/Contents/Resources/base/usr/local "$HOME/.local/opt/fish-v${WEBI_VERSION}"
mkdir -p "$pkg_src_dir/bin"
mv fish "$pkg_src_dir/bin/"
}
pkg_link() {
if test "Darwin" = "${my_os}"; then
mkdir -p "$HOME/.local/bin"
ln -sf /Applications/fish.app/Contents/Resources/base/usr/local/bin/fish "$HOME/.local/bin/fish"
return 0
fi
rm -rf "$pkg_dst_cmd"
ln -s "$pkg_src_cmd" "$pkg_dst_cmd"
}
pkg_post_install() {
# don't skip what webi would do automatically
webi_post_install
# don't skip what webi would do automatically
webi_post_install
# try again to update default shells, now that all files should exist
if test "Darwin" = "${my_os}"; then
_macos_post_install
else
_linux_post_install
fi
if ! test -e ~/.config/fish/config.fish; then
mkdir -p ~/.config/fish
touch ~/.config/fish/config.fish
chmod 0600 ~/.config/fish/config.fish
fi
# try again to update default shells, now that all files should exist
_macos_post_install
if [ ! -e ~/.config/fish/config.fish ]; then
mkdir -p ~/.config/fish
touch ~/.config/fish/config.fish
chmod 0600 ~/.config/fish/config.fish
fi
}
# pkg_get_current_version is recommended, but (soon) not required
pkg_get_current_version() {
# 'fish --version' has output in this format:
# fish, version 4.3.3
# This trims it down to just the version number:
# 4.3.3
fish --version 2> /dev/null | head -n 1 | cut -d ' ' -f 3
# 'fish --version' has output in this format:
# fish, version 3.1.2
# This trims it down to just the version number:
# 3.1.2
fish --version 2> /dev/null | head -n 1 | cut -d ' ' -f 3
}

View File

@@ -23,7 +23,7 @@ __init_gh() {
# ~/.local/opt/gh-v0.99.9/bin
mkdir -p "$(dirname "$pkg_src_cmd")"
# mv ./gh_*/bin/gh ~/.local/opt/gh-v0.99.9/bin/gh
# mv ./gh-*/gh ~/.local/opt/gh-v0.99.9/bin/gh
mv ./"$pkg_cmd_name"*/bin/gh "$pkg_src_cmd"
}

View File

@@ -1,3 +1,2 @@
github_releases = go-gitea/gitea
exclude = -src- -docs-
variants = gogit

15
go.mod
View File

@@ -2,4 +2,17 @@ module github.com/webinstall/webi-installers
go 1.26.1
require github.com/joho/godotenv v1.5.1
require (
github.com/jackc/pgx/v5 v5.9.2
github.com/joho/godotenv v1.5.1
github.com/jszwec/csvutil v1.10.0
github.com/therootcompany/golib/http/middleware/v2 v2.0.1
)
require (
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
golang.org/x/sync v0.17.0 // indirect
golang.org/x/text v0.29.0 // indirect
)

30
go.sum
View File

@@ -1,2 +1,32 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/jszwec/csvutil v1.10.0 h1:upMDUxhQKqZ5ZDCs/wy+8Kib8rZR8I8lOR34yJkdqhI=
github.com/jszwec/csvutil v1.10.0/go.mod h1:/E4ONrmGkwmWsk9ae9jpXnv9QT8pLHEPcCirMFhxG9I=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/therootcompany/golib/http/middleware/v2 v2.0.1 h1:VNKpHcwyEW7cMct7/eO4fyrxwIQk2ycb6juVXSPs2Sk=
github.com/therootcompany/golib/http/middleware/v2 v2.0.1/go.mod h1:g5gb9qBidw74nW6/mwIauTKMpOKchiN2l0gt5qzJ2aQ=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -23,7 +23,7 @@ __init_goreleaser() {
# ~/.local/opt/goreleaser-v1.21.2/bin
mkdir -p "$(dirname "$pkg_src_cmd")"
# mv ./goreleaser ~/.local/opt/goreleaser-v1.21.2/bin/goreleaser
# mv ./goreleaser-*/goreleaser ~/.local/opt/goreleaser-v1.21.2/bin/goreleaser
mv ./goreleaser "$pkg_src_cmd"
}

View File

@@ -12,7 +12,6 @@ package classify
import (
"path"
"regexp"
"slices"
"strings"
"github.com/webinstall/webi-installers/internal/buildmeta"
@@ -269,11 +268,16 @@ func IsMetaAsset(name string) bool {
return true
}
}
return slices.Contains([]string{
for _, exact := range []string{
"install.sh",
"install.ps1",
"compat.json",
"b3sums",
"dist-manifest.json",
}, lower)
} {
if lower == exact {
return true
}
}
return false
}

View File

@@ -25,6 +25,7 @@ import (
"github.com/webinstall/webi-installers/internal/releases/chromedist"
"github.com/webinstall/webi-installers/internal/releases/cmake"
"github.com/webinstall/webi-installers/internal/releases/fish"
"github.com/webinstall/webi-installers/internal/releases/gitea"
"github.com/webinstall/webi-installers/internal/releases/flutterdist"
"github.com/webinstall/webi-installers/internal/releases/git"
"github.com/webinstall/webi-installers/internal/releases/golang"
@@ -40,7 +41,7 @@ import (
"github.com/webinstall/webi-installers/internal/releases/postgres"
"github.com/webinstall/webi-installers/internal/releases/sass"
"github.com/webinstall/webi-installers/internal/releases/servicemandist"
sttrdist "github.com/webinstall/webi-installers/internal/releases/sttr"
"github.com/webinstall/webi-installers/internal/releases/sttr"
"github.com/webinstall/webi-installers/internal/releases/uuidv7"
"github.com/webinstall/webi-installers/internal/releases/watchexec"
"github.com/webinstall/webi-installers/internal/releases/xcaddy"
@@ -97,7 +98,7 @@ func Package(pkg string, conf *installerconf.Conf, d *rawcache.Dir, gitTagDir *r
assets = append(assets, gitAssets...)
}
TagVariants(pkg, conf.Variants, assets)
TagVariants(pkg, assets)
assets = expandUniversal(assets)
NormalizeVersions(pkg, assets)
processGitTagHEAD(assets)
@@ -193,19 +194,9 @@ func NormalizeVersions(pkg string, assets []storage.Asset) {
}
}
// TagVariants applies variant tags to classified assets.
// conf variants (from releases.conf) are applied first: each variant is
// matched as a case-folded substring of the filename. Package-specific
// logic runs after for cases that require more than a substring check.
func TagVariants(pkg string, confVariants []string, assets []storage.Asset) {
for i := range assets {
lower := strings.ToLower(assets[i].Filename)
for _, v := range confVariants {
if strings.Contains(lower, strings.ToLower(v)) {
assets[i].Variants = append(assets[i].Variants, v)
}
}
}
// TagVariants applies package-specific variant tags to classified assets.
// Each case delegates to a per-installer package under internal/releases/.
func TagVariants(pkg string, assets []storage.Asset) {
switch pkg {
case "atomicparsley":
atomicparsleydist.TagVariants(assets)
@@ -219,6 +210,8 @@ func TagVariants(pkg string, confVariants []string, assets []storage.Asset) {
flutterdist.TagVariants(assets)
case "git":
gitdist.TagVariants(assets)
case "gitea":
gitea.TagVariants(assets)
case "lsd":
lsddist.TagVariants(assets)
case "node":

View File

@@ -21,6 +21,9 @@ import (
func TagVariants(assets []storage.Asset) {
for i := range assets {
lower := strings.ToLower(assets[i].Filename)
if strings.Contains(lower, "-profile") {
assets[i].Variants = append(assets[i].Variants, "profile")
}
if assets[i].Arch == "x86_64" {
if strings.Contains(lower, "-baseline") {
// Baseline is plain x86_64 — strip the suffix from

View File

@@ -1,10 +1,14 @@
// Package lsd provides variant tagging for lsd (LSDeluxe) releases.
//
// lsd publishes .deb packages alongside the standard archives.
// msvc builds are excluded via releases.conf variants.
// lsd publishes .deb packages and windows-msvc builds alongside
// the standard archives.
package lsddist
import "github.com/webinstall/webi-installers/internal/storage"
import (
"strings"
"github.com/webinstall/webi-installers/internal/storage"
)
// TagVariants tags lsd-specific build variants.
func TagVariants(assets []storage.Asset) {
@@ -12,5 +16,8 @@ func TagVariants(assets []storage.Asset) {
if assets[i].Format == ".deb" {
assets[i].Variants = append(assets[i].Variants, "deb")
}
if strings.Contains(strings.ToLower(assets[i].Filename), "-msvc") {
assets[i].Variants = append(assets[i].Variants, "msvc")
}
}
}

View File

@@ -1,4 +1,7 @@
// Package ollama provides variant tagging for Ollama releases.
//
// Ollama publishes GPU accelerator builds: -rocm (AMD), -jetpack5
// and -jetpack6 (NVIDIA Jetson).
package ollamadist
import (
@@ -8,10 +11,14 @@ import (
)
// TagVariants tags ollama-specific build variants.
// Suffix variants (mlx, rocm, jetpack5, jetpack6) are handled by the
// conf-driven loop in classifypkg.TagVariants; this handles the rest.
func TagVariants(assets []storage.Asset) {
for i := range assets {
lower := strings.ToLower(assets[i].Filename)
for _, v := range []string{"rocm", "jetpack5", "jetpack6"} {
if strings.Contains(lower, "-"+v) {
assets[i].Variants = append(assets[i].Variants, v)
}
}
// Ollama-darwin.zip (capital O) is the macOS .app bundle.
// Installable by Go (extract .app), but not in legacy cache.
if strings.HasPrefix(assets[i].Filename, "Ollama-") {

View File

@@ -23,8 +23,15 @@ var winVersionRe = regexp.MustCompile(`(?i)-win(?:7|8|81|10|2008|2012|2016)`)
func TagVariants(assets []storage.Asset) {
for i := range assets {
lower := strings.ToLower(assets[i].Filename)
if winVersionRe.MatchString(lower) {
switch {
case strings.Contains(lower, "-fxdependentwindesktop"):
assets[i].Variants = append(assets[i].Variants, "fxdependentWinDesktop")
case strings.Contains(lower, "-fxdependent"):
assets[i].Variants = append(assets[i].Variants, "fxdependent")
case winVersionRe.MatchString(lower):
assets[i].Variants = append(assets[i].Variants, "win-version-specific")
case strings.HasSuffix(lower, ".appimage"):
assets[i].Variants = append(assets[i].Variants, "appimage")
}
}
}

View File

@@ -1,8 +1,14 @@
// Package sttr provides variant tagging for sttr releases.
//
// sttr_Darwin_all.tar.gz is the only macOS release — a universal binary
// with no arch token. Mark it universal2 so expandUniversal serves it
// to both arm64 and amd64 Mac users.
// sttr ships a darwin_all (universal macOS) archive alongside per-arch builds.
// These universal archives have no arch in the filename — Go classifies them as
// os="darwin", arch="" which the Node builds-cacher rejects with FORMAT CHANGE
// (Node's classifier extracts a different arch from "all"). Production Node
// also stores these as os="", arch="" (unroutable).
//
// .sbom.json files are software bill-of-materials metadata — not installable
// archives. They pass through the format filter (ext="") but should not be
// served.
package sttrdist
import (
@@ -11,11 +17,20 @@ import (
"github.com/webinstall/webi-installers/internal/storage"
)
// TagVariants tags sttr-specific build variants.
// TagVariants tags sttr-specific build variants for exclusion from legacy export.
func TagVariants(assets []storage.Asset) {
for i := range assets {
if strings.Contains(strings.ToLower(assets[i].Filename), "darwin_all") {
assets[i].Arch = "universal2"
lower := strings.ToLower(assets[i].Filename)
// darwin_all / Darwin_all: universal macOS archive with no arch info.
// Node's classifier extracts a different result → FORMAT CHANGE.
// Production LIVE_cache has these as os="", arch="" (unroutable).
if strings.Contains(lower, "darwin_all") {
assets[i].Variants = append(assets[i].Variants, "universal-all")
continue
}
// .sbom.json: software bill-of-materials, not an installable archive.
if strings.HasSuffix(lower, ".sbom.json") {
assets[i].Variants = append(assets[i].Variants, "metadata")
}
}
}

267
internal/render/render.go Normal file
View File

@@ -0,0 +1,267 @@
// Package render generates installer scripts by injecting release
// metadata into the package-install template.
//
// The template uses shell-style variable markers:
//
// #WEBI_VERSION= → WEBI_VERSION='1.2.3'
// #export WEBI_PKG_URL= → export WEBI_PKG_URL='https://...'
//
// The package's install.sh is injected at the {{ installer }} marker.
package render
import (
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
)
// Params holds all the values to inject into the installer template.
type Params struct {
// Host is the base URL of the webi server (e.g. "https://webinstall.dev").
Host string
// Checksum is the webi.sh bootstrap script checksum (first 8 hex chars of SHA-1).
Checksum string
// Package name (e.g. "bat", "node").
PkgName string
// Tag is the version selector from the URL (e.g. "20", "stable", "").
Tag string
// OS, Arch, Libc are the detected platform strings.
OS string
Arch string
Libc string
// Resolved release info.
Version string
Major string
Minor string
Patch string
Build string
GitTag string
GitBranch string
GitCommitHash string
LTS string // "true" or "false"
Channel string
Ext string // archive extension (e.g. "tar.gz", "zip")
Formats string // comma-separated format list
// Download info.
PkgURL string // download URL
PkgFile string // filename
// Releases API URL for this request.
ReleasesURL string
// CSV line for WEBI_CSV.
CSV string
// Package catalog info.
PkgStable string
PkgLatest string
PkgOSes string // space-separated
PkgArches string // space-separated
PkgLibcs string // space-separated
PkgFormats string // space-separated
}
// Bash renders a complete bash installer script by injecting params
// into the template and splicing in the package's install.sh.
func Bash(tplPath, installersDir, pkgName string, p Params) (string, error) {
tpl, err := os.ReadFile(tplPath)
if err != nil {
return "", fmt.Errorf("render: read template: %w", err)
}
// Read the package's install.sh.
installPath := filepath.Join(installersDir, pkgName, "install.sh")
installSh, err := os.ReadFile(installPath)
if err != nil {
return "", fmt.Errorf("render: read %s/install.sh: %w", pkgName, err)
}
text := string(tpl)
// Inject environment variables.
vars := []struct {
name string
value string
}{
{"WEBI_CHECKSUM", p.Checksum},
{"WEBI_PKG", p.PkgName + "@" + p.Tag},
{"WEBI_HOST", p.Host},
{"WEBI_OS", p.OS},
{"WEBI_ARCH", p.Arch},
{"WEBI_LIBC", p.Libc},
{"WEBI_TAG", p.Tag},
{"WEBI_RELEASES", p.ReleasesURL},
{"WEBI_CSV", p.CSV},
{"WEBI_VERSION", p.Version},
{"WEBI_MAJOR", p.Major},
{"WEBI_MINOR", p.Minor},
{"WEBI_PATCH", p.Patch},
{"WEBI_BUILD", p.Build},
{"WEBI_GIT_BRANCH", p.GitBranch},
{"WEBI_GIT_TAG", p.GitTag},
{"WEBI_GIT_COMMIT_HASH", p.GitCommitHash},
{"WEBI_LTS", p.LTS},
{"WEBI_CHANNEL", p.Channel},
{"WEBI_EXT", p.Ext},
{"WEBI_FORMATS", p.Formats},
{"WEBI_PKG_URL", p.PkgURL},
{"WEBI_PKG_PATHNAME", p.PkgFile},
{"WEBI_PKG_FILE", p.PkgFile},
{"PKG_NAME", p.PkgName},
{"PKG_STABLE", p.PkgStable},
{"PKG_LATEST", p.PkgLatest},
{"PKG_OSES", p.PkgOSes},
{"PKG_ARCHES", p.PkgArches},
{"PKG_LIBCS", p.PkgLibcs},
{"PKG_FORMATS", p.PkgFormats},
}
for _, v := range vars {
text = InjectVar(text, v.name, v.value)
}
// Inject the installer script at the {{ installer }} marker.
// The marker sits inside __init_installer() at 8-space indent.
// Production pads every line of install.sh to match, and replaces
// the entire line (including leading whitespace).
padded := padScript(string(installSh), " ")
text = replaceMarkerLine(text, "{{ installer }}", padded)
return text, nil
}
// PowerShell renders a complete PowerShell installer script by injecting
// params into the template and splicing in the package's install.ps1.
func PowerShell(tplPath, installersDir, pkgName string, p Params) (string, error) {
tpl, err := os.ReadFile(tplPath)
if err != nil {
return "", fmt.Errorf("render: read template: %w", err)
}
installPath := filepath.Join(installersDir, pkgName, "install.ps1")
installPs1, err := os.ReadFile(installPath)
if err != nil {
return "", fmt.Errorf("render: read %s/install.ps1: %w", pkgName, err)
}
text := string(tpl)
vars := []struct {
name string
value string
}{
{"WEBI_PKG", p.PkgName + "@" + p.Tag},
{"WEBI_HOST", p.Host},
{"WEBI_VERSION", p.Version},
{"WEBI_GIT_TAG", p.GitTag},
{"WEBI_GIT_COMMIT_HASH", p.GitCommitHash},
{"WEBI_PKG_URL", p.PkgURL},
{"WEBI_PKG_FILE", p.PkgFile},
{"WEBI_PKG_PATHNAME", p.PkgFile},
{"PKG_NAME", p.PkgName},
}
for _, v := range vars {
text = InjectPSVar(text, v.name, v.value)
}
// PS1 marker is at column 0, no padding needed.
text = replaceMarkerLine(text, "{{ installer }}", string(installPs1))
return text, nil
}
// InjectPSVar replaces a PowerShell template variable line with its value.
// Matches lines like:
//
// #$Env:WEBI_VERSION = v12.16.2
// $Env:WEBI_HOST = 'https://webinstall.dev'
func InjectPSVar(text, name, value string) string {
p := getPSVarPattern(name)
return p.ReplaceAllString(text, "${1}$$Env:"+name+" = '"+sanitizePSValue(value)+"'")
}
var psVarPatterns = map[string]*regexp.Regexp{}
func getPSVarPattern(name string) *regexp.Regexp {
if p, ok := psVarPatterns[name]; ok {
return p
}
// Match: optional leading whitespace, optional #, $Env:NAME, =, rest of line
p := regexp.MustCompile(`(?m)^([ \t]*)#?\$Env:` + regexp.QuoteMeta(name) + `\s*=.*$`)
psVarPatterns[name] = p
return p
}
// sanitizePSValue escapes single quotes for PowerShell single-quoted strings.
// In PowerShell, single quotes inside single-quoted strings are doubled: ''
func sanitizePSValue(s string) string {
return strings.ReplaceAll(s, "'", "''")
}
// varPattern matches shell variable declarations in the template.
// Matches lines like:
//
// #WEBI_VERSION=
// #export WEBI_PKG_URL=
// #WEBI_OS=
var varPatterns = map[string]*regexp.Regexp{}
func getVarPattern(name string) *regexp.Regexp {
if p, ok := varPatterns[name]; ok {
return p
}
// Match: optional leading whitespace, optional #, optional export, the var name, =, rest of line
p := regexp.MustCompile(`(?m)^([ \t]*)#?([ \t]*)(export[ \t]+)?[ \t]*(` + regexp.QuoteMeta(name) + `)=.*$`)
varPatterns[name] = p
return p
}
// InjectVar replaces a template variable line with its value.
// It matches lines like:
//
// #WEBI_VERSION=
// #export WEBI_PKG_URL=
// export WEBI_HOST=
//
// and replaces them with the value in single quotes.
func InjectVar(text, name, value string) string {
p := getVarPattern(name)
return p.ReplaceAllString(text, "${1}${3}"+name+"='"+sanitizeShellValue(value)+"'")
}
// sanitizeShellValue ensures a value is safe to embed in single quotes.
// Single quotes in shell can't be escaped inside single quotes, so we
// close-quote, add escaped quote, re-open quote: 'foo'\''bar'
func sanitizeShellValue(s string) string {
return strings.ReplaceAll(s, "'", `'\''`)
}
// padScript prepends each line of a script with the given indent string.
// This matches production behavior where install.sh content is indented
// to align with the surrounding template code.
func padScript(script, indent string) string {
lines := strings.Split(script, "\n")
for i, line := range lines {
if line != "" {
lines[i] = indent + line
}
}
return strings.Join(lines, "\n")
}
// replaceMarkerLine replaces an entire line containing the marker
// (including any leading whitespace) with the replacement text.
// This matches production's regex: /\s*#?\s*{{ installer }}/
func replaceMarkerLine(text, marker, replacement string) string {
re := regexp.MustCompile(`(?m)^[ \t]*#?[ \t]*` + regexp.QuoteMeta(marker) + `[^\n]*`)
return re.ReplaceAllLiteralString(text, replacement)
}

View File

@@ -0,0 +1,90 @@
package render
import (
"strings"
"testing"
)
func TestInjectVar(t *testing.T) {
tests := []struct {
name string
input string
key string
value string
want string
}{
{
name: "commented var",
input: " #WEBI_VERSION=",
key: "WEBI_VERSION",
value: "1.2.3",
want: " WEBI_VERSION='1.2.3'",
},
{
name: "commented export var",
input: " #export WEBI_PKG_URL=",
key: "WEBI_PKG_URL",
value: "https://example.com/foo.tar.gz",
want: " export WEBI_PKG_URL='https://example.com/foo.tar.gz'",
},
{
name: "existing value replaced",
input: " export WEBI_HOST=",
key: "WEBI_HOST",
value: "https://webinstall.dev",
want: " export WEBI_HOST='https://webinstall.dev'",
},
{
name: "value with single quotes",
input: " #PKG_NAME=",
key: "PKG_NAME",
value: "it's-a-test",
want: " PKG_NAME='it'\\''s-a-test'",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := InjectVar(tt.input, tt.key, tt.value)
if strings.TrimSpace(got) != strings.TrimSpace(tt.want) {
t.Errorf("got %q\nwant %q", got, tt.want)
}
})
}
}
func TestInjectVarInTemplate(t *testing.T) {
tpl := `#!/bin/sh
__bootstrap_webi() {
#PKG_NAME=
#WEBI_OS=
#WEBI_ARCH=
#WEBI_VERSION=
export WEBI_HOST=
WEBI_PKG_DOWNLOAD=""
`
result := tpl
result = InjectVar(result, "PKG_NAME", "bat")
result = InjectVar(result, "WEBI_OS", "linux")
result = InjectVar(result, "WEBI_ARCH", "x86_64")
result = InjectVar(result, "WEBI_VERSION", "0.26.1")
result = InjectVar(result, "WEBI_HOST", "https://webinstall.dev")
if !strings.Contains(result, "PKG_NAME='bat'") {
t.Error("PKG_NAME not injected")
}
if !strings.Contains(result, "WEBI_OS='linux'") {
t.Error("WEBI_OS not injected")
}
if !strings.Contains(result, "WEBI_VERSION='0.26.1'") {
t.Error("WEBI_VERSION not injected")
}
if !strings.Contains(result, "export WEBI_HOST='https://webinstall.dev'") {
t.Error("WEBI_HOST not injected")
}
// Should not have #PKG_NAME= anymore.
if strings.Contains(result, "#PKG_NAME=") {
t.Error("#PKG_NAME= should have been replaced")
}
}

303
internal/resolve/resolve.go Normal file
View File

@@ -0,0 +1,303 @@
// Package resolve picks the best release for a given platform query.
//
// Given a set of classified distributables and a target query (OS, arch,
// libc, format preferences, version constraint), it returns the single
// best matching release — or nil if nothing matches.
package resolve
import (
"strings"
"github.com/webinstall/webi-installers/internal/buildmeta"
"github.com/webinstall/webi-installers/internal/lexver"
)
// Dist is one downloadable distributable — matches the CSV row from classify.
type Dist struct {
Package string
Version string
Channel string
OS string
Arch string
Libc string
Format string
Download string
Filename string
SHA256 string
Size int64
LTS bool
Date string
Extra string // extra version info for sorting
GitTag string // original git tag or branch — only for format="git"
GitCommitHash string // short commit hash — only for format="git"
Variants []string // build qualifiers: "installer", "rocm", "fxdependent", etc.
}
// Query describes what the caller wants.
type Query struct {
OS buildmeta.OS
Arch buildmeta.Arch
Libc buildmeta.Libc
Formats []string // acceptable formats (e.g. ".tar.gz", ".zip"), in preference order
Channel string // "stable" (default), "beta", etc.
Version string // version prefix constraint ("24", "24.14", ""), empty = latest
Variants []string // if non-empty, only match assets with these variants
}
// Match is the resolved release.
type Match struct {
Version string
OS string
Arch string
Libc string
Format string
Download string
Filename string
LTS bool
Date string
Channel string
}
// Best finds the single best release matching the query.
// Returns nil if nothing matches.
func Best(dists []Dist, q Query) *Match {
channel := q.Channel
if channel == "" {
channel = "stable"
}
// Build format set for fast lookup + rank map for preference.
formatRank := make(map[string]int, len(q.Formats))
for i, f := range q.Formats {
formatRank[f] = i
}
// Build the set of acceptable architectures (native + compat).
compatArches := buildmeta.CompatArches(q.OS, q.Arch)
archRank := make(map[string]int, len(compatArches))
for i, a := range compatArches {
archRank[string(a)] = i
}
// Parse version prefix for constraint matching.
var versionPrefix lexver.Version
hasVersionConstraint := q.Version != ""
if hasVersionConstraint {
versionPrefix = lexver.Parse(q.Version)
}
var best *candidate
for i := range dists {
d := &dists[i]
// Channel filter.
if channel == "stable" && d.Channel != "stable" && d.Channel != "" {
continue
}
// OS filter: exact match, POSIX fallback, or ANYOS.
if !osMatches(q.OS, d.OS) {
continue
}
// Arch filter (including compat arches).
// Empty arch, ANYARCH, or "*" means "universal/platform-agnostic" —
// accept it but rank it lower than an exact match.
aRank, archOK := archRank[d.Arch]
if !archOK && (d.Arch == "" || d.Arch == "*" || d.Arch == string(buildmeta.ArchAny)) {
// Universal binary — rank after all specific arches.
aRank = len(compatArches)
archOK = true
}
if !archOK {
continue
}
// Libc filter.
if !libcMatches(q.OS, q.Libc, d.Libc) {
continue
}
// Format filter.
// Empty format means bare binary — accept as last resort.
fRank, formatOK := formatRank[d.Format]
if !formatOK && d.Format == "" {
// Bare binary — rank after all explicit formats.
fRank = len(q.Formats)
formatOK = true
}
if !formatOK && len(q.Formats) > 0 {
continue
}
if !formatOK {
fRank = 999
}
// Version constraint.
ver := lexver.Parse(d.Version)
if hasVersionConstraint && !ver.HasPrefix(versionPrefix) {
continue
}
c := &candidate{
dist: d,
ver: ver,
archRank: aRank,
formatRank: fRank,
hasVariants: len(d.Variants) > 0,
}
if best == nil || c.betterThan(best) {
best = c
}
}
if best == nil {
return nil
}
d := best.dist
return &Match{
Version: d.Version,
OS: d.OS,
Arch: d.Arch,
Libc: d.Libc,
Format: d.Format,
Download: d.Download,
Filename: d.Filename,
LTS: d.LTS,
Date: d.Date,
Channel: d.Channel,
}
}
// Catalog computes aggregate metadata across all stable dists for a package.
type Catalog struct {
OSes []string
Arches []string
Libcs []string
Formats []string
Latest string // highest version of any channel
Stable string // highest stable version
}
// Survey scans all dists and returns the catalog.
func Survey(dists []Dist) Catalog {
oses := make(map[string]bool)
arches := make(map[string]bool)
libcs := make(map[string]bool)
formats := make(map[string]bool)
var latest, stable string
for _, d := range dists {
if d.OS != "" {
oses[d.OS] = true
}
if d.Arch != "" {
arches[d.Arch] = true
}
if d.Libc != "" {
libcs[d.Libc] = true
}
if d.Format != "" {
formats[d.Format] = true
}
v := lexver.Parse(d.Version)
if latest == "" || lexver.Compare(v, lexver.Parse(latest)) > 0 {
latest = d.Version
}
if d.Channel == "stable" || d.Channel == "" {
if stable == "" || lexver.Compare(v, lexver.Parse(stable)) > 0 {
stable = d.Version
}
}
}
return Catalog{
OSes: sortedKeys(oses),
Arches: sortedKeys(arches),
Libcs: sortedKeys(libcs),
Formats: sortedKeys(formats),
Latest: latest,
Stable: stable,
}
}
type candidate struct {
dist *Dist
ver lexver.Version
archRank int
formatRank int
hasVariants bool // true if dist has variant qualifiers (GPU, installer, etc.)
}
// betterThan returns true if c is a better match than other.
// Priority: version (higher) > base over variant > arch rank (lower=native) > format rank (lower=preferred).
func (c *candidate) betterThan(other *candidate) bool {
cmp := lexver.Compare(c.ver, other.ver)
if cmp != 0 {
return cmp > 0
}
// Prefer base build over variant builds (rocm, installer, etc.)
if c.hasVariants != other.hasVariants {
return !c.hasVariants
}
if c.archRank != other.archRank {
return c.archRank < other.archRank
}
return c.formatRank < other.formatRank
}
// osMatches checks whether a dist's OS is acceptable for the query.
// Matches exact OS, ANYOS (universal), and POSIX compatibility levels
// (posix_2017 matches any non-Windows OS).
func osMatches(want buildmeta.OS, have string) bool {
if have == string(want) {
return true
}
if have == string(buildmeta.OSAny) {
return true
}
// POSIX assets run on any non-Windows system.
if want != buildmeta.OSWindows {
if have == string(buildmeta.OSPosix2017) || have == string(buildmeta.OSPosix2024) {
return true
}
}
return false
}
// libcMatches checks whether a dist's libc is acceptable for the query.
func libcMatches(os buildmeta.OS, want buildmeta.Libc, have string) bool {
// Darwin and Windows don't use libc tagging — accept anything.
if os == buildmeta.OSDarwin || os == buildmeta.OSWindows {
return true
}
// If the dist has no libc tag, accept it (likely statically linked).
if have == "" || have == "none" || have == string(buildmeta.LibcNone) {
return true
}
// If the query has no libc preference, accept any.
if want == "" || want == buildmeta.LibcNone {
return true
}
return have == string(want)
}
func sortedKeys(m map[string]bool) []string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
// Simple insertion sort — these are tiny sets.
for i := 1; i < len(keys); i++ {
for j := i; j > 0 && strings.Compare(keys[j-1], keys[j]) > 0; j-- {
keys[j-1], keys[j] = keys[j], keys[j-1]
}
}
return keys
}

View File

@@ -0,0 +1,416 @@
// Package resolver selects the best release asset for a given platform
// and version constraint.
//
// The resolver takes a package's full asset list and a request describing
// what the client needs (OS, arch, libc, version prefix, channel, format
// preferences). It returns the single best matching asset or an error.
//
// Resolution order:
// 1. Filter assets by channel (inclusive: @stable includes stable+lts)
// 2. Sort versions descending, filter by version prefix if given
// 3. For each candidate version, try compatible platform triplets
// (OS × CompatArches fallback × libc) in preference order
// 4. Among platform matches, pick the best format
// 5. Among format matches, prefer assets without build variants
package resolver
import (
"errors"
"slices"
"strings"
"github.com/webinstall/webi-installers/internal/buildmeta"
"github.com/webinstall/webi-installers/internal/lexver"
"github.com/webinstall/webi-installers/internal/storage"
)
// ErrNoMatch is returned when no asset matches the request.
var ErrNoMatch = errors.New("resolver: no matching asset")
// Request describes what the client is looking for.
type Request struct {
// OS is the target operating system (e.g. "linux", "darwin", "windows").
OS string
// Arch is the target architecture (e.g. "aarch64", "x86_64").
Arch string
// Libc is the preferred C library (e.g. "gnu", "musl", "msvc").
// Empty means no preference — the resolver tries all libc values.
Libc string
// Version is a version prefix constraint (e.g. "1.20", "1", "").
// Empty means latest. Exact versions like "1.20.3" also work.
Version string
// Channel selects the release stability level. Values:
// ""/"stable" — stable and LTS only (default)
// "lts" — LTS releases only
// "rc" — rc + stable + LTS
// "beta" — beta + rc + stable + LTS
// "alpha" — everything (alpha + beta + rc + stable + LTS)
// "pre" — alias for beta (package-specific meaning)
Channel string
// LTS when true selects only LTS-flagged releases.
LTS bool
// Formats lists acceptable archive formats in preference order.
// If empty, a default preference order is used.
Formats []string
// Variant selects a specific build variant (e.g. "rocm", "jetpack6").
// If empty, assets with variants are deprioritized.
Variant string
}
// Result holds the resolved asset and metadata about the match.
type Result struct {
// Asset is the selected download.
Asset storage.Asset
// Version is the matched version string.
Version string
// Triplet is the matched platform triplet (os-arch-libc).
Triplet string
}
// Resolve finds the best matching asset for the given request.
func Resolve(assets []storage.Asset, req Request) (Result, error) {
if len(assets) == 0 {
return Result{}, ErrNoMatch
}
// Parse the version prefix for filtering.
var versionPrefix lexver.Version
hasPrefix := req.Version != ""
if hasPrefix {
versionPrefix = lexver.Parse(req.Version)
}
// Build the channel filter.
channelOK := channelFilter(req.Channel, req.LTS)
// Parse and sort all unique versions descending.
type versionEntry struct {
parsed lexver.Version
raw string
}
seen := make(map[string]bool)
var versions []versionEntry
for _, a := range assets {
if seen[a.Version] {
continue
}
seen[a.Version] = true
v := lexver.Parse(a.Version)
v.Raw = a.Version
versions = append(versions, versionEntry{parsed: v, raw: a.Version})
}
slices.SortFunc(versions, func(a, b versionEntry) int {
return lexver.Compare(b.parsed, a.parsed) // descending
})
// Build platform fallback list: ordered (os, arch, libc) combinations.
triplets := enumerateTriplets(req.OS, req.Arch, req.Libc)
// Build format preference list.
formats := req.Formats
if len(formats) == 0 {
formats = defaultFormats(req.OS)
}
// Index assets by version+triplet for fast lookup.
// Assets with empty OS/Arch (like git repos) use "" keys.
type tripletKey struct {
version string
os string
arch string
libc string
}
index := make(map[tripletKey][]storage.Asset)
for _, a := range assets {
key := tripletKey{
version: a.Version,
os: a.OS,
arch: a.Arch,
libc: a.Libc,
}
index[key] = append(index[key], a)
}
// Walk versions in descending order.
for _, ve := range versions {
// Check version prefix.
if hasPrefix && !ve.parsed.HasPrefix(versionPrefix) {
continue
}
// Check channel.
if !channelOK(ve.parsed.Channel, ve.raw) {
continue
}
// Try each compatible triplet.
for _, tri := range triplets {
key := tripletKey{
version: ve.raw,
os: tri.os,
arch: tri.arch,
libc: tri.libc,
}
candidates := index[key]
if len(candidates) == 0 {
continue
}
// Pick the best asset from candidates.
best, ok := pickBest(candidates, formats, req.Variant, req.LTS)
if !ok {
continue
}
triplet := tri.os + "-" + tri.arch + "-" + tri.libc
return Result{
Asset: best,
Version: ve.raw,
Triplet: triplet,
}, nil
}
}
return Result{}, ErrNoMatch
}
// channelFilter returns a function that checks whether a given channel
// is acceptable for the requested channel level.
func channelFilter(requested string, ltsOnly bool) func(channel string, version string) bool {
if ltsOnly {
return func(_ string, _ string) bool {
// LTS filtering happens at the asset level, not version level.
// We let all versions through and filter by LTS flag later.
// Actually, LTS is per-asset, so we handle it in pickBest.
return true
}
}
requested = strings.ToLower(requested)
if requested == "" {
requested = "stable"
}
if requested == "pre" {
requested = "beta"
}
if requested == "latest" {
requested = "stable"
}
// channelRank maps channel names to a numeric rank.
// Higher rank = less stable. A request for rank N accepts
// everything at rank N or below.
rank := func(ch string) int {
ch = strings.ToLower(ch)
switch ch {
case "", "stable":
return 0
case "rc":
return 1
case "beta", "preview":
return 2
case "alpha", "dev":
return 3
default:
return 2 // unknown pre-release channels default to beta-level
}
}
maxRank := rank(requested)
return func(channel string, _ string) bool {
return rank(channel) <= maxRank
}
}
type platformTriple struct {
os string
arch string
libc string
}
// enumerateTriplets builds the ordered list of platform combinations to try.
// It uses CompatArches for arch fallback and tries multiple libc values.
func enumerateTriplets(osStr, archStr, libcStr string) []platformTriple {
// OS candidates: specific OS first, then POSIX compat, then any.
var oses []string
switch osStr {
case "windows":
oses = []string{"windows", "ANYOS", ""}
case "android":
oses = []string{"android", "linux", "posix_2024", "posix_2017", "ANYOS", ""}
case "":
oses = []string{"ANYOS", ""}
default:
oses = []string{osStr, "posix_2024", "posix_2017", "ANYOS", ""}
}
// Arch candidates: use CompatArches for fallback chain.
arches := buildmeta.CompatArches(buildmeta.OS(osStr), buildmeta.Arch(archStr))
var archStrs []string
for _, a := range arches {
archStrs = append(archStrs, string(a))
}
// Also try ANYARCH and empty (for platform-agnostic assets like git repos).
archStrs = append(archStrs, "ANYARCH", "")
// Libc candidates.
var libcs []string
if libcStr != "" {
libcs = []string{libcStr, "none", ""}
} else {
// No preference: try all common options.
switch osStr {
case "linux":
// none first (static, no deps), then gnu, musl, empty.
libcs = []string{"none", "gnu", "musl", ""}
case "windows":
// none first (no deps), msvc last (needs vcredist).
libcs = []string{"none", "msvc", ""}
default:
libcs = []string{"none", ""}
}
}
var triplets []platformTriple
for _, os := range oses {
for _, arch := range archStrs {
for _, libc := range libcs {
triplets = append(triplets, platformTriple{
os: os,
arch: arch,
libc: libc,
})
}
}
}
return triplets
}
// pickBest selects the best asset from a set of candidates for the same
// version and platform. Prefers the requested variant (or no-variant if
// none requested), then picks by format preference.
func pickBest(candidates []storage.Asset, formats []string, wantVariant string, ltsOnly bool) (storage.Asset, bool) {
// Filter by LTS if requested.
if ltsOnly {
var lts []storage.Asset
for _, a := range candidates {
if a.LTS {
lts = append(lts, a)
}
}
if len(lts) == 0 {
return storage.Asset{}, false
}
candidates = lts
}
// Separate into variant-matched and non-variant pools.
var preferred []storage.Asset
var fallback []storage.Asset
for _, a := range candidates {
if wantVariant != "" {
// User requested a specific variant.
if hasVariant(a.Variants, wantVariant) {
preferred = append(preferred, a)
} else if len(a.Variants) == 0 {
fallback = append(fallback, a)
}
} else {
// No variant requested: prefer no-variant assets.
if len(a.Variants) == 0 {
preferred = append(preferred, a)
} else {
fallback = append(fallback, a)
}
}
}
// Try preferred pool first, then fallback.
for _, pool := range [][]storage.Asset{preferred, fallback} {
if len(pool) == 0 {
continue
}
if best, ok := pickByFormat(pool, formats); ok {
return best, true
}
}
return storage.Asset{}, false
}
// pickByFormat selects the asset with the most preferred format.
func pickByFormat(assets []storage.Asset, formats []string) (storage.Asset, bool) {
for _, fmt := range formats {
for _, a := range assets {
if a.Format == fmt {
return a, true
}
}
}
// No format match — return the first asset as last resort.
if len(assets) > 0 {
return assets[0], true
}
return storage.Asset{}, false
}
func hasVariant(variants []string, want string) bool {
for _, v := range variants {
if v == want {
return true
}
}
return false
}
// defaultFormats returns the format preference order for an OS.
// zst is preferred as the modern standard, but availability varies.
func defaultFormats(os string) []string {
switch os {
case "windows":
return []string{
".tar.zst",
".tar.xz",
".zip",
".tar.gz",
".exe.xz",
".7z",
".exe",
".msi",
"git",
}
case "darwin":
return []string{
".tar.zst",
".tar.xz",
".zip",
".tar.gz",
".gz",
".app.zip",
".dmg",
".pkg",
"git",
}
default:
// Linux and other POSIX.
return []string{
".tar.zst",
".tar.xz",
".tar.gz",
".gz",
".zip",
".xz",
"git",
}
}
}

View File

@@ -0,0 +1,290 @@
package resolver_test
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"github.com/webinstall/webi-installers/internal/resolver"
"github.com/webinstall/webi-installers/internal/storage"
)
func loadAssets(t *testing.T, pkg string) []storage.Asset {
t.Helper()
cacheDir := filepath.Join("..", "..", "_cache", "2026-03")
path := filepath.Join(cacheDir, pkg+".json")
data, err := os.ReadFile(path)
if err != nil {
t.Skipf("no cache file for %s: %v", pkg, err)
}
var lc storage.LegacyCache
if err := json.Unmarshal(data, &lc); err != nil {
t.Fatalf("parse %s: %v", pkg, err)
}
pd := storage.ImportLegacy(lc)
return pd.Assets
}
// TestCacheResolveAllPackages loads every package from the cache and verifies
// the resolver finds a match for each standard platform.
func TestCacheResolveAllPackages(t *testing.T) {
cacheDir := filepath.Join("..", "..", "_cache", "2026-03")
entries, err := os.ReadDir(cacheDir)
if err != nil {
t.Skipf("no cache dir: %v", err)
}
var pkgs []string
for _, e := range entries {
if strings.HasSuffix(e.Name(), ".json") {
pkgs = append(pkgs, strings.TrimSuffix(e.Name(), ".json"))
}
}
if len(pkgs) < 50 {
t.Fatalf("expected at least 50 packages, got %d", len(pkgs))
}
platforms := []struct {
name string
os string
arch string
}{
{"darwin-arm64", "darwin", "aarch64"},
{"darwin-amd64", "darwin", "x86_64"},
{"linux-amd64", "linux", "x86_64"},
{"linux-arm64", "linux", "aarch64"},
{"windows-amd64", "windows", "x86_64"},
}
for _, pkg := range pkgs {
t.Run(pkg, func(t *testing.T) {
assets := loadAssets(t, pkg)
if len(assets) == 0 {
t.Skip("no releases")
}
// Determine which OSes this package has.
osSet := make(map[string]bool)
for _, a := range assets {
if a.OS != "" {
osSet[a.OS] = true
}
}
// Also check for platform-agnostic assets.
hasAgnostic := false
for _, a := range assets {
if a.OS == "" {
hasAgnostic = true
break
}
}
for _, plat := range platforms {
supported := osSet[plat.os] ||
osSet["ANYOS"] ||
hasAgnostic ||
(plat.os != "windows" && (osSet["posix_2017"] || osSet["posix_2024"]))
if !supported {
continue
}
t.Run(plat.name, func(t *testing.T) {
res, err := resolver.Resolve(assets, resolver.Request{
OS: plat.os,
Arch: plat.arch,
})
if err != nil {
// Not a test failure — some packages don't have
// all arch builds. Log for visibility.
t.Logf("WARN: no match for %s on %s (has OSes: %v)",
pkg, plat.name, sortedOSes(osSet))
return
}
if res.Version == "" {
t.Error("matched but Version is empty")
}
if res.Asset.Download == "" {
t.Error("matched but Download is empty")
}
})
}
})
}
}
// TestCacheKnownPackages verifies specific packages resolve correctly.
var knownPackages = []struct {
pkg string
version string // expected latest stable version prefix
platforms []string
}{
{"bat", "0.26", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"caddy", "2.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"delta", "0.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"fd", "10.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"fzf", "0.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"gh", "2.", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"rg", "", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"node", "", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"terraform", "", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
{"zig", "", []string{"darwin-arm64", "darwin-amd64", "linux-amd64", "linux-arm64", "windows-amd64"}},
}
func TestCacheKnownPackages(t *testing.T) {
platMap := map[string]resolver.Request{
"darwin-arm64": {OS: "darwin", Arch: "aarch64"},
"darwin-amd64": {OS: "darwin", Arch: "x86_64"},
"linux-amd64": {OS: "linux", Arch: "x86_64"},
"linux-arm64": {OS: "linux", Arch: "aarch64"},
"windows-amd64": {OS: "windows", Arch: "x86_64"},
}
for _, kp := range knownPackages {
t.Run(kp.pkg, func(t *testing.T) {
assets := loadAssets(t, kp.pkg)
for _, platName := range kp.platforms {
req := platMap[platName]
t.Run(platName, func(t *testing.T) {
res, err := resolver.Resolve(assets, req)
if err != nil {
t.Fatalf("no match for %s on %s", kp.pkg, platName)
}
if kp.version != "" {
v := strings.TrimPrefix(res.Version, "v")
if !strings.HasPrefix(v, kp.version) {
t.Errorf("Version = %q, want prefix %q", res.Version, kp.version)
}
}
})
}
})
}
}
// TestCacheVersionConstraints tests version pinning with real data.
func TestCacheVersionConstraints(t *testing.T) {
tests := []struct {
pkg string
version string
wantPfx string
}{
{"bat", "0.25", "0.25"},
{"bat", "0.26", "0.26"},
{"gh", "2.40", "2.40"},
{"node", "20", "20."},
{"node", "22", "22."},
}
for _, tt := range tests {
t.Run(tt.pkg+"@"+tt.version, func(t *testing.T) {
assets := loadAssets(t, tt.pkg)
res, err := resolver.Resolve(assets, resolver.Request{
OS: "linux",
Arch: "x86_64",
Version: tt.version,
})
if err != nil {
t.Fatalf("no match for %s@%s", tt.pkg, tt.version)
}
v := strings.TrimPrefix(res.Version, "v")
if !strings.HasPrefix(v, tt.wantPfx) {
t.Errorf("Version = %q, want prefix %q", res.Version, tt.wantPfx)
}
})
}
}
// TestCacheArchFallback verifies Rosetta-style fallback with real data.
func TestCacheArchFallback(t *testing.T) {
// awless only has amd64 builds — macOS ARM64 should fall back.
assets := loadAssets(t, "awless")
res, err := resolver.Resolve(assets, resolver.Request{
OS: "darwin",
Arch: "aarch64",
})
if err != nil {
t.Fatal("expected Rosetta 2 fallback for awless")
}
if res.Asset.Arch != "x86_64" {
t.Errorf("Arch = %q, want x86_64", res.Asset.Arch)
}
}
// TestCacheGitPackages verifies git-only packages resolve on any platform.
func TestCacheGitPackages(t *testing.T) {
gitPkgs := []string{"vim-essentials", "vim-spell"}
for _, pkg := range gitPkgs {
t.Run(pkg, func(t *testing.T) {
assets := loadAssets(t, pkg)
if len(assets) == 0 {
t.Skip("no releases")
}
// Should work on any platform.
for _, plat := range []struct {
os, arch string
}{
{"linux", "x86_64"},
{"darwin", "aarch64"},
{"windows", "x86_64"},
} {
res, err := resolver.Resolve(assets, resolver.Request{
OS: plat.os,
Arch: plat.arch,
})
if err != nil {
t.Errorf("expected match on %s-%s", plat.os, plat.arch)
continue
}
if res.Asset.Format != "git" {
t.Errorf("format = %q, want git", res.Asset.Format)
}
}
})
}
}
// TestCacheLibcPreference tests explicit libc selection.
// bat is Rust — its musl builds are static (tagged 'none').
func TestCacheLibcPreference(t *testing.T) {
assets := loadAssets(t, "bat")
// Musl host requesting bat: gets static musl build (tagged 'none').
res, err := resolver.Resolve(assets, resolver.Request{
OS: "linux",
Arch: "x86_64",
Libc: "musl",
})
if err != nil {
t.Fatal("expected match for musl host")
}
if res.Asset.Libc != "none" {
t.Errorf("Libc = %q, want none (static musl)", res.Asset.Libc)
}
// Explicit gnu.
res, err = resolver.Resolve(assets, resolver.Request{
OS: "linux",
Arch: "x86_64",
Libc: "gnu",
})
if err != nil {
t.Fatal("expected gnu match")
}
if res.Asset.Libc != "gnu" {
t.Errorf("Libc = %q, want gnu", res.Asset.Libc)
}
}
func sortedOSes(m map[string]bool) []string {
var keys []string
for k := range m {
keys = append(keys, k)
}
return keys
}

View File

@@ -0,0 +1,397 @@
package resolver
import (
"testing"
"github.com/webinstall/webi-installers/internal/storage"
)
func TestResolveSimple(t *testing.T) {
assets := []storage.Asset{
{
Filename: "bat-v0.25.0-x86_64-unknown-linux-musl.tar.gz",
Version: "0.25.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Libc: "musl",
Format: ".tar.gz",
Download: "https://example.com/bat-0.25.0-linux-x86_64.tar.gz",
},
{
Filename: "bat-v0.26.0-x86_64-unknown-linux-musl.tar.gz",
Version: "0.26.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Libc: "musl",
Format: ".tar.gz",
Download: "https://example.com/bat-0.26.0-linux-x86_64.tar.gz",
},
{
Filename: "bat-v0.26.0-aarch64-unknown-linux-musl.tar.gz",
Version: "0.26.0",
Channel: "stable",
OS: "linux",
Arch: "aarch64",
Libc: "musl",
Format: ".tar.gz",
Download: "https://example.com/bat-0.26.0-linux-aarch64.tar.gz",
},
{
Filename: "bat-v0.26.0-x86_64-pc-windows-msvc.zip",
Version: "0.26.0",
Channel: "stable",
OS: "windows",
Arch: "x86_64",
Libc: "msvc",
Format: ".zip",
Download: "https://example.com/bat-0.26.0-windows-x86_64.zip",
},
{
Filename: "bat-v0.26.0-x86_64-apple-darwin.tar.gz",
Version: "0.26.0",
Channel: "stable",
OS: "darwin",
Arch: "x86_64",
Format: ".tar.gz",
Download: "https://example.com/bat-0.26.0-darwin-x86_64.tar.gz",
},
}
t.Run("latest linux x86_64", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
})
if err != nil {
t.Fatal(err)
}
if res.Version != "0.26.0" {
t.Errorf("version = %q, want 0.26.0", res.Version)
}
if res.Asset.OS != "linux" {
t.Errorf("os = %q, want linux", res.Asset.OS)
}
if res.Asset.Arch != "x86_64" {
t.Errorf("arch = %q, want x86_64", res.Asset.Arch)
}
})
t.Run("latest linux aarch64", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "aarch64",
})
if err != nil {
t.Fatal(err)
}
if res.Version != "0.26.0" {
t.Errorf("version = %q, want 0.26.0", res.Version)
}
if res.Asset.Arch != "aarch64" {
t.Errorf("arch = %q, want aarch64", res.Asset.Arch)
}
})
t.Run("version prefix 0.25", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
Version: "0.25",
})
if err != nil {
t.Fatal(err)
}
if res.Version != "0.25.0" {
t.Errorf("version = %q, want 0.25.0", res.Version)
}
})
t.Run("darwin arm64 falls back to x86_64", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "darwin",
Arch: "aarch64",
})
if err != nil {
t.Fatal(err)
}
if res.Asset.Arch != "x86_64" {
t.Errorf("arch = %q, want x86_64 (Rosetta fallback)", res.Asset.Arch)
}
})
t.Run("no match returns error", func(t *testing.T) {
_, err := Resolve(assets, Request{
OS: "freebsd",
Arch: "x86_64",
})
if err != ErrNoMatch {
t.Errorf("err = %v, want ErrNoMatch", err)
}
})
t.Run("windows gets zip", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "windows",
Arch: "x86_64",
})
if err != nil {
t.Fatal(err)
}
if res.Asset.Format != ".zip" {
t.Errorf("format = %q, want .zip", res.Asset.Format)
}
})
}
func TestResolveChannels(t *testing.T) {
assets := []storage.Asset{
{
Filename: "tool-v2.0.0-rc1-linux-x86_64.tar.gz",
Version: "2.0.0-rc1",
Channel: "rc",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
},
{
Filename: "tool-v1.5.0-linux-x86_64.tar.gz",
Version: "1.5.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
},
{
Filename: "tool-v2.0.0-beta2-linux-x86_64.tar.gz",
Version: "2.0.0-beta2",
Channel: "beta",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
},
}
t.Run("stable skips rc and beta", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
})
if err != nil {
t.Fatal(err)
}
if res.Version != "1.5.0" {
t.Errorf("version = %q, want 1.5.0", res.Version)
}
})
t.Run("rc includes rc and stable", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
Channel: "rc",
})
if err != nil {
t.Fatal(err)
}
if res.Version != "2.0.0-rc1" {
t.Errorf("version = %q, want 2.0.0-rc1", res.Version)
}
})
t.Run("beta includes beta, rc, and stable", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
Channel: "beta",
})
if err != nil {
t.Fatal(err)
}
// beta2 sorts after rc1 for the same numeric version (2.0.0),
// but rc1 is more stable. However, the user asked for beta channel
// which includes everything — and beta sorts before rc alphabetically.
// With lexver: 2.0.0-rc1 > 2.0.0-beta2 (rc > beta alphabetically).
if res.Version != "2.0.0-rc1" {
t.Errorf("version = %q, want 2.0.0-rc1", res.Version)
}
})
}
func TestResolveVariants(t *testing.T) {
assets := []storage.Asset{
{
Filename: "ollama-linux-amd64.tgz",
Version: "0.6.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
},
{
Filename: "ollama-linux-amd64-rocm.tgz",
Version: "0.6.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
Variants: []string{"rocm"},
},
}
t.Run("no variant prefers plain", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
})
if err != nil {
t.Fatal(err)
}
if len(res.Asset.Variants) != 0 {
t.Errorf("variants = %v, want empty", res.Asset.Variants)
}
})
t.Run("explicit variant selects it", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
Variant: "rocm",
})
if err != nil {
t.Fatal(err)
}
if !hasVariant(res.Asset.Variants, "rocm") {
t.Errorf("variants = %v, want [rocm]", res.Asset.Variants)
}
})
}
func TestResolveFormatPreference(t *testing.T) {
assets := []storage.Asset{
{
Filename: "tool-v1.0.0-linux-x86_64.tar.gz",
Version: "1.0.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
},
{
Filename: "tool-v1.0.0-linux-x86_64.tar.xz",
Version: "1.0.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.xz",
},
{
Filename: "tool-v1.0.0-linux-x86_64.tar.zst",
Version: "1.0.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.zst",
},
}
t.Run("default prefers zst", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
})
if err != nil {
t.Fatal(err)
}
if res.Asset.Format != ".tar.zst" {
t.Errorf("format = %q, want .tar.zst", res.Asset.Format)
}
})
t.Run("explicit format preference", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
Formats: []string{".tar.gz"},
})
if err != nil {
t.Fatal(err)
}
if res.Asset.Format != ".tar.gz" {
t.Errorf("format = %q, want .tar.gz", res.Asset.Format)
}
})
}
func TestResolveGitAssets(t *testing.T) {
assets := []storage.Asset{
{
Filename: "vim-commentary-v1.2",
Version: "1.2",
Channel: "stable",
Format: "git",
Download: "https://github.com/tpope/vim-commentary.git",
},
{
Filename: "vim-commentary-v1.1",
Version: "1.1",
Channel: "stable",
Format: "git",
Download: "https://github.com/tpope/vim-commentary.git",
},
}
t.Run("git assets match any platform", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
})
if err != nil {
t.Fatal(err)
}
if res.Version != "1.2" {
t.Errorf("version = %q, want 1.2", res.Version)
}
if res.Asset.Format != "git" {
t.Errorf("format = %q, want git", res.Asset.Format)
}
})
}
func TestResolveLTS(t *testing.T) {
assets := []storage.Asset{
{
Filename: "node-v22.0.0-linux-x64.tar.gz",
Version: "22.0.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
LTS: false,
},
{
Filename: "node-v20.15.0-linux-x64.tar.gz",
Version: "20.15.0",
Channel: "stable",
OS: "linux",
Arch: "x86_64",
Format: ".tar.gz",
LTS: true,
},
}
t.Run("LTS selects older LTS version", func(t *testing.T) {
res, err := Resolve(assets, Request{
OS: "linux",
Arch: "x86_64",
LTS: true,
})
if err != nil {
t.Fatal(err)
}
if res.Version != "20.15.0" {
t.Errorf("version = %q, want 20.15.0", res.Version)
}
})
}

View File

@@ -67,6 +67,12 @@ func (la LegacyAsset) ToAsset() Asset {
arch = "x86_64"
case "arm64":
arch = "aarch64"
case "armv7l":
arch = "armv7"
case "armv6l":
arch = "armv6"
case "arm":
arch = "armv5"
case "*":
arch = ""
}

View File

@@ -0,0 +1,295 @@
// Package pgstore implements [storage.Store] on PostgreSQL.
//
// Schema uses double-buffering: two asset generations per package (0 and 1).
// The active generation pointer in webi_packages is updated atomically on
// Commit, so readers always see a complete consistent snapshot.
//
// Write path:
//
// BeginRefresh → clears inactive generation, returns tx
// Put → stages assets in-memory
// Commit → bulk-inserts assets (COPY), swaps generation pointer
//
// Read path:
//
// Load → reads active generation from webi_packages, fetches assets
//
// Connection string format: standard libpq / pgx DSN, e.g.:
//
// postgres://user:pass@host/dbname?sslmode=require
// host=localhost user=webi dbname=webi sslmode=disable
package pgstore
import (
"context"
"fmt"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/webinstall/webi-installers/internal/storage"
)
// Schema holds the DDL for creating the required tables.
// Run once on startup or deploy to ensure the schema exists.
const Schema = `
CREATE TABLE IF NOT EXISTS webi_packages (
name TEXT NOT NULL PRIMARY KEY,
active_gen SMALLINT NOT NULL DEFAULT 0,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS webi_assets (
id BIGSERIAL PRIMARY KEY,
pkg TEXT NOT NULL,
gen SMALLINT NOT NULL,
filename TEXT NOT NULL DEFAULT '',
version TEXT NOT NULL DEFAULT '',
lts BOOLEAN NOT NULL DEFAULT FALSE,
channel TEXT NOT NULL DEFAULT '',
date TEXT NOT NULL DEFAULT '',
os TEXT NOT NULL DEFAULT '',
arch TEXT NOT NULL DEFAULT '',
libc TEXT NOT NULL DEFAULT '',
format TEXT NOT NULL DEFAULT '',
download TEXT NOT NULL DEFAULT '',
extra TEXT NOT NULL DEFAULT '',
variants TEXT[] NOT NULL DEFAULT '{}'
);
CREATE INDEX IF NOT EXISTS webi_assets_pkg_gen ON webi_assets (pkg, gen);
`
// Store is a PostgreSQL-backed asset store.
type Store struct {
pool *pgxpool.Pool
}
// New opens a connection pool to the given DSN and applies the schema.
// Returns an error if the connection or schema creation fails.
func New(ctx context.Context, dsn string) (*Store, error) {
cfg, err := pgxpool.ParseConfig(dsn)
if err != nil {
return nil, fmt.Errorf("pgstore: parse dsn: %w", err)
}
pool, err := pgxpool.NewWithConfig(ctx, cfg)
if err != nil {
return nil, fmt.Errorf("pgstore: connect: %w", err)
}
if err := applySchema(ctx, pool); err != nil {
pool.Close()
return nil, err
}
return &Store{pool: pool}, nil
}
// Close releases the connection pool.
func (s *Store) Close() {
s.pool.Close()
}
// ListPackages returns the names of all packages in the store.
func (s *Store) ListPackages(ctx context.Context) ([]string, error) {
rows, err := s.pool.Query(ctx,
`SELECT name FROM webi_packages ORDER BY name`,
)
if err != nil {
return nil, fmt.Errorf("pgstore: list packages: %w", err)
}
defer rows.Close()
var pkgs []string
for rows.Next() {
var name string
if err := rows.Scan(&name); err != nil {
return nil, fmt.Errorf("pgstore: scan package name: %w", err)
}
pkgs = append(pkgs, name)
}
return pkgs, rows.Err()
}
// Load returns all assets for a package using the active generation.
// Returns nil (not an error) if the package is not cached.
func (s *Store) Load(ctx context.Context, pkg string) (*storage.PackageData, error) {
// Fetch active generation and updated_at.
var gen int16
var updatedAt time.Time
err := s.pool.QueryRow(ctx,
`SELECT active_gen, updated_at FROM webi_packages WHERE name = $1`,
pkg,
).Scan(&gen, &updatedAt)
if err == pgx.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("pgstore: load %s: %w", pkg, err)
}
// Fetch all assets for this generation.
rows, err := s.pool.Query(ctx, `
SELECT filename, version, lts, channel, date,
os, arch, libc, format, download, extra, variants
FROM webi_assets
WHERE pkg = $1 AND gen = $2
ORDER BY id
`, pkg, gen)
if err != nil {
return nil, fmt.Errorf("pgstore: load assets %s: %w", pkg, err)
}
defer rows.Close()
var assets []storage.Asset
for rows.Next() {
var a storage.Asset
if err := rows.Scan(
&a.Filename, &a.Version, &a.LTS, &a.Channel, &a.Date,
&a.OS, &a.Arch, &a.Libc, &a.Format, &a.Download,
&a.Extra, &a.Variants,
); err != nil {
return nil, fmt.Errorf("pgstore: scan asset %s: %w", pkg, err)
}
assets = append(assets, a)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("pgstore: rows %s: %w", pkg, err)
}
return &storage.PackageData{
Assets: assets,
UpdatedAt: updatedAt,
}, nil
}
// BeginRefresh starts a write transaction for a package.
// It determines the inactive generation and clears it, ready for new data.
func (s *Store) BeginRefresh(ctx context.Context, pkg string) (storage.RefreshTx, error) {
// Determine which generation to write into (the inactive one).
var activeGen int16
err := s.pool.QueryRow(ctx,
`SELECT active_gen FROM webi_packages WHERE name = $1`,
pkg,
).Scan(&activeGen)
if err != nil && err != pgx.ErrNoRows {
return nil, fmt.Errorf("pgstore: begin refresh %s: %w", pkg, err)
}
// If package doesn't exist yet, activeGen defaults to 0 and we write to gen 1.
// If package exists, we write to the inactive generation (1 - activeGen).
var writeGen int16
if err == pgx.ErrNoRows {
writeGen = 1
} else {
writeGen = 1 - activeGen
}
// Clear the write generation so we start fresh.
if _, err := s.pool.Exec(ctx,
`DELETE FROM webi_assets WHERE pkg = $1 AND gen = $2`,
pkg, writeGen,
); err != nil {
return nil, fmt.Errorf("pgstore: clear gen %d for %s: %w", writeGen, pkg, err)
}
return &refreshTx{
pool: s.pool,
pkg: pkg,
gen: writeGen,
}, nil
}
// refreshTx is an in-progress write for one package.
type refreshTx struct {
pool *pgxpool.Pool
pkg string
gen int16
assets []storage.Asset
}
// Put stages assets for writing. May be called multiple times.
func (tx *refreshTx) Put(assets []storage.Asset) error {
tx.assets = append(tx.assets, assets...)
return nil
}
// Commit bulk-inserts all staged assets, then atomically swaps the
// active generation pointer in webi_packages.
func (tx *refreshTx) Commit(ctx context.Context) error {
if len(tx.assets) == 0 {
return tx.swapGeneration(ctx)
}
// Build rows for pgx.CopyFromRows.
rows := make([][]any, len(tx.assets))
for i, a := range tx.assets {
variants := a.Variants
if variants == nil {
variants = []string{}
}
rows[i] = []any{
tx.pkg,
tx.gen,
a.Filename,
a.Version,
a.LTS,
a.Channel,
a.Date,
a.OS,
a.Arch,
a.Libc,
a.Format,
a.Download,
a.Extra,
variants,
}
}
cols := []string{
"pkg", "gen",
"filename", "version", "lts", "channel", "date",
"os", "arch", "libc", "format", "download", "extra", "variants",
}
_, err := tx.pool.CopyFrom(ctx,
pgx.Identifier{"webi_assets"},
cols,
pgx.CopyFromRows(rows),
)
if err != nil {
return fmt.Errorf("pgstore: copy assets %s: %w", tx.pkg, err)
}
return tx.swapGeneration(ctx)
}
// swapGeneration atomically updates the active generation pointer.
func (tx *refreshTx) swapGeneration(ctx context.Context) error {
_, err := tx.pool.Exec(ctx, `
INSERT INTO webi_packages (name, active_gen, updated_at)
VALUES ($1, $2, now())
ON CONFLICT (name)
DO UPDATE SET active_gen = $2, updated_at = now()
`, tx.pkg, tx.gen)
if err != nil {
return fmt.Errorf("pgstore: swap gen %s: %w", tx.pkg, err)
}
tx.assets = nil
return nil
}
// Rollback discards all staged assets without writing anything.
func (tx *refreshTx) Rollback() error {
tx.assets = nil
return nil
}
// applySchema runs the schema DDL idempotently.
func applySchema(ctx context.Context, pool *pgxpool.Pool) error {
if _, err := pool.Exec(ctx, Schema); err != nil {
return fmt.Errorf("pgstore: apply schema: %w", err)
}
return nil
}

View File

@@ -0,0 +1,247 @@
// Package uadetect identifies the requesting agent's OS, CPU architecture,
// and libc so the server can select the correct release artifact.
//
// An agent identifies itself through multiple signals:
// - The User-Agent header: Webi's bootstrap scripts send "$(uname -srm)",
// e.g. "Darwin 23.1.0 arm64". Browsers, curl, and PowerShell send their
// own UA strings.
// - Query parameters: ?os=linux&arch=arm64 are an explicit declaration
// that takes precedence over the header.
//
// Use [FromRequest] to detect from an HTTP request (preferred).
// Use [Parse] to detect from a raw UA string.
package uadetect
import (
"net/http"
"strings"
"github.com/webinstall/webi-installers/internal/buildmeta"
)
// Result holds the detected platform info from a User-Agent string.
type Result struct {
OS buildmeta.OS
Arch buildmeta.Arch
Libc buildmeta.Libc
}
// FromRequest detects the agent's platform from an HTTP request.
// Query parameters ?os and ?arch override the User-Agent header.
func FromRequest(r *http.Request) Result {
qOS := r.URL.Query().Get("os")
qArch := r.URL.Query().Get("arch")
var ua string
switch {
case qOS != "" && qArch != "":
ua = qOS + " " + qArch
case qOS != "":
ua = qOS
case qArch != "":
ua = qArch
default:
ua = r.Header.Get("User-Agent")
}
return Parse(ua)
}
// Parse extracts OS, arch, and libc from a User-Agent string.
func Parse(ua string) Result {
if ua == "-" {
return Result{}
}
tokens := tokenize(ua)
return Result{
OS: matchOS(tokens),
Arch: matchArch(tokens),
Libc: matchLibc(tokens),
}
}
// tokenize splits a User-Agent into lowercase tokens for matching.
// Splits on whitespace, '/', and ';', since UAs come in various forms:
//
// "Darwin 23.1.0 arm64" (uname -srm)
// "PowerShell/7.3.0" (PowerShell)
// "MS AMD64" (Windows shorthand)
// "Macintosh; Intel Mac OS X 10_15_7" (browser)
func tokenize(ua string) []string {
// Strip xnu kernel info that can mislead arch detection under Rosetta.
// "xnu-7195.60.75~1/RELEASE_ARM64_T8101" contains ARM64 even when
// running as x86_64. This only appears in verbose uname output.
if i := strings.Index(ua, "xnu-"); i >= 0 {
end := strings.IndexByte(ua[i:], ' ')
if end < 0 {
ua = ua[:i]
} else {
ua = ua[:i] + ua[i+end:]
}
}
return strings.FieldsFunc(strings.ToLower(ua), func(r rune) bool {
return r == ' ' || r == '/' || r == ';' || r == '\t'
})
}
// matchOS identifies the operating system from tokens.
// Order matters: Android before Linux, Linux before Windows (for WSL).
func matchOS(tokens []string) buildmeta.OS {
has := func(s string) bool {
for _, t := range tokens {
if strings.Contains(t, s) {
return true
}
}
return false
}
// Android must be checked before Linux.
if has("android") {
return buildmeta.OSAndroid
}
if has("darwin") || has("macos") || has("macintosh") || has("iphone") || has("ios") || has("ipad") {
return buildmeta.OSDarwin
}
// "mac" alone (not in "macintosh" which is already matched)
for _, t := range tokens {
if t == "mac" {
return buildmeta.OSDarwin
}
}
// FreeBSD before Linux (both are POSIX, but FreeBSD never reports "linux").
if has("freebsd") {
return buildmeta.OSFreeBSD
}
// Linux before Windows because WSL UAs contain both "linux" and "microsoft".
// But exclude Cygwin/Msys/MINGW which report Linux-like strings on Windows.
if has("linux") && !has("cygwin") && !has("msysgit") && !has("msys") && !has("mingw") {
return buildmeta.OSLinux
}
// Cygwin, Msys, and MINGW are Windows environments.
if has("windows") || has("win32") || has("microsoft") || has("powershell") ||
has("cygwin") || has("msys") || has("mingw") {
return buildmeta.OSWindows
}
for _, t := range tokens {
if t == "ms" || t == "win" {
return buildmeta.OSWindows
}
}
// Fallback: curl and wget imply a POSIX system, almost always Linux.
if has("curl") || has("wget") {
return buildmeta.OSLinux
}
return ""
}
// matchArch identifies the CPU architecture from tokens.
// More specific patterns are checked before less specific ones.
func matchArch(tokens []string) buildmeta.Arch {
has := func(s string) bool {
for _, t := range tokens {
if strings.Contains(t, s) {
return true
}
}
return false
}
exact := func(s string) bool {
for _, t := range tokens {
if t == s {
return true
}
}
return false
}
// ARM 64-bit (most specific first)
if has("aarch64") || has("arm64") || has("armv8") {
return buildmeta.ArchARM64
}
// ARM 32-bit variants
if has("armv7") || has("arm32") {
return buildmeta.ArchARMv7
}
if has("armv6") {
return buildmeta.ArchARMv6
}
// Bare "arm" without a version qualifier → armv6 (conservative).
if exact("arm") {
return buildmeta.ArchARMv6
}
// POWER (check before generic 64-bit)
if has("ppc64le") {
return buildmeta.ArchPPC64LE
}
if has("ppc64") {
return buildmeta.ArchPPC64
}
// s390x (IBM Z)
if has("s390x") {
return buildmeta.ArchS390X
}
// RISC-V
if has("riscv64") {
return buildmeta.ArchRISCV64
}
// MIPS (check before generic 64-bit)
if has("mips64") {
return buildmeta.ArchMIPS64
}
if has("mips") {
return buildmeta.ArchMIPS
}
// x86-64
if has("x86_64") || has("amd64") || exact("x64") {
return buildmeta.ArchAMD64
}
// x86 32-bit (after x86_64 to avoid false match)
if has("i386") || has("i686") || exact("x86") {
return buildmeta.ArchX86
}
return ""
}
// matchLibc identifies the C library from tokens.
func matchLibc(tokens []string) buildmeta.Libc {
has := func(s string) bool {
for _, t := range tokens {
if strings.Contains(t, s) {
return true
}
}
return false
}
if has("musl") {
return buildmeta.LibcMusl
}
// Don't match "microsoft" — it appears in WSL kernel version strings
// (e.g. "5.15.146.1-microsoft-standard-WSL2") and doesn't indicate MSVC.
if has("msvc") || has("windows") {
return buildmeta.LibcMSVC
}
if has("gnu") || has("glibc") || has("linux") {
return buildmeta.LibcGNU
}
return buildmeta.LibcNone
}

View File

@@ -1,2 +1 @@
github_releases = lsd-rs/lsd
variants = msvc

View File

@@ -0,0 +1,2 @@
source = mariadbdist
asset_filter = galera

View File

@@ -2,67 +2,62 @@
# shellcheck disable=SC2034
__init_ollama() {
set -e
set -u
set -e
set -u
##################
# Install ollama #
##################
##################
# Install ollama #
##################
# Every package should define these 6 variables
pkg_cmd_name="ollama"
# Every package should define these 6 variables
pkg_cmd_name="ollama"
pkg_dst_dir="${HOME}/.local/opt/ollama"
pkg_dst_cmd="${HOME}/.local/bin/ollama"
pkg_dst_cmd="${HOME}/.local/opt/ollama/bin/ollama"
pkg_dst_dir="${HOME}/.local/opt/ollama"
pkg_dst="${pkg_dst_dir}"
pkg_src_dir="${HOME}/.local/opt/ollama-v${WEBI_VERSION}"
pkg_src_cmd="${HOME}/.local/opt/ollama-v${WEBI_VERSION}/bin/ollama"
pkg_src_cmd="${HOME}/.local/opt/ollama-v${WEBI_VERSION}/bin/ollama"
pkg_src_dir="${HOME}/.local/opt/ollama-v${WEBI_VERSION}"
pkg_src="${pkg_src_dir}"
my_os=$(uname -s)
if test "Darwin" = "${my_os}"; then
pkg_dst_cmd="${HOME}/.local/bin/ollama"
pkg_src_cmd="${HOME}/.local/opt/ollama-v${WEBI_VERSION}/ollama"
fi
# pkg_install must be defined by every package
pkg_install() {
# ~/.local/opt/
mkdir -p "$(dirname "${pkg_src_dir}")"
pkg_dst="${pkg_dst_cmd}"
pkg_src="${pkg_src_cmd}"
if test -d ./ollama-*/; then
# the de facto way (in case it's supported in the future)
# mv ./ollama-*/ ~/.local/opt/ollama-v3.27.0/
mv ./ollama-*/ "${pkg_src}"
elif test -d ./bin; then
# how linux is presently done
mkdir -p "${pkg_src_dir}"
mv ./bin "${pkg_src_dir}"
if test -f ./lib; then
mv ./lib "${pkg_src_dir}"
fi
else
# how macOS is presently done
mkdir -p "$(dirname "${pkg_src_cmd}")"
mv ./ollama-* "${pkg_src_cmd}"
fi
# pkg_install must be defined by every package
pkg_install() {
if test -d ./bin; then
# linux tar.zst: bin/ollama + lib/ollama/
mkdir -p "${pkg_src_dir}"
mv ./bin "${pkg_src_dir}/bin"
if test -d ./lib; then
mv ./lib "${pkg_src_dir}/lib"
fi
elif test -f ./ollama; then
# macOS tgz: flat — bare binary + dylibs/mlx backends in root
mkdir -p "${pkg_src_dir}"
mv ./* "${pkg_src_dir}/"
elif test -d ./Ollama.app; then
# macOS zip: install app bundle to /Applications
mv -f ./Ollama.app /Applications/Ollama.app
elif test -f ./ollama-*; then
# older bare binary format
mkdir -p "$(dirname "${pkg_src_cmd}")"
mv ./ollama-* "${pkg_src_cmd}"
else
echo "error: unrecognized ollama archive format" >&2
return 1
fi
}
# remove previous location
if test -f ~/.local/bin/ollama; then
rm ~/.local/bin/ollama
fi
}
pkg_get_current_version() {
# 'ollama --version' has output in this format:
# ollama version is 0.3.10
# This trims it down to just the version number:
# 0.3.10
ollama --version 2> /dev/null |
head -n 1 |
cut -d' ' -f4 |
sed 's:^v::'
}
pkg_get_current_version() {
# 'ollama --version' has output in this format:
# ollama version is 0.3.10
# This trims it down to just the version number:
# 0.3.10
ollama --version 2> /dev/null |
head -n 1 |
cut -d' ' -f4 |
sed 's:^v::'
}
}
__init_ollama

View File

@@ -1,2 +1,2 @@
github_releases = jmorganca/ollama
variants = mlx rocm jetpack5 jetpack6
variants = rocm jetpack5 jetpack6

View File

@@ -1,2 +1,2 @@
github_releases = powershell/powershell
variants = fxdependent fxdependentWinDesktop appimage
variants = fxdependent fxdependentWinDesktop

View File

@@ -11,22 +11,14 @@ g_out="agents/tmp/${g_bin}"
g_remote_bin="~/bin/${g_bin}"
case "${g_host}" in
beta.webi.sh) g_remote_conf="~/srv/beta.webinstall.dev/installers/" ;;
next.webi.sh) g_remote_conf="~/srv/next.webinstall.dev/installers/" ;;
webi.sh) g_remote_conf="~/srv/webinstall.dev/installers/" ;;
*) g_remote_conf="~/srv/webinstall.dev/installers/" ;;
beta.webi.sh) g_remote_conf="~/srv/beta.webinstall.dev/installers/" ;;
next.webi.sh) g_remote_conf="~/srv/next.webinstall.dev/installers/" ;;
*) g_remote_conf="~/srv/webid/installers/" ;;
esac
fn_build() {
b_tag="$(git describe --tags --abbrev=0 --match 'cmd/webicached/*' 2> /dev/null || echo 'cmd/webicached/v0.0.0')"
b_tag_ver="$(printf '%s' "${b_tag}" | sed 's:^cmd/webicached/::')"
b_count="$(git log --oneline "${b_tag}..HEAD" -- cmd/ internal/ 2> /dev/null | wc -l | tr -d ' \t')"
b_version="$(git describe --tags --always 2>/dev/null || echo '0.0.0-dev')"
b_commit="$(git rev-parse --short HEAD)"
if test "${b_count}" -gt 0; then
b_version="${b_tag_ver}-${b_count}-g${b_commit}"
else
b_version="${b_tag_ver}"
fi
b_date="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
b_ldflags="-X main.version=${b_version} -X main.commit=${b_commit} -X main.date=${b_date}"
@@ -37,7 +29,7 @@ fn_build() {
fn_deploy() {
printf 'Stopping %s on %s...\n' "${g_bin}" "${g_host}"
ssh "${g_host}" "~/.local/bin/serviceman stop ${g_bin}" 2> /dev/null || true
ssh "${g_host}" "~/.local/bin/serviceman stop ${g_bin}" 2>/dev/null || true
printf 'Uploading binary...\n'
scp "${g_out}" "${g_host}:${g_remote_bin}"

81
scripts/deploy-webid.sh Executable file
View File

@@ -0,0 +1,81 @@
#!/bin/sh
set -e
set -u
# Build and deploy webid to a target host
g_host="${1:-next.webi.sh}"
g_bin="webid"
g_out="agents/tmp/${g_bin}"
g_remote_bin="~/bin/${g_bin}"
case "${g_host}" in
beta.webi.sh) g_remote_conf="~/srv/beta.webinstall.dev/installers/" ;;
next.webi.sh) g_remote_conf="~/srv/next.webinstall.dev/installers/" ;;
*) g_remote_conf="~/srv/webid/installers/" ;;
esac
fn_build() {
b_version="$(git describe --tags --always 2> /dev/null || echo '0.0.0-dev')"
b_commit="$(git rev-parse --short HEAD)"
b_date="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
b_ldflags="-X main.version=${b_version} -X main.commit=${b_commit} -X main.date=${b_date}"
printf 'Building %s %s %s (%s)...\n' "${g_bin}" "${b_version}" "${b_commit}" "${b_date}"
GOOS=linux GOARCH=amd64 GOAMD64=v2 go build -ldflags "${b_ldflags}" -o "${g_out}" ./cmd/webid
printf 'Built: %s\n' "${g_out}"
}
fn_deploy() {
printf 'Stopping %s on %s...\n' "${g_bin}" "${g_host}"
ssh "${g_host}" "~/.local/bin/serviceman stop ${g_bin}" 2> /dev/null || true
printf 'Uploading binary...\n'
scp "${g_out}" "${g_host}:${g_remote_bin}"
printf 'Syncing install scripts and templates...\n'
rsync -av \
--exclude='_cache' --exclude='.git' --exclude='agents' \
--exclude='bin' --exclude='cmd' --exclude='internal' \
--exclude='docs' --exclude='scripts' --exclude='node_modules' \
--include='*/' --include='install.sh' --include='install.ps1' \
--include='_webi/*.tpl.sh' --include='_webi/*.tpl.ps1' \
--exclude='*' \
./ "${g_host}:${g_remote_conf}"
}
fn_start() {
printf 'Starting %s...\n' "${g_bin}"
ssh "${g_host}" "~/.local/bin/serviceman start ${g_bin}" || {
printf 'Service not configured. Run serviceman add on the host:\n'
printf ' serviceman add --name %s \\\n' "${g_bin}"
printf ' --workdir %s -- \\\n' "${g_remote_conf}"
printf ' %s \\\n' "${g_remote_bin}"
printf ' --addr :3082 \\\n'
printf ' --legacy ~/.cache/webi/legacy \\\n'
printf ' --installers %s\n' "${g_remote_conf}"
exit 1
}
}
fn_verify() {
printf 'Waiting 3s for startup...\n'
sleep 3
printf 'Checking version...\n'
ssh "${g_host}" "${g_remote_bin} -V"
printf 'Checking health...\n'
ssh "${g_host}" "curl -s http://localhost:3082/api/releases/bat.json | head -c 100"
printf '\n'
printf 'Checking logs...\n'
ssh "${g_host}" "sudo journalctl -u ${g_bin} --no-pager -n 5"
}
fn_build
fn_deploy
fn_start
fn_verify
printf '\nDone. %s deployed to %s.\n' "${g_bin}" "${g_host}"

View File

@@ -22,11 +22,11 @@ __init_sd() {
pkg_install() {
# mv ./sd-*/sd "$pkg_src_cmd"
if test -f sd-*; then
# old format: bare binary named sd-{triplet}
# ~/.local/opt/sd-v0.99.9/bin
mkdir -p "$(dirname "$pkg_src_cmd")"
mv sd-* "$pkg_src_cmd"
elif test -f sd-*/sd; then
# current format: sd-v{ver}-{triplet}/ directory
# ~/.local/opt/sd-v0.99.9/bin
mkdir -p "$(dirname "$pkg_src_cmd")"
mv sd-*/sd "$pkg_src_cmd"
if test -f sd-*/sd.1; then

View File

@@ -1,118 +0,0 @@
---
title: sql-migrate
homepage: https://github.com/therootcompany/golib/tree/main/cmd/sql-migrate
tagline: |
sql-migrate: A lightweight, feature-branch-friendly SQL migrator.
---
To update or switch versions, run `webi sql-migrate@stable` (or `@v2`, `@beta`,
etc).
### Files
These are the files that are created and/or modified with this installer:
```text
~/.config/envman/PATH.env
~/.local/bin/sql-migrate
~/.local/opt/sql-migrate-VERSION/bin/sql-migrate
<PROJECT-DIR>/migrations.log
<PROJECT-DIR>/sql/migrations/<yyyy-mm-dd>-<number>_<name>.<up|down>.sql
```
## Cheat Sheet
> `sql-migrate` is a lightweight migration tool that gets out of your way - it
> works with your existing SQL tools and allows working with distinct sets of
> migrations, such as is typical in feature branches.
### How to use sql-migrate
Migration commands output a POSIX shell script which should be run with `sh`.
`init`, `sync`, `up`, `down`, `list`, `status`
```sh
# Initialize a migration directory
sql-migrate -d ./sql/migrations/ init --sql-command psql
# Export ENVs for your database
export PG_URL='postgres://user:pass@example.com/dbname?sslmode=require&sslnegotiation=direct'
# Create a new migration (auto-generates up/down pair with incrementing number)
sql-migrate -d ./sql/migrations/ create add-users-table
# SELECT and log existing migrations
sql-migrate -d ./sql/migrations/ sync | sh
# Run all pending migrations up
sql-migrate -d ./sql/migrations/ up | sh
# Run 3 migrations up
sql-migrate -d ./sql/migrations/ up 3 | sh
# Run 1 migration down (roll back)
sql-migrate -d ./sql/migrations/ down | sh
# Run 2 migrations down
sql-migrate -d ./sql/migrations/ down 2 | sh
# List all migrations
sql-migrate -d ./sql/migrations/ list
# See which migrations have been applied
sql-migrate -d ./sql/migrations/ status
```
### Migration directory layout
Migrations follow the naming format
`<yyyy-mm-dd>-<number>_<name>.<up|down>.sql`:
```text
sql/
├── migrations.log # transaction log (auto-managed)
└── migrations/
├── 0001-01-01-001000_init-migrations.up.sql # generated by 'init' (has config vars)
├── 2021-02-03-001000_init-app.up.sql
├── 2021-02-03-001000_init-app.down.sql
├── 2021-02-03-002000_add-products.up.sql
├── 2021-02-03-002000_add-products.down.sql
└── 2021-02-03-003000_add-customers.up.sql
```
The initial `0001-01-01-001000_init-migrations.up.sql` migration contains
configuration variables:
```sql
-- migrations_log: ./sql/migrations.log
-- sql_command: psql "$PG_URL" -v ON_ERROR_STOP=on --no-align --tuples-only --file %s
```
Environment variables by database:
- PostgreSQL: `PG_URL` (auth url), `PGOPTIONS` (to set `schema` and other
specific options)
- SQLite3: `SQLITE_PATH`
- SQL Server: `SQLCMDSERVER`, `SQLCMDDATABASE`, `SQLCMDUSER`, `SQLCMDPASSWORD`
- MySQL / MariaDB: `MY_CNF` (path to `my.cnf`, containing credentials)
### Database client compatibility
The `--sql-command` flag tells sql-migrate how to talk to your database:
```sh
sql-migrate -d ./sql/migrations/ init --sql-command psql
```
The following clients are known and will have the correct options applied:
- psql (PostgreSQL)
- sqlite3
- sqlcmd (mssql / Microsoft SQL Server)
- mariadb / mysql
Since the migrations run via shell commands, you can make `sql-migrate`
compatible with any SQL client by setting `sql_command` in
`migrations/0001-01-01-001000_init-migrations.up.sql`.

View File

@@ -1,78 +0,0 @@
---
name: sql-migrate
description:
Manage SQL database migrations as plain .sql files with a transaction log. Use
when asked about sql-migrate, database migrations, or how to set up migration
tooling.
---
# sql-migrate (CLI)
The agent skill is embedded in the help output.
Run `sql-migrate --help` use its output to guide usage decisions.
The `up`, `down`, and `sync` subcommands produce POSIX shell scripts - pipe them
to `sh` to run.
## Migration layout
Migrations follow the naming format
`<yyyy-mm-dd>-<number>_<name>.<up|down>.sql`:
```
sql/
├── migrations.log # transaction log (auto-managed)
└── migrations/
├── 0001-01-01-001000_init-migrations.up.sql # generated by 'init'
├── 2021-02-03-001000_init-app.up.sql
├── 2021-02-03-001000_init-app.down.sql
└── ...
```
The initial migration file contains configuration variables:
```sql
-- migrations_log: ./sql/migrations.log
-- sql_command: psql "$PG_URL" -v ON_ERROR_STOP=on --no-align --tuples-only --file %s
```
## Migration file structure
The migration files contain their own management, including a randomly-generated
id:
```sql
-- change_me (up)
SELECT 'place your UP migration here';
-- leave this as the last line
INSERT INTO _migrations (name, id) VALUES ('2026-05-27-002000_change_me', 'e22295e5');
```
## Environment variables by database
- PostgreSQL: `PG_URL` (auth URL), `PGOPTIONS` (set `schema` and other options)
- SQLite3: `SQLITE_PATH`
- SQL Server: `SQLCMDSERVER`, `SQLCMDDATABASE`, `SQLCMDUSER`, `SQLCMDPASSWORD`
- MySQL / MariaDB: `MY_CNF` (path to `my.cnf` containing credentials)
## SQL client extensibility
The `--sql-command` flag tells sql-migrate how to talk to your database. Known
clients (`psql`, `sqlite3`, `sqlcmd`, `mariadb`/`mysql`) have correct options
applied automatically. Since migrations run via shell commands, you can make
sql-migrate compatible with any SQL client by setting `sql_command` in the init
migration file.
# sqlmigrate (Go module)
See `go doc` for each independent module (golib is a monorepo):
- `github.com/therootcompany/golib/database/sqlmigrate/v2`
- `github.com/therootcompany/golib/database/sqlmigrate/pgmigrate`
- `github.com/therootcompany/golib/database/sqlmigrate/litemigrate`
- `github.com/therootcompany/golib/database/sqlmigrate/msmigrate`
- `github.com/therootcompany/golib/database/sqlmigrate/mymigrate`
DO NOT search the parent golib module.

View File

@@ -1,47 +0,0 @@
#!/usr/bin/env pwsh
######################
# Install sql-migrate #
######################
$pkg_cmd_name = "sql-migrate"
$pkg_dst_cmd = "$Env:USERPROFILE\.local\bin\sql-migrate.exe"
$pkg_dst = "$pkg_dst_cmd"
$pkg_src_cmd = "$Env:USERPROFILE\.local\opt\sql-migrate-v$Env:WEBI_VERSION\bin\sql-migrate.exe"
$pkg_src_bin = "$Env:USERPROFILE\.local\opt\sql-migrate-v$Env:WEBI_VERSION\bin"
$pkg_src_dir = "$Env:USERPROFILE\.local\opt\sql-migrate-v$Env:WEBI_VERSION"
$pkg_src = "$pkg_src_cmd"
New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
$pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
# Fetch archive
if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
Write-Output "Downloading sql-migrate from $Env:WEBI_PKG_URL to $pkg_download"
& curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
& Move-Item "$pkg_download.part" "$pkg_download"
}
if (!(Test-Path -Path "$pkg_src_cmd")) {
Write-Output "Installing sql-migrate"
Push-Location .local\tmp
Remove-Item -Path ".\sql-migrate-v*" -Recurse -ErrorAction Ignore
Remove-Item -Path ".\sql-migrate.exe" -Recurse -ErrorAction Ignore
Write-Output "Unpacking $pkg_download"
& tar xf "$pkg_download"
Write-Output "Install Location: $pkg_src_cmd"
New-Item "$pkg_src_bin" -ItemType Directory -Force | Out-Null
Move-Item -Path ".\sql-migrate.exe" -Destination "$pkg_src_bin"
Pop-Location
}
Write-Output "Copying into '$pkg_dst_cmd' from '$pkg_src_cmd'"
Remove-Item -Path "$pkg_dst_cmd" -Recurse -ErrorAction Ignore | Out-Null
Copy-Item -Path "$pkg_src" -Destination "$pkg_dst" -Recurse

View File

@@ -1,33 +0,0 @@
#!/bin/sh
# shellcheck disable=SC2034
set -e
set -u
__init_sql_migrate() {
pkg_cmd_name="sql-migrate"
pkg_dst_cmd="${HOME}/.local/bin/sql-migrate"
pkg_dst="${pkg_dst_cmd}"
pkg_src_cmd="${HOME}/.local/opt/sql-migrate-v${WEBI_VERSION}/bin/sql-migrate"
pkg_src_dir="${HOME}/.local/opt/sql-migrate-v${WEBI_VERSION}"
pkg_src="${pkg_src_cmd}"
pkg_install() {
pkg_src_bin=$(dirname "${pkg_src_cmd}")
mkdir -p "${pkg_src_bin}"
mv ./sql-migrate "${pkg_src_cmd}"
}
# pkg_get_current_version is recommended, but (soon) not required
pkg_get_current_version() {
# 'sql-migrate version' has output in this format:
# sql-migrate v0.0.0-dev 0000000 (0001-01-01)
# This trims it down to just the version number:
# v0.0.0-dev
sql-migrate version 2> /dev/null | head -n 1 | cut -d ' ' -f 2
}
}
__init_sql_migrate

View File

@@ -1,2 +0,0 @@
github_releases = therootcompany/golib
tag_prefix = cmd/sql-migrate/v

View File

@@ -1,3 +1 @@
github_releases = abhimanyu003/sttr
exclude = .sbom.json
variants = .pkg.tar.

View File

@@ -10,9 +10,7 @@ __rmrf_local() {
arc \
archiver \
awless \
basecamp \
bat \
btop \
caddy \
chromedriver \
cmake \
@@ -108,7 +106,6 @@ __rmrf_local() {
arc \
archiver \
awless \
basecamp \
bat \
caddy \
chromedriver \
@@ -208,7 +205,6 @@ __test() {
arc \
archiver \
awless \
basecamp \
bat \
caddy \
chromedriver \

View File

@@ -4,34 +4,34 @@ set -u
__init_yq() {
pkg_cmd_name="yq"
pkg_cmd_name="yq"
pkg_dst_cmd="$HOME/.local/bin/yq"
pkg_dst="$pkg_dst_cmd"
pkg_dst_cmd="$HOME/.local/bin/yq"
pkg_dst="$pkg_dst_cmd"
pkg_src_cmd="$HOME/.local/opt/yq-v$WEBI_VERSION/bin/yq"
pkg_src_dir="$HOME/.local/opt/yq-v$WEBI_VERSION"
pkg_src="$pkg_src_cmd"
pkg_src_cmd="$HOME/.local/opt/yq-v$WEBI_VERSION/bin/yq"
pkg_src_dir="$HOME/.local/opt/yq-v$WEBI_VERSION"
pkg_src="$pkg_src_cmd"
pkg_install() {
mkdir -p "$(dirname "$pkg_src_cmd")"
# yq_linux_amd64.tar.gz contains:
# - yq_linux_amd64
# - yq.1
# - install-man-page.sh
if [ -e ./yq.1 ]; then
mkdir -p ~/.local/share/man/man1
mv ./yq.1 ~/.local/share/man/man1/
fi
mv ./"$pkg_cmd_name"* "$pkg_src_cmd"
chmod a+x "$pkg_src_cmd"
}
pkg_install() {
mkdir -p "$(dirname "$pkg_src_cmd")"
# yq_linux_amd64.tar.gz contains:
# - yq_linux_amd64
# - yq.1
# - install-man-page.sh
if [ -e ./yq.1 ]; then
mkdir -p ~/.local/share/man/man1
mv ./yq.1 ~/.local/share/man/man1/
fi
mv ./"$pkg_cmd_name"* "$pkg_src_cmd"
chmod a+x "$pkg_src_cmd"
}
pkg_get_current_version() {
yq --version 2> /dev/null |
head -n 1 |
cut -d ' ' -f 2
}
pkg_get_current_version() {
yq --version 2> /dev/null |
head -n 1 |
cut -d ' ' -f 2
}
}