Compare commits

..

1 Commits

Author SHA1 Message Date
AJ ONeal
b209bfe46f feat(installers): install shell completions and man pages (bat, fd, lsd, rg, watchexec, zoxide)
Install bash/fish/zsh completions from release archive into
pkg_src_dir/share/, and man pages into pkg_src_dir/share/man/man1/.
2026-05-14 15:26:50 -06:00
221 changed files with 344 additions and 10992 deletions

23
.gitignore vendored
View File

@@ -3,45 +3,22 @@ install-*.sh
install-*.bat
install-*.ps1
# Go build outputs (from go run/build in repo root)
/classify
/e2etest
/fetchraw
/inspect
/uaparse
/webicached
/zigtest
/distributables.csv
# local config
.env.*
*.env
.env
!example.env
LOCAL.md
# caches
_cache/
node_modules/
# temporary & backup files
agents/
.*.sw*
*.bak
*.bak.*
tmp
*.tmp
tmp.*
*.tmp.*
# agent session files
agents/
# other
.DS_Store
desktop.ini
.directory
LIVE_cache
/webid
bin/
.claude/

120
AGENTS.md
View File

@@ -370,123 +370,3 @@ Commit messages: `feat(<pkg>): add installer`, `fix(<pkg>): update install.sh`,
`arch`, or `ext` explicitly in `releases.js`.
- **Goreleaser archives**: Typically contain a bare binary at the archive root
(not nested in a directory). Use `mv ./cmd "$pkg_src_cmd"`.
---
## Go Cache Daemon
The Go pipeline (`cmd/webicached`) replaces the Node.js release-fetching code.
It reads `releases.conf` files, fetches upstream release metadata, classifies
build assets, and writes to `~/.cache/webi/legacy/` in the format the Node.js server expects.
### Canonical Vocabulary
The classifier MUST use exactly these strings. They match the production API.
**OS**: `macos` (NOT `darwin`), `linux`, `windows`, `freebsd`, `openbsd`,
`netbsd`, `dragonfly`, `aix`, `illumos`, `plan9`, `solaris`, `posix_2017`
**Arch** — exact equivalences:
- `amd64` (NOT `x86_64`), `x86` (NOT `i386`/`i686`/`386`)
- `arm64` (NOT `aarch64`)
- `armv7l` (NOT `armv7`), `armv6l` (NOT `armv6`)
- `mipsle` (NOT `mipsel`), `mips64le` (NOT `mips64el`)
**Arch** — compatibility downcasts:
- `armhf` → `armv7l`, `armv7a` → `armv7l`, `armel` → `arm`
**Arch** — other: `arm`, `ppc64le`, `ppc64`, `loong64`, `riscv64`, `s390x`,
`mips`, `mips64`
**Libc**: `none` (never empty), `gnu`, `musl`, `msvc`
**Ext**: `tar.gz`, `tar.xz`, `zip`, `exe`, `7z`, `pkg`, `msi`
(no leading dot; `exe` for bare binaries)
### releases.conf
Each package directory contains a `releases.conf` that tells the daemon where
to fetch releases. Format is `key = value`, one per line. `#` comments and
blank lines are ignored.
#### Source types (mutually exclusive — pick one)
```ini
# GitHub binary releases (most common)
github_releases = sharkdp/bat
# GitHub source tarballs (with optional git fallback)
github_sources = bnnanet/serviceman
git_url = https://github.com/bnnanet/serviceman.git
# Git tag enumeration (vim plugins, shell scripts — git_url alone)
git_url = https://github.com/tpope/vim-commentary.git
# Gitea (full URL required, or short form + base_url)
gitea_releases = https://git.rootprojects.org/root/pathman
# GitLab (defaults to gitlab.com)
gitlab_releases = owner/repo
# HashiCorp releases API
hashicorp_product = terraform
# Custom source (servicemandist, nodedist, zigdist, etc.)
source = nodedist
url = https://nodejs.org/download/release
```
#### Filtering, versioning, and platform
```ini
tag_prefix = bun- # monorepo: strip prefix from version
version_prefixes = jq- # strip from version string (space-separated)
asset_filter = MinGit # filename must contain this substring
exclude = busybox -src- -docs- # skip assets containing these (space-separated)
os = posix_2017 # restrict ALL versions to this OS (blanket)
alias_of = rg # mirrors another package's releases
```
#### Design rules
- `os` is a blanket tag on ALL versions. Only use for packages that are always
POSIX-only. For version-dependent OS tagging, use a custom `TagVariants` in
`internal/releases/{pkg}/variants.go`.
- `git_url` can be primary (gittag source when it's the only key) or secondary
fallback alongside `github_sources`/`gitea_sources`.
- Full URL forms accepted for github/gitea/gitlab (e.g.
`github_releases = https://github.com/sharkdp/bat`).
### Testing
Test tools: `cmd/e2etest` (pipeline comparison), `cmd/comparecache` (cache diff),
`cmd/inspect` (single-package debug). Run each with `--help` for usage.
### Classifier vs Per-Package Tagger
The general classifier (`internal/classify/`) handles patterns common across
many projects. It MUST NOT contain one-off logic for a single package.
Per-package taggers (`internal/releases/{pkg}/variants.go`) handle
project-specific knowledge. Read the existing taggers for conventions.
MUST: Derive arch/OS from concrete evidence — not blanket defaults.
MUST: New general classifier patterns must apply to 2-3+ packages.
### Deploying
```sh
./scripts/deploy-webicached.sh beta.webi.sh
./scripts/deploy-webicached.sh next.webi.sh
```
First-time setup on a new host uses `serviceman`:
```sh
serviceman add --name webicached \
--workdir ~/srv/webid/installers/ -- \
~/bin/webicached \
--envfile ~/srv/webid/.env.secret \
--conf ~/srv/webid/installers/ \
--raw ~/.cache/webi/raw
```

View File

@@ -1,3 +0,0 @@
# Example releases.conf — uses ripgrep as a sample project.
# Copy this file into your package directory and adjust.
github_releases = BurntSushi/ripgrep

View File

@@ -621,15 +621,13 @@ BuildsCacher.create = function ({ ALL_TERMS, installers }) {
let arches = waterfall[hostTarget.arch] ||
HostTargets.WATERFALL.ANYOS[hostTarget.arch] || [hostTarget.arch];
arches = arches.concat(['ANYARCH']);
// termsToTarget omits libc for plain UAs; 'libc' → waterfall ['none','libc',...]
let libc = hostTarget.libc || 'libc';
let libcs = waterfall[libc] ||
HostTargets.WATERFALL.ANYOS[libc] || [libc];
let libcs = waterfall[hostTarget.libc] ||
HostTargets.WATERFALL.ANYOS[hostTarget.libc] || [hostTarget.libc];
// Extend the glibc-host waterfall: the table only lists [none, libc]
// but Rust projects (bat, rg) and node ship libc='gnu' builds, and
// static musl builds also run on glibc hosts.
if (libc === 'libc' && !libcs.includes('gnu')) {
if (hostTarget.libc === 'libc' && !libcs.includes('gnu')) {
libcs = ['none', 'gnu', 'musl', 'libc'];
}

View File

@@ -71,14 +71,6 @@ Builds.getPackage({ name: projName }).then(async function (/*projInfo*/) {
var nodeOs = os.platform();
var nodeOsRelease = os.release();
var nodeArch = os.arch();
// To make arch names compatible across all helpers
if (nodeArch === 'x64') {
nodeArch = 'amd64';
} else if (nodeArch === 'arm64') {
nodeArch = 'arm64';
}
var nodeLibc = 'libc';
if (process.platform === 'linux') {
nodeLibc = 'gnu';

View File

@@ -38,8 +38,9 @@ function getOs(ua) {
// It's the year of the Linux Desktop!
// See also http://www.mslinux.org/
// 'linux' must be tested before 'Microsoft' because WSL
// (TODO: does this affect cygwin / msysgit?)
return 'linux';
} else if (/^ms$|Microsoft|Windows|win32|win|PowerShell|CYGWIN|MINGW/i.test(ua)) {
} else if (/^ms$|Microsoft|Windows|win32|win|PowerShell/i.test(ua)) {
// 'win' must be tested after 'darwin'
return 'windows';
} else if (/Linux|curl|wget/i.test(ua)) {

View File

@@ -1,2 +0,0 @@
github_sources = BeyondCodeBootcamp/aliasman
git_url = https://github.com/BeyondCodeBootcamp/aliasman.git

View File

@@ -1 +0,0 @@
github_releases = mholt/archiver

View File

@@ -1 +0,0 @@
github_releases = wez/atomicparsley

View File

@@ -1 +0,0 @@
github_releases = wallix/awless

View File

@@ -1,128 +0,0 @@
---
title: basecamp
homepage: https://github.com/basecamp/basecamp-cli
tagline: |
basecamp: CLI for Basecamp 3 — manage projects, todos, messages, cards, and more from the terminal.
---
To update or switch versions, run `webi basecamp@stable` (or `@v0.7`,
`@beta`, etc).
### Files
These are the files / directories that are created and/or modified with this
install:
```text
~/.config/envman/PATH.env
~/.local/bin/basecamp
~/.local/opt/basecamp-VERSION/bin/basecamp
~/.local/opt/basecamp-VERSION/completions/
```
## Cheat Sheet
> `basecamp` is the official CLI for Basecamp 3. It provides full API coverage
> for projects, todos, messages, cards, schedule, files, and more — all from the
> command line.
### How to authenticate
```sh
basecamp auth login
```
For headless environments (CI, remote servers):
```sh
basecamp auth login --device-code
```
Check auth status:
```sh
basecamp auth status
```
### How to list projects and todos
```sh
basecamp projects list --md
basecamp todos list --assignee me --in PROJECT_ID --md
```
Cross-project view of your assigned work:
```sh
basecamp assignments --md
```
### How to create and complete todos
```sh
basecamp todo "Write release notes" --in PROJECT_ID --list TODOLIST_ID --assignee me --due tomorrow
basecamp done TODO_ID
```
### How to post a message or comment
```sh
basecamp message "Sprint Update" "Shipped v2.1 to production." --in PROJECT_ID
basecamp comment RECORDING_ID "Looks good." --in PROJECT_ID
```
### How to move cards through a workflow
```sh
basecamp cards columns --in PROJECT_ID --md
basecamp cards move CARD_ID --to COLUMN_ID --in PROJECT_ID
```
### How to set up per-project defaults
Create `.basecamp/config.json` in your repo (commit it):
```json
{
"project_id": "12345",
"todolist_id": "67890"
}
```
Then trust it once:
```sh
basecamp config trust
```
After that, you can omit `--in` for most commands in that repo.
### Shell completions
Completions for bash, fish, and zsh ship with the installer. Find them at:
```text
~/.local/opt/basecamp-VERSION/completions/
```
Bash:
```sh
echo "source ~/.local/opt/basecamp-VERSION/completions/basecamp.bash" >> ~/.bashrc
```
Fish:
```sh
ln -s ~/.local/opt/basecamp-VERSION/completions/basecamp.fish ~/.config/fish/completions/
```
Zsh:
```sh
echo "fpath+=( ~/.local/opt/basecamp-VERSION/completions )" >> ~/.zshrc
```

View File

@@ -1,46 +0,0 @@
#!/usr/bin/env pwsh
$pkg_cmd_name = "basecamp"
$pkg_dst_cmd = "$Env:USERPROFILE\.local\bin\basecamp.exe"
$pkg_dst = "$pkg_dst_cmd"
$pkg_src_cmd = "$Env:USERPROFILE\.local\opt\basecamp-v$Env:WEBI_VERSION\bin\basecamp.exe"
$pkg_src_bin = "$Env:USERPROFILE\.local\opt\basecamp-v$Env:WEBI_VERSION\bin"
$pkg_src_dir = "$Env:USERPROFILE\.local\opt\basecamp-v$Env:WEBI_VERSION"
$pkg_src = "$pkg_src_cmd"
New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
$pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
Write-Output "Downloading basecamp from $Env:WEBI_PKG_URL to $pkg_download"
& curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
& Move-Item "$pkg_download.part" "$pkg_download"
}
if (!(Test-Path -Path "$pkg_src_cmd")) {
Write-Output "Installing basecamp"
Push-Location .local\tmp
Remove-Item -Path ".\basecamp-*" -Recurse -ErrorAction Ignore
Remove-Item -Path ".\basecamp.exe" -Recurse -ErrorAction Ignore
Write-Output "Unpacking $pkg_download"
& tar xf "$pkg_download"
New-Item "$pkg_src_bin" -ItemType Directory -Force | Out-Null
Move-Item -Path ".\basecamp.exe" -Destination "$pkg_src_bin"
New-Item "$pkg_src_dir\completions" -ItemType Directory -Force | Out-Null
if (Test-Path -Path ".\completions") {
Copy-Item -Path ".\completions\*" -Destination "$pkg_src_dir\completions" -Recurse
}
Pop-Location
}
Write-Output "Copying into '$pkg_dst_cmd' from '$pkg_src_cmd'"
Remove-Item -Path "$pkg_dst_cmd" -Recurse -ErrorAction Ignore | Out-Null
Copy-Item -Path "$pkg_src" -Destination "$pkg_dst" -Recurse

View File

@@ -1,44 +0,0 @@
#!/bin/sh
# shellcheck disable=SC2034
set -e
set -u
__init_basecamp() {
pkg_cmd_name="basecamp"
pkg_src_dir="$HOME/.local/opt/basecamp-v$WEBI_VERSION"
pkg_src_cmd="$pkg_src_dir/bin/basecamp"
pkg_src="$pkg_src_cmd"
pkg_dst_cmd="$HOME/.local/bin/basecamp"
pkg_dst="$pkg_dst_cmd"
pkg_install() {
mkdir -p "$(dirname "$pkg_src_cmd")"
mkdir -p "$pkg_src_dir/completions"
if test -f ./basecamp; then
mv ./basecamp "$pkg_src_cmd"
elif test -e ./basecamp-*/basecamp; then
mv ./basecamp-*/basecamp "$pkg_src_cmd"
elif test -e ./basecamp-*; then
mv ./basecamp-* "$pkg_src_cmd"
else
echo >&2 "failed to find 'basecamp' executable"
return 1
fi
if test -d ./completions; then
cp -a ./completions/. "$pkg_src_dir/completions/"
fi
}
pkg_get_current_version() {
basecamp --version 2> /dev/null |
head -n 1 |
cut -d' ' -f3
}
}
__init_basecamp

View File

@@ -1,2 +0,0 @@
github_releases = basecamp/basecamp-cli
exclude = .bundle .txt

View File

@@ -28,6 +28,22 @@ __init_bat() {
# chmod a+x ~/.local/opt/bat-v0.15.4/bin/bat
chmod a+x "$pkg_src_cmd"
# install completions if present (autocomplete/)
if test -d ./bat-*/autocomplete; then
mkdir -p "$pkg_src_dir/share/bash-completion/completions"
mkdir -p "$pkg_src_dir/share/fish/vendor_completions.d"
mkdir -p "$pkg_src_dir/share/zsh/site-functions"
mv ./bat-*/autocomplete/bat.bash "$pkg_src_dir/share/bash-completion/completions/bat" 2>/dev/null || true
mv ./bat-*/autocomplete/bat.fish "$pkg_src_dir/share/fish/vendor_completions.d/bat.fish" 2>/dev/null || true
mv ./bat-*/autocomplete/bat.zsh "$pkg_src_dir/share/zsh/site-functions/_bat" 2>/dev/null || true
fi
# install man page if present
if test -f ./bat-*/bat.1; then
mkdir -p "$pkg_src_dir/share/man/man1"
mv ./bat-*/bat.1 "$pkg_src_dir/share/man/man1/bat.1"
fi
if ! [ -e ~/.config/bat/config ]; then
mkdir -p ~/.config/bat/
touch ~/.config/bat/config

View File

@@ -1 +0,0 @@
github_releases = sharkdp/bat

View File

@@ -4,143 +4,73 @@ set -e
set -u
_install_brew() {
# Straight from https://brew.sh
#/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
# Straight from https://brew.sh
#/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
if test "Darwin" = "$(uname -s)"; then
needs_xcode="$(/usr/bin/xcode-select -p > /dev/null 2> /dev/null || echo "true")"
if test -n "${needs_xcode}"; then
echo ""
echo ""
echo "ERROR: Run this command to install XCode Command Line Tools first:"
echo ""
echo " xcode-select --install"
echo ""
echo "After the install, close this terminal, open a new one, and try again."
echo ""
fi
else
if ! command -v gcc > /dev/null; then
echo >&2 "Warning: to install 'gcc' et al on Linux use the built-in package manager."
echo >&2 " For example, try: sudo apt install -y build-essential"
fi
if ! command -v git > /dev/null; then
echo >&2 "Error: to install 'git' on Linux use the built-in package manager."
echo >&2 " For example, try: sudo apt install -y git"
exit 1
fi
fi
if test "Darwin" = "$(uname -s)"; then
needs_xcode="$(/usr/bin/xcode-select -p > /dev/null 2> /dev/null || echo "true")"
if test -n "${needs_xcode}"; then
echo ""
echo ""
echo "ERROR: Run this command to install XCode Command Line Tools first:"
echo ""
echo " xcode-select --install"
echo ""
echo "After the install, close this terminal, open a new one, and try again."
echo ""
fi
else
if ! command -v gcc > /dev/null; then
echo >&2 "Warning: to install 'gcc' et al on Linux use the built-in package manager."
echo >&2 " For example, try: sudo apt install -y build-essential"
fi
if ! command -v git > /dev/null; then
echo >&2 "Error: to install 'git' on Linux use the built-in package manager."
echo >&2 " For example, try: sudo apt install -y git"
exit 1
fi
fi
# From Straight from https://brew.sh
if ! test -d "$HOME/.local/opt/brew"; then
echo "Installing to '$HOME/.local/opt/brew'"
echo ""
echo "If you prefer to have brew installed to '/usr/local' cancel now and do the following:"
echo " rm -rf '$HOME/.local/opt/brew'"
# shellcheck disable=2016
echo ' /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"'
echo ""
sleep 3
#mkdir -p "$HOME/.local/opt/brew"
#curl -fsSL https://github.com/Homebrew/brew/tarball/main |
# tar xz --strip-components 1 -C "$HOME/.local/opt/brew"
git clone --depth 3 https://github.com/Homebrew/brew "$HOME/.local/opt/brew"
fi
# From Straight from https://brew.sh
if ! test -d "$HOME/.local/opt/brew"; then
echo "Installing to '$HOME/.local/opt/brew'"
echo ""
echo "If you prefer to have brew installed to '/usr/local' cancel now and do the following:"
echo " rm -rf '$HOME/.local/opt/brew'"
# shellcheck disable=2016
echo ' /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"'
echo ""
sleep 3
git clone --depth=1 https://github.com/Homebrew/brew "$HOME/.local/opt/brew"
fi
my_shellenv="$("$HOME/.local/opt/brew/bin/brew" shellenv)"
eval "${my_shellenv}"
chmod -R go-w "$(brew --prefix)/share/zsh" 2> /dev/null || true
rm -rf "$HOME/.local/bin/brew-update-service-install"
webi_download \
"$WEBI_HOST/packages/brew/brew-update-service-install" \
"$HOME/.local/bin/brew-update-service-install" \
brew-update-service-install
chmod a+x "$HOME/.local/bin/brew-update-service-install"
rm -rf "$HOME/.local/bin/brew-update-service-install"
webi_download \
"$WEBI_HOST/packages/brew/brew-update-service-install" \
"$HOME/.local/bin/brew-update-service-install" \
brew-update-service-install
chmod a+x "$HOME/.local/bin/brew-update-service-install"
webi_path_add "$HOME/.local/opt/brew/bin"
export PATH="$HOME/.local/opt/brew/bin:$PATH"
webi_path_add "$HOME/.local/opt/brew/bin"
webi_path_add "$HOME/.local/opt/brew/sbin"
echo "Updating brew..."
brew update
fn_brew_shell_integrate_bash
fn_brew_shell_integrate_zsh
fn_brew_shell_integrate_fish
webi_path_add "$HOME/.local/opt/brew/sbin"
export PATH="$HOME/.local/opt/brew/sbin:$PATH"
echo "Installed 'brew' to '$HOME/.local/opt/brew'"
echo ""
echo "If you prefer to have brew installed to '/usr/local' do the following:"
echo " mv '$HOME/.local/opt/brew' '$HOME/.local/opt/brew.$(date '+%F_%H-%M-%S').bak'"
# shellcheck disable=2016
echo ' /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"'
echo ""
echo "Installed 'brew' to '$HOME/.local/opt/brew'"
echo ""
echo "If you prefer to have brew installed to '/usr/local' do the following:"
echo " mv '$HOME/.local/opt/brew' '$HOME/.local/opt/brew.$(date '+%F_%H-%M-%S').bak'"
# shellcheck disable=2016
echo ' /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"'
echo ""
echo "To register 'brew update' as a hourly system service:"
echo " brew-update-service-install"
echo ""
}
fn_brew_shell_integrate_bash() {
if ! command -v bash > /dev/null; then
return 0
fi
if ! test -e ~/.bashrc && ! test -e ~/.bash_history; then
return 0
fi
touch -a ~/.bashrc
if grep -q 'brew shellenv' ~/.bashrc; then
return 0
fi
echo >&2 " Edit ~/.bashrc to init brew"
# shellcheck disable=SC2016
{
echo ''
echo '# Generated by Webi. Do not edit.'
echo 'eval "$('"$HOME/.local/opt/brew/bin/brew"' shellenv)"'
} >> ~/.bashrc
}
fn_brew_shell_integrate_zsh() {
if ! command -v zsh > /dev/null; then
return 0
fi
if ! test -e ~/.zshrc &&
! test -e ~/.zsh_sessions &&
! test -e ~/.zsh_history; then
return 0
fi
touch -a ~/.zshrc
if grep -q 'brew shellenv' ~/.zshrc; then
return 0
fi
echo >&2 " Edit ~/.zshrc to init brew"
# shellcheck disable=SC2016
{
echo ''
echo '# Generated by Webi. Do not edit.'
echo 'eval "$('"$HOME/.local/opt/brew/bin/brew"' shellenv)"'
} >> ~/.zshrc
}
fn_brew_shell_integrate_fish() {
if ! command -v fish > /dev/null; then
return 0
fi
mkdir -p ~/.config/fish
touch -a ~/.config/fish/config.fish
if grep -q 'brew shellenv' ~/.config/fish/config.fish; then
return 0
fi
echo >&2 " Edit ~/.config/fish/config.fish to init brew"
{
echo ''
echo '# Generated by Webi. Do not edit.'
echo "$HOME/.local/opt/brew/bin/brew shellenv | source"
} >> ~/.config/fish/config.fish
echo "To register 'brew update' as a hourly system service:"
echo " brew-update-service-install"
echo ""
}
_install_brew

View File

@@ -1,109 +0,0 @@
---
title: btop
homepage: https://github.com/aristocratos/btop
tagline: |
btop: a beautiful, interactive resource monitor
description: |
btop++ is a fast, feature-rich terminal resource monitor written in C++.
It shows real-time usage and stats for CPU, memory, disks, network, and
processes — with full mouse support, customizable themes, and an
easy-to-use menu system. The spiritual successor to bashtop and bpytop.
---
To update or switch versions, run `webi btop@stable` (or `@v1.4`, `@beta`, etc).
### Files
These are the files / directories that are created and/or modified with this
install:
```
~/.config/envman/PATH.env
~/.local/bin/btop
~/.local/opt/btop/
~/.local/opt/btop-<VERSION>/
```
## Cheat Sheet
![](https://static.linuxblog.io/wp-content/uploads/2021/11/btop.png)
> btop gives you a gorgeous, interactive view of what your system is doing —
> CPU cores, RAM, swap, disk I/O, network throughput, and a filterable process
> list — all in one terminal window.
### Launch btop
```sh
btop
```
### Navigation
| Key | Action |
| -------------- | ----------------------------------- |
| `Arrow keys` | Move selection in process list |
| `Enter` | Show detailed stats for process |
| `F` | Filter / search processes |
| `K` | Send signal (kill, SIGTERM, etc.) |
| `R` | Renice (change process priority) |
| `T` | Toggle tree / flat process view |
| `M` | Change sort field |
| `ESC` | Open settings menu |
| `Q` | Quit |
Mouse support is fully enabled by default — scroll and click anywhere in the UI.
### Change the color theme
Press `ESC` to open the menu, navigate to **Options → Color theme**, and pick
from the built-in themes (Default, TTY, Dracula, Gruvbox, and more).
Custom themes can be placed in:
```
~/.config/btop/themes/
```
### Adjust update interval
In the Options menu, set **Update interval** (in milliseconds). The default is
`2000` (2 seconds). Lower values give a more live feel; higher values reduce CPU
overhead from btop itself.
### Config file location
btop's settings are saved automatically at:
```
~/.config/btop/btop.conf
```
You can edit this file directly to set options like `update_ms`, `color_theme`,
`proc_sorting`, or `net_iface`.
### Run btop with a specific network interface shown
```sh
btop --utf-foce # force UTF-8 box drawing
btop --debug # verbose debug output to btop.log
```
Network interface selection is done interactively inside btop via the network
panel — press `B` / `N` to cycle interfaces.
### GPU monitoring (Linux x86\_64)
On Linux, btop supports Nvidia, AMD, and Intel GPUs out of the box provided the
correct drivers are installed. If wattage or GPU stats are missing, you may need
to grant extended capabilities:
```sh
# Run once after install (requires sudo)
sudo setcap cap_perfmon,cap_sys_ptrace+ep ~/.local/bin/btop
```
### See also
- [btop releases](https://github.com/aristocratos/btop/releases)
- [Theme gallery](https://github.com/aristocratos/btop/tree/main/themes)

View File

@@ -1,58 +0,0 @@
#!/bin/sh
__init_btop() {
set -e
set -u
################
# Install btop #
################
# Every package should define these 6 variables
pkg_cmd_name="btop"
pkg_dst_cmd="$HOME/.local/bin/btop"
pkg_dst="$pkg_dst_cmd"
pkg_src_cmd="$HOME/.local/opt/btop-v$WEBI_VERSION/bin/btop"
pkg_src_dir="$HOME/.local/opt/btop-v$WEBI_VERSION"
pkg_src="$pkg_src_cmd"
# pkg_install must be defined by every package
pkg_install() {
# ~/.local/opt/btop-v1.4.6/bin
mkdir -p "$(dirname "${pkg_src_cmd}")"
# mv ./btop/bin/btop ~/.local/opt/btop-v1.4.6/bin/btop
mv ./btop/bin/btop "${pkg_src_cmd}"
}
# pkg_get_current_version is recommended, but not required
pkg_get_current_version() {
# 'btop --version' has output in this format:
# btop 1.4.6 (rev abcdef0123)
# This trims it down to just the version number:
# 1.4.6
btop --version 2> /dev/null |
head -n 1 |
cut -d ' ' -f 2
}
}
fn_btop_brew_install() {
if ! command -v brew > /dev/null; then
"$HOME/.local/bin/webi" brew
export PATH="$HOME/.local/opt/brew/bin:$HOME/.local/opt/brew/sbin:$PATH"
fi
export HOMEBREW_NO_AUTO_UPDATE=1
export HOMEBREW_NO_ENV_HINTS=1
brew install btop
}
my_os=$(uname -s)
if test "Darwin" = "${my_os}" && test "$WEBI_CHANNEL" = "error"; then
fn_btop_brew_install
return 0
fi
__init_btop

View File

@@ -1,2 +0,0 @@
github_releases = aristocratos/btop
exclude = -m68k -bigsur -monterey -ventura -macos

View File

@@ -1,5 +0,0 @@
github_releases = oven-sh/bun
tag_prefix = bun-
default_x86_64 = x86_64_v3
x86_64_v2 = baseline
variants = profile

View File

@@ -1 +0,0 @@
github_releases = caddyserver/caddy

View File

@@ -1 +0,0 @@
source = chromedist

View File

@@ -1 +0,0 @@
github_releases = cilium/cilium-cli

View File

@@ -1 +0,0 @@
github_releases = Kitware/CMake

View File

@@ -1,924 +0,0 @@
// Command webicached is the release cache daemon. It fetches releases
// from upstream sources, classifies build assets, and writes them to
// the _cache/ directory in the format the Node.js server expects.
//
// This is the Go replacement for the Node.js release-fetching pipeline.
// It reads releases.conf files to discover packages, fetches from the
// configured source, classifies assets, and writes to fsstore.
//
// Default mode: classify all from existing rawcache on startup, then
// fetch+refresh one package per tick (round-robin, 15m default).
//
// Usage:
//
// go run ./cmd/webicached # default: round-robin, one per tick
// go run ./cmd/webicached -eager # fetch all packages on startup
// go run ./cmd/webicached -once -no-fetch # classify from rawcache and exit
// go run ./cmd/webicached bat goreleaser # only these packages
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"log"
"math/rand/v2"
"net/http"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/joho/godotenv"
"github.com/webinstall/webi-installers/internal/classifypkg"
"github.com/webinstall/webi-installers/internal/httpclient"
"github.com/webinstall/webi-installers/internal/installerconf"
"github.com/webinstall/webi-installers/internal/rawcache"
"github.com/webinstall/webi-installers/internal/releases/chromedist"
"github.com/webinstall/webi-installers/internal/releases/flutterdist"
"github.com/webinstall/webi-installers/internal/releases/gitea"
"github.com/webinstall/webi-installers/internal/releases/github"
"github.com/webinstall/webi-installers/internal/releases/githubish"
"github.com/webinstall/webi-installers/internal/releases/gittag"
"github.com/webinstall/webi-installers/internal/releases/golang"
"github.com/webinstall/webi-installers/internal/releases/gpgdist"
"github.com/webinstall/webi-installers/internal/releases/hashicorp"
"github.com/webinstall/webi-installers/internal/releases/iterm2dist"
"github.com/webinstall/webi-installers/internal/releases/juliadist"
"github.com/webinstall/webi-installers/internal/releases/mariadbdist"
"github.com/webinstall/webi-installers/internal/releases/nodedist"
"github.com/webinstall/webi-installers/internal/releases/servicemandist"
"github.com/webinstall/webi-installers/internal/releases/zigdist"
"github.com/webinstall/webi-installers/internal/storage"
"github.com/webinstall/webi-installers/internal/storage/fsstore"
)
var (
name = "webicached"
version = "0.0.0-dev"
commit = "0000000"
date = "0001-01-01"
licenseYear = "2024"
licenseOwner = "AJ ONeal"
licenseType = "MPL-2.0"
)
func printVersion(w io.Writer) {
b_ver := strings.TrimPrefix(version, "v")
_, _ = fmt.Fprintf(w, "%s v%s %s (%s)\n", name, b_ver, commit[:7], date)
_, _ = fmt.Fprintf(w, "Copyright (C) %s %s\n", licenseYear, licenseOwner)
_, _ = fmt.Fprintf(w, "Licensed under %s\n", licenseType)
}
type MainConfig struct {
envFile string
confDir string
cacheDir string
rawDir string
token string
once bool
noFetch bool
shallow bool
eager bool
interval time.Duration
pageDelay time.Duration
}
// WebiCache holds the configuration for the cache daemon.
type WebiCache struct {
ConfDir string // root directory with {pkg}/releases.conf files
Store storage.Store // classified asset storage (fsstore)
RawDir string // raw upstream response cache
Client *http.Client // HTTP client for upstream calls
Auth *githubish.Auth // GitHub API auth (optional)
Shallow bool // fetch only the first page of releases
NoFetch bool // skip fetching, classify from existing raw data only
PageDelay time.Duration // delay between paginated API requests
}
// delayTransport wraps an http.RoundTripper to add a delay between requests.
type delayTransport struct {
base http.RoundTripper
delay time.Duration
last time.Time
}
func (t *delayTransport) RoundTrip(req *http.Request) (*http.Response, error) {
if !t.last.IsZero() && t.delay > 0 {
if wait := t.delay - time.Since(t.last); wait > 0 {
time.Sleep(wait)
}
}
t.last = time.Now()
return t.base.RoundTrip(req)
}
func main() {
if len(os.Args) > 1 {
switch os.Args[1] {
case "-V", "-version", "--version", "version":
printVersion(os.Stdout)
os.Exit(0)
case "help", "-help", "--help":
printVersion(os.Stdout)
fmt.Fprintln(os.Stdout, "")
fs := flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
fs.SetOutput(os.Stdout)
registerFlags(fs, &MainConfig{})
fs.Usage()
os.Exit(0)
}
}
cfg := MainConfig{}
fs := flag.NewFlagSet(os.Args[0], flag.ContinueOnError)
registerFlags(fs, &cfg)
if err := fs.Parse(os.Args[1:]); err != nil {
if errors.Is(err, flag.ErrHelp) {
os.Exit(0)
}
os.Exit(1)
}
cfg.cacheDir = expandHome(cfg.cacheDir)
cfg.rawDir = expandHome(cfg.rawDir)
if cfg.envFile != "" {
if err := godotenv.Load(cfg.envFile); err != nil {
log.Fatalf("envfile: %v", err)
}
}
if cfg.token == "" {
cfg.token = os.Getenv("GITHUB_TOKEN")
}
fss, err := fsstore.New(cfg.cacheDir)
if err != nil {
log.Fatalf("fsstore: %v", err)
}
var store storage.Store = fss
var auth *githubish.Auth
if cfg.token != "" {
auth = &githubish.Auth{Token: cfg.token}
}
client := httpclient.New()
if cfg.pageDelay > 0 {
client.Transport = &delayTransport{
base: client.Transport,
delay: cfg.pageDelay,
}
}
wc := &WebiCache{
ConfDir: cfg.confDir,
Store: store,
RawDir: cfg.rawDir,
Client: client,
Auth: auth,
Shallow: cfg.shallow,
NoFetch: cfg.noFetch,
PageDelay: cfg.pageDelay,
}
filterPkgs := fs.Args()
if cfg.eager {
wc.Run(filterPkgs)
if cfg.once {
return
}
} else if cfg.once {
wc.Run(filterPkgs)
return
} else {
saved := wc.NoFetch
wc.NoFetch = true
wc.Run(filterPkgs)
wc.NoFetch = saved
}
packages, err := discover(wc.ConfDir)
if err != nil {
log.Fatalf("discover: %v", err)
}
nameSet := make(map[string]bool, len(filterPkgs))
for _, a := range filterPkgs {
nameSet[a] = true
}
if len(filterPkgs) > 0 {
var filtered []pkgConf
for _, p := range packages {
if nameSet[p.name] {
filtered = append(filtered, p)
}
}
packages = filtered
}
var real []pkgConf
for _, pkg := range packages {
if pkg.conf.AliasOf == "" {
real = append(real, pkg)
}
}
// rescanNew appends any conf files added since the last scan.
// Returns true when at least one new package was added so the caller
// can restart the batch loop and process new packages immediately.
rescanNew := func() bool {
discovered, err := discover(wc.ConfDir)
if err != nil {
log.Printf("rescan: %v", err)
return false
}
known := make(map[string]bool, len(real))
for _, p := range real {
known[p.name] = true
}
added := false
for _, p := range discovered {
if p.conf.AliasOf != "" || known[p.name] {
continue
}
if len(filterPkgs) > 0 && !nameSet[p.name] {
continue
}
log.Printf("discovered new package: %s (source=%s)", p.name, p.conf.Source)
real = append(real, p)
added = true
}
return added
}
log.Printf("refreshing %d packages, interval %s, batch size 20 (ctrl-c to stop)", len(real), cfg.interval)
for {
// Rescan before computing staleness so newly added conf files are
// included immediately. New packages have a zero timestamp and sort
// to the front of the stale list, so they are processed next.
rescanNew()
stale := wc.stalest(real)
if len(stale) == 0 {
log.Printf("all packages fresh, sleeping %s", cfg.interval)
time.Sleep(cfg.interval)
continue
}
batch := stale
if len(batch) > 20 {
batch = batch[:20]
}
rand.Shuffle(len(batch), func(i, j int) {
batch[i], batch[j] = batch[j], batch[i]
})
log.Printf("batch: %d stale, refreshing %d (most stale first)", len(stale), len(batch))
for _, pkg := range batch {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
if err := wc.refreshPackage(ctx, pkg); err != nil {
log.Printf(" ERROR %s: %v", pkg.name, err)
}
cancel()
time.Sleep(cfg.interval)
// Rescan mid-batch so new packages preempt remaining batch items.
if rescanNew() {
break
}
}
}
}
func registerFlags(fs *flag.FlagSet, cfg *MainConfig) {
fs.StringVar(&cfg.envFile, "envfile", "", "path to .env file to load before running")
fs.StringVar(&cfg.confDir, "conf", ".", "root directory containing {pkg}/releases.conf files")
fs.StringVar(&cfg.cacheDir, "legacy", "~/.cache/webi/legacy", "legacy cache directory (fsstore root)")
fs.StringVar(&cfg.rawDir, "raw", "~/.cache/webi/raw", "raw cache directory for upstream responses")
fs.StringVar(&cfg.token, "token", "", "GitHub API token (or set $GITHUB_TOKEN)")
fs.BoolVar(&cfg.once, "once", false, "run once then exit (no periodic refresh)")
fs.BoolVar(&cfg.noFetch, "no-fetch", false, "skip fetching, classify from existing raw data only")
fs.BoolVar(&cfg.shallow, "shallow", false, "fetch only the first page of releases (latest)")
fs.BoolVar(&cfg.eager, "eager", false, "fetch all packages on startup (default: one per tick)")
fs.DurationVar(&cfg.interval, "interval", 9*time.Second, "delay between individual package fetches")
fs.DurationVar(&cfg.pageDelay, "page-delay", 2*time.Second, "delay between paginated API requests")
}
func expandHome(path string) string {
if !strings.HasPrefix(path, "~/") {
return path
}
home, err := os.UserHomeDir()
if err != nil {
return path
}
return filepath.Join(home, path[2:])
}
// stalest returns packages sorted by most stale first (oldest UpdatedAt).
// Packages with no cache entry or empty assets are considered most stale.
func (wc *WebiCache) stalest(packages []pkgConf) []pkgConf {
type stamped struct {
pkg pkgConf
updatedAt time.Time
}
var stale []stamped
ctx := context.Background()
for _, pkg := range packages {
data, err := wc.Store.Load(ctx, pkg.name)
var t time.Time
if err == nil && data != nil {
t = data.UpdatedAt
}
// Never fetched, or older than 10 minutes.
// 0-asset results are not treated as perpetually stale — packages that
// produce no classifiable assets (e.g. galera) respect the timestamp.
if t.IsZero() || time.Since(t) > 10*time.Minute {
stale = append(stale, stamped{pkg: pkg, updatedAt: t})
}
}
sort.SliceStable(stale, func(i, j int) bool {
ti, tj := stale[i].updatedAt, stale[j].updatedAt
if ti.Equal(tj) {
return stale[i].pkg.name < stale[j].pkg.name
}
return ti.Before(tj)
})
result := make([]pkgConf, len(stale))
for i, s := range stale {
result[i] = s.pkg
}
return result
}
// Run discovers packages and refreshes each one.
func (wc *WebiCache) Run(filterPkgs []string) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
packages, err := discover(wc.ConfDir)
if err != nil {
log.Printf("discover: %v", err)
return
}
if len(filterPkgs) > 0 {
nameSet := make(map[string]bool, len(filterPkgs))
for _, a := range filterPkgs {
nameSet[a] = true
}
var filtered []pkgConf
for _, p := range packages {
if nameSet[p.name] {
filtered = append(filtered, p)
}
}
packages = filtered
}
var real []pkgConf
for _, pkg := range packages {
if pkg.conf.AliasOf != "" {
continue
}
real = append(real, pkg)
}
log.Printf("refreshing %d packages", len(real))
runStart := time.Now()
for _, pkg := range real {
if err := wc.refreshPackage(ctx, pkg); err != nil {
log.Printf(" ERROR %s: %v", pkg.name, err)
}
}
log.Printf("refreshed %d packages in %s", len(real), time.Since(runStart))
}
type pkgConf struct {
name string
conf *installerconf.Conf
}
func discover(dir string) ([]pkgConf, error) {
pattern := filepath.Join(dir, "*", "releases.conf")
matches, err := filepath.Glob(pattern)
if err != nil {
return nil, err
}
var packages []pkgConf
for _, path := range matches {
pkgDir := filepath.Dir(path)
name := filepath.Base(pkgDir)
if strings.HasPrefix(name, "_") {
continue
}
// If the package directory is a symlink, treat it as an alias
// of the symlink target (e.g. rust.vim → vim-rust).
fi, err := os.Lstat(filepath.Join(dir, name))
if err != nil {
log.Printf("warning: %s: %v", name, err)
continue
}
if fi.Mode()&os.ModeSymlink != 0 {
target, err := os.Readlink(filepath.Join(dir, name))
if err != nil {
log.Printf("warning: readlink %s: %v", name, err)
continue
}
packages = append(packages, pkgConf{
name: name,
conf: &installerconf.Conf{AliasOf: target},
})
continue
}
conf, err := installerconf.Read(path)
if err != nil {
log.Printf("warning: %s: %v", path, err)
continue
}
packages = append(packages, pkgConf{name: name, conf: conf})
}
sort.Slice(packages, func(i, j int) bool {
return packages[i].name < packages[j].name
})
return packages, nil
}
// refreshPackage does the full pipeline for one package:
// fetch raw → classify → write to fsstore.
func (wc *WebiCache) refreshPackage(ctx context.Context, pkg pkgConf) error {
pkgStart := time.Now()
name := pkg.name
conf := pkg.conf
// Step 1: Fetch raw upstream data to rawcache (unless -no-fetch).
if !wc.NoFetch {
shallow := wc.Shallow
if !shallow {
d, err := rawcache.Open(filepath.Join(wc.RawDir, name))
if err == nil && d.Populated() {
shallow = true
}
}
fetchStart := time.Now()
if err := wc.fetchRaw(ctx, pkg, shallow); err != nil {
return fmt.Errorf("fetch: %w", err)
}
log.Printf(" %s: fetch %s", name, time.Since(fetchStart))
}
// Step 2: Classify raw data into assets, tag variants, apply config.
classifyStart := time.Now()
d, err := rawcache.Open(filepath.Join(wc.RawDir, name))
if err != nil {
return fmt.Errorf("rawcache open: %w", err)
}
// Open supplementary gittag raw cache if available (for packages with
// git_url that use a non-gittag source type like servicemandist).
var gitTagDir *rawcache.Dir
if conf.GitURL != "" && conf.Source != "gittag" {
gd, gdErr := rawcache.Open(filepath.Join(wc.RawDir, "_gittag", name))
if gdErr == nil && gd.Populated() {
gitTagDir = gd
}
}
assets, err := classifypkg.Package(name, conf, d, gitTagDir)
if err != nil {
return fmt.Errorf("classify: %w", err)
}
classifyDur := time.Since(classifyStart)
// Step 3: Write to fsstore.
writeStart := time.Now()
tx, err := wc.Store.BeginRefresh(ctx, name)
if err != nil {
return fmt.Errorf("begin refresh: %w", err)
}
if err := tx.Put(assets); err != nil {
tx.Rollback()
return fmt.Errorf("put: %w", err)
}
if err := tx.Commit(ctx); err != nil {
return fmt.Errorf("commit: %w", err)
}
writeDur := time.Since(writeStart)
log.Printf(" %s: %d assets (classify %s, write %s, total %s)",
name, len(assets), classifyDur, writeDur, time.Since(pkgStart))
return nil
}
// --- Fetch raw ---
func (wc *WebiCache) fetchRaw(ctx context.Context, pkg pkgConf, shallow bool) error {
switch pkg.conf.Source {
case "github", "githubsource":
if err := wc.fetchGitHub(ctx, pkg.name, pkg.conf, shallow); err != nil {
return err
}
case "nodedist":
return wc.fetchNodeDist(ctx, pkg.name, pkg.conf)
case "gittag":
return wc.fetchGitTag(ctx, pkg.name, pkg.conf, shallow)
case "gitea":
return wc.fetchGitea(ctx, pkg.name, pkg.conf, shallow)
case "chromedist":
return fetchChromeDist(ctx, wc.Client, wc.RawDir, pkg.name)
case "flutterdist":
return fetchFlutterDist(ctx, wc.Client, wc.RawDir, pkg.name)
case "golang":
return fetchGolang(ctx, wc.Client, wc.RawDir, pkg.name)
case "gpgdist":
return fetchGPGDist(ctx, wc.Client, wc.RawDir, pkg.name)
case "hashicorp":
return fetchHashiCorp(ctx, wc.Client, wc.RawDir, pkg.name, pkg.conf)
case "iterm2dist":
return fetchITerm2Dist(ctx, wc.Client, wc.RawDir, pkg.name)
case "juliadist":
return fetchJuliaDist(ctx, wc.Client, wc.RawDir, pkg.name)
case "mariadbdist":
return fetchMariaDBDist(ctx, wc.Client, wc.RawDir, pkg.name)
case "servicemandist":
if err := servicemandist.Fetch(ctx, wc.Client, wc.RawDir, pkg.name, wc.Auth, shallow); err != nil {
return err
}
case "zigdist":
return fetchZigDist(ctx, wc.Client, wc.RawDir, pkg.name)
default:
log.Printf(" %s: source %q not yet supported, skipping", pkg.name, pkg.conf.Source)
return nil
}
// For non-gittag sources with a git_url, also clone the repo to get
// commit hashes. Git entries are classified from this data in
// refreshPackage, not from the main raw cache.
if pkg.conf.GitURL != "" && pkg.conf.Source != "gittag" {
gitShallow := shallow
if !wc.Shallow {
gd, gdErr := rawcache.Open(filepath.Join(wc.RawDir, "_gittag", pkg.name))
if gdErr == nil && !gd.Populated() {
gitShallow = false
}
}
if err := wc.fetchGitTagSupplementary(ctx, pkg.name, pkg.conf.GitURL, gitShallow); err != nil {
log.Printf(" %s: supplementary gittag fetch: %v", pkg.name, err)
}
}
return nil
}
// fetchGitTagSupplementary clones a git repo to get commit hashes for
// packages that use a non-gittag source type (servicemandist, githubsource)
// but also have a git_url for source installs.
func (wc *WebiCache) fetchGitTagSupplementary(ctx context.Context, pkgName, gitURL string, shallow bool) error {
d, err := rawcache.Open(filepath.Join(wc.RawDir, "_gittag", pkgName))
if err != nil {
return err
}
repoDir := filepath.Join(wc.RawDir, "_repos")
os.MkdirAll(repoDir, 0o755)
for batch, err := range gittag.Fetch(ctx, gitURL, repoDir) {
if err != nil {
return err
}
for _, entry := range batch {
tag := entry.Version
if tag == "" {
tag = "HEAD-" + entry.CommitHash
}
data, _ := json.Marshal(entry)
d.Merge(tag, data)
}
if shallow {
break
}
}
return nil
}
func (wc *WebiCache) fetchGitHub(ctx context.Context, pkgName string, conf *installerconf.Conf, shallow bool) error {
owner, repo := conf.Owner, conf.Repo
if owner == "" || repo == "" {
return fmt.Errorf("missing owner or repo")
}
d, err := rawcache.Open(filepath.Join(wc.RawDir, pkgName))
if err != nil {
return err
}
tagPrefix := conf.TagPrefix
for batch, err := range github.Fetch(ctx, wc.Client, owner, repo, wc.Auth) {
if err != nil {
return fmt.Errorf("github %s/%s: %w", owner, repo, err)
}
for _, rel := range batch {
if rel.Draft {
continue
}
tag := rel.TagName
if tagPrefix != "" && !strings.HasPrefix(tag, tagPrefix) {
continue
}
data, _ := json.Marshal(rel)
d.Merge(tag, data)
}
if shallow {
break
}
}
return nil
}
func (wc *WebiCache) fetchNodeDist(ctx context.Context, pkgName string, conf *installerconf.Conf) error {
baseURL := conf.BaseURL
if baseURL == "" {
return fmt.Errorf("missing url")
}
d, err := rawcache.Open(filepath.Join(wc.RawDir, pkgName))
if err != nil {
return err
}
// Fetch from primary URL. Tag with "official/" prefix so unofficial
// entries for the same version don't overwrite.
for batch, err := range nodedist.Fetch(ctx, wc.Client, baseURL) {
if err != nil {
return err
}
for _, entry := range batch {
data, _ := json.Marshal(entry)
d.Merge("official/"+entry.Version, data)
}
}
// Fetch from unofficial URL if configured (e.g. Node.js unofficial builds
// which add musl, riscv64, loong64 targets).
if unofficialURL := conf.Extra["unofficial_url"]; unofficialURL != "" {
for batch, err := range nodedist.Fetch(ctx, wc.Client, unofficialURL) {
if err != nil {
log.Printf("warning: %s unofficial fetch: %v", pkgName, err)
break
}
for _, entry := range batch {
data, _ := json.Marshal(entry)
d.Merge("unofficial/"+entry.Version, data)
}
}
}
return nil
}
func (wc *WebiCache) fetchGitTag(ctx context.Context, pkgName string, conf *installerconf.Conf, shallow bool) error {
gitURL := conf.BaseURL
if gitURL == "" {
return fmt.Errorf("missing url")
}
d, err := rawcache.Open(filepath.Join(wc.RawDir, pkgName))
if err != nil {
return err
}
repoDir := filepath.Join(wc.RawDir, "_repos")
os.MkdirAll(repoDir, 0o755)
for batch, err := range gittag.Fetch(ctx, gitURL, repoDir) {
if err != nil {
return err
}
for _, entry := range batch {
tag := entry.Version
if tag == "" {
tag = "HEAD-" + entry.CommitHash
}
data, _ := json.Marshal(entry)
d.Merge(tag, data)
}
if shallow {
break
}
}
return nil
}
func (wc *WebiCache) fetchGitea(ctx context.Context, pkgName string, conf *installerconf.Conf, shallow bool) error {
baseURL, owner, repo := conf.BaseURL, conf.Owner, conf.Repo
if baseURL == "" || owner == "" || repo == "" {
return fmt.Errorf("missing base_url, owner, or repo")
}
d, err := rawcache.Open(filepath.Join(wc.RawDir, pkgName))
if err != nil {
return err
}
for batch, err := range gitea.Fetch(ctx, wc.Client, baseURL, owner, repo, nil) {
if err != nil {
return err
}
for _, rel := range batch {
if rel.Draft {
continue
}
data, _ := json.Marshal(rel)
d.Merge(rel.TagName, data)
}
if shallow {
break
}
}
return nil
}
func fetchChromeDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
if err != nil {
return err
}
for batch, err := range chromedist.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("chromedist: %w", err)
}
for _, ver := range batch {
data, _ := json.Marshal(ver)
d.Merge(ver.Version, data)
}
}
return nil
}
func fetchFlutterDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
if err != nil {
return err
}
for batch, err := range flutterdist.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("flutterdist: %w", err)
}
for _, rel := range batch {
// Key by version+channel+os for uniqueness.
key := rel.Version + "-" + rel.Channel + "-" + rel.OS
data, _ := json.Marshal(rel)
d.Merge(key, data)
}
}
return nil
}
func fetchGolang(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
if err != nil {
return err
}
for batch, err := range golang.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("golang: %w", err)
}
for _, rel := range batch {
data, _ := json.Marshal(rel)
d.Merge(rel.Version, data)
}
}
return nil
}
func fetchGPGDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
if err != nil {
return err
}
for batch, err := range gpgdist.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("gpgdist: %w", err)
}
for _, entry := range batch {
data, _ := json.Marshal(entry)
d.Merge(entry.Version, data)
}
}
return nil
}
func fetchHashiCorp(ctx context.Context, client *http.Client, rawDir, pkgName string, conf *installerconf.Conf) error {
product := conf.Repo
if product == "" {
product = pkgName
}
d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
if err != nil {
return err
}
for idx, err := range hashicorp.Fetch(ctx, client, product) {
if err != nil {
return fmt.Errorf("hashicorp %s: %w", product, err)
}
for ver, vdata := range idx.Versions {
data, _ := json.Marshal(vdata)
d.Merge(ver, data)
}
}
return nil
}
func fetchITerm2Dist(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
if err != nil {
return err
}
for batch, err := range iterm2dist.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("iterm2dist: %w", err)
}
for _, entry := range batch {
key := entry.Version
if entry.Channel == "beta" {
key += "-beta"
}
data, _ := json.Marshal(entry)
d.Merge(key, data)
}
}
return nil
}
func fetchJuliaDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
if err != nil {
return err
}
for batch, err := range juliadist.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("juliadist: %w", err)
}
for _, rel := range batch {
data, _ := json.Marshal(rel)
d.Merge(rel.Version, data)
}
}
return nil
}
func fetchMariaDBDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
if err != nil {
return err
}
for batch, err := range mariadbdist.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("mariadbdist: %w", err)
}
for _, rel := range batch {
data, _ := json.Marshal(rel)
d.Merge(rel.ReleaseID, data)
}
}
return nil
}
func fetchZigDist(ctx context.Context, client *http.Client, rawDir, pkgName string) error {
d, err := rawcache.Open(filepath.Join(rawDir, pkgName))
if err != nil {
return err
}
for batch, err := range zigdist.Fetch(ctx, client) {
if err != nil {
return fmt.Errorf("zigdist: %w", err)
}
for _, rel := range batch {
data, _ := json.Marshal(rel)
d.Merge(rel.Version, data)
}
}
return nil
}

View File

@@ -1 +0,0 @@
github_releases = kivikakk/comrak

View File

@@ -1 +0,0 @@
github_releases = sstadick/crabz

View File

@@ -1 +0,0 @@
github_releases = rs/curlie

View File

@@ -1 +0,0 @@
github_releases = dashpay/dash

View File

@@ -1 +0,0 @@
alias_of = dashcore

View File

@@ -1 +0,0 @@
github_releases = dashhive/dashmsg

View File

@@ -1 +0,0 @@
github_releases = dandavison/delta

View File

@@ -1 +0,0 @@
github_releases = denoland/deno

View File

@@ -1,121 +0,0 @@
# Installer Archive Patterns
Every package falls into one of these archive structure patterns. When writing
or modifying an `install.sh`, identify the pattern first — it determines the
extraction and installation strategy.
## Pattern A: Bare Binary in Archive
Archive contains the binary (and maybe LICENSE/README) at the top level.
Examples: awless, caddy, cilium, curlie, dashmsg, deno, dotenv, dotenv-linter,
ffuf, fzf, gitdeploy, gprox, grype, hugo, hugo-extended, k9s, keypairs, koji,
lf, monorel, ots, runzip, sclient, sqlc, sqlpkg, sttr, terraform, uuidv7, xcaddy
Install: extract, move binary to `~/.local/opt/{pkg}-{ver}/bin/{binary}`, symlink.
## Pattern B: Subdirectory with Binary Only
Archive contains a version-named directory wrapping the binary and docs.
Examples: delta, hexyl, kubectx, kubens, shellcheck, trip, xsv
Typical directory naming: `{tool}-{ver}-{triplet}/`
Install: extract, find binary in subdirectory, move to opt, symlink.
Special cases:
- `pathman`: bare binary named with full release tag (needs rename)
- `yq`: binary named with platform suffix `yq_linux_amd64` (needs rename)
## Pattern C: Binary + Completions + Man Pages
Archive includes shell completions and/or man pages alongside the binary.
| Package | Completions Dir | Man Page |
|---------|----------------|----------|
| bat | `autocomplete/` | `bat.1` |
| fd | `autocomplete/{fd.bash,.fish,_fd}` | `fd.1` |
| goreleaser | `completions/{.bash,.fish,.zsh}` | `manpages/*.1.gz` |
| lsd | `autocomplete/{lsd.bash-completion,.fish,_lsd}` | `lsd.1` |
| rg | `complete/{rg.bash,.fish,_rg}` | `doc/rg.1` |
| sd | `completions/{sd.bash,.fish,_sd}` | `sd.1` |
| watchexec | `completions/{bash,fish,zsh}` | `watchexec.1` |
| zoxide | `completions/{zoxide.bash,.fish,_zoxide}` | `man/man1/zoxide*.1` |
Install: extract, install binary, install completions to standard dirs, install
man pages. Completion naming varies: `autocomplete/`, `completions/`, `complete/`.
## Pattern D: Binary + Libraries
Complex packages that bundle shared libraries.
| Package | Layout |
|---------|--------|
| ollama (Linux) | `bin/ollama` + `lib/ollama/{cuda_v12,cuda_v13,vulkan}/` |
| pg/postgres/psql | `bin/psql` + `lib/{libpq,libz,...}.so` + `include/` |
| sass | `dart-sass/sass` (wrapper) + `dart-sass/src/{dart,sass.snapshot}` |
| syncthing | `syncthing-{triplet}-{ver}/syncthing` + `etc/{systemd,...}/` |
| xz | `xz-{ver}-{triplet}/xz` + `xz-{ver}-{triplet}/unxz` |
Install: extract entire directory tree into opt, symlink binary.
## Pattern E: FHS-like Layout (bin/ + share/)
Archive already follows standard layout.
| Package | Layout |
|---------|--------|
| gh | `gh_{ver}_{os}_{arch}/bin/gh` + `share/man/man1/*.1` |
| pandoc | `pandoc-{ver}/bin/{pandoc,...}` + `share/man/man1/*.1.gz` |
Install: extract directly into opt (already correct layout).
## Pattern G: Full SDK/Toolchain
Self-contained toolchain with compiler, runtime, standard library.
| Package | Layout |
|---------|--------|
| cmake | `cmake-{ver}-{os}-{arch}/bin/{cmake,ctest,...}` + `share/` + `man/` |
| tinygo | `tinygo/bin/tinygo` + `tinygo/src/` + `tinygo/targets/` |
| go | `go/bin/{go,gofmt}` + `go/src/` + `go/pkg/` |
| zig | `zig-{os}-{arch}-{ver}/zig` + `lib/` |
| flutter | `flutter/bin/flutter` + full SDK |
| julia | `julia-{ver}/bin/julia` + full SDK |
| node | `node-{ver}-{os}-{arch}/bin/{node,npm,npx}` + `lib/` |
Install: extract entire tree into `~/.local/opt/{pkg}-{ver}/`, symlink `bin/*`.
## Pattern H: .NET Runtime Bundle
Flat archive with hundreds of DLLs.
Example: pwsh — `pwsh` binary + `*.dll` + locale dirs
Install: extract entire directory into opt, symlink primary binary.
## Pattern I: Multi-Binary Distribution
Archive contains multiple related binaries + libs.
| Package | Layout |
|---------|--------|
| dashcore | `dashcore-{ver}/bin/{dashd,dash-cli,...}` + `lib/` + `share/man/` |
| mutagen | `mutagen` + `mutagen-agents.tar.gz` (embedded agent archive) |
Install: extract into opt, symlink primary binary.
## Format Changes Over Time
Most packages have stable formats. Notable structural changes:
| Package | When | Change |
|---------|------|--------|
| sd | 2023 | zip → tar.gz, added completions + man page |
| ollama | 2025-2026 | bare binary → no GitHub release → tar.zst with lib/ |
| deno | 2020-2021 | .gz (gzipped binary) → .zip |
| hugo | 2017-2018 | zip → tar.gz; 2024: macOS → .pkg only |
| gh | 2024 | darwin: tar.gz → .pkg |
| sclient | 2023 | tar.gz → tar.xz |
| watchexec | 2019-2020 | tar.gz → tar.xz |

View File

@@ -1,74 +0,0 @@
# Version & Release Oddities
Non-standard version formats and tag prefixes that affect parsing, sorting,
and classification. The Go classifier and `internal/lexver` must handle all
of these.
## Non-Numeric Tag Prefixes
| Package | Raw Tag | Cleaned | Transform |
|---------|---------|---------|-----------|
| lf | `r21` | `0.21.0` | `r` prefix → prepend `0.` |
| bun | `bun-v1.0.0` | `1.0.0` | Strip `bun-` prefix |
| jq | `jq-1.7` | `1.7` | Strip `jq-` prefix |
| watchexec | `cli-v1.2.3` | `1.2.3` | Strip `cli-` prefix |
| ffmpeg | `b6.0` | `6.0` | Strip `b` prefix |
## Underscore-Delimited Tags
| Package | Raw Tag | Cleaned | Transform |
|---------|---------|---------|-----------|
| postgres | `REL_17_0` | `17.0` | Strip `REL_`, replace `_` with `.` |
| psql | `REL_17_0` | `17.0` | Same as postgres |
## Platform Suffix in Version
| Package | Raw Tag | Cleaned | Transform |
|---------|---------|---------|-----------|
| git (Windows) | `2.41.0.windows.1` | `2.41.0` | Strip `.windows.N` suffix |
## 4-Part Versions
| Package | Example | Notes |
|---------|---------|-------|
| chromedriver | `121.0.6120.0` | Google Chrome's versioning |
| gpg | `2.2.19.0` | 4th segment is build metadata |
## Date-Based Versions
| Package | Notes |
|---------|-------|
| atomicparsley | Date-based version strings |
## Complex Pre-Release Formats
| Package | Example | Notes |
|---------|---------|-------|
| flutter | `2.3.0-16.0.pre` | Extra dots and numeric segments |
| iterm2 | `iTerm2_3_5_0beta17` | Underscores, beta attached → `3.5.0-beta17` |
## Channel Detection
- Node.js: odd major = "current" not LTS (v15, v17, v19, v21, v23)
- Go: `go` prefix stripped (`go1.23.6``1.23.6`)
- Terraform: `-alpha`, `-beta`, `-rc` suffixes → beta channel
## Directory Symlinks (Aliases)
These are directory-level symlinks. They share all files (including
releases.conf) with their target automatically.
```
msvc-runtime → vcruntime
msvcruntime → vcruntime
rust.vim → vim-rust
vc-redist → vcruntime
vc-runtime → vcruntime
vc_redist → vcruntime
vcredist → vcruntime
vcruntime140 → vcruntime
vim-essential → vim-essentials
vim-mouse → vim-gui
vps-myip → myip
xcode-cli → commandlinetools
```

View File

@@ -1 +0,0 @@
github_releases = dotenv-linter/dotenv-linter

View File

@@ -1 +0,0 @@
github_releases = therootcompany/dotenv

View File

@@ -1,2 +0,0 @@
github_sources = BeyondCodeBootcamp/DuckDNS.sh
git_url = https://github.com/BeyondCodeBootcamp/DuckDNS.sh.git

View File

@@ -27,6 +27,22 @@ __init_fd() {
# chmod a+x "$HOME/.local/opt/fd-v8.1.1/bin/fd"
chmod a+x "$pkg_src_cmd"
# install completions if present (autocomplete/{fd.bash,fd.fish,_fd})
if test -d ./fd-*/autocomplete; then
mkdir -p "$pkg_src_dir/share/bash-completion/completions"
mkdir -p "$pkg_src_dir/share/fish/vendor_completions.d"
mkdir -p "$pkg_src_dir/share/zsh/site-functions"
mv ./fd-*/autocomplete/fd.bash "$pkg_src_dir/share/bash-completion/completions/fd" 2>/dev/null || true
mv ./fd-*/autocomplete/fd.fish "$pkg_src_dir/share/fish/vendor_completions.d/fd.fish" 2>/dev/null || true
mv ./fd-*/autocomplete/_fd "$pkg_src_dir/share/zsh/site-functions/_fd" 2>/dev/null || true
fi
# install man page if present
if test -f ./fd-*/fd.1; then
mkdir -p "$pkg_src_dir/share/man/man1"
mv ./fd-*/fd.1 "$pkg_src_dir/share/man/man1/fd.1"
fi
}
}

View File

@@ -1 +0,0 @@
github_releases = sharkdp/fd

View File

@@ -1,4 +0,0 @@
source = ffmpegdist
github_releases = eugeneware/ffmpeg-static
asset_filter = ffmpeg
version_prefix = b

View File

@@ -1 +0,0 @@
github_releases = ffuf/ffuf

View File

@@ -3,19 +3,23 @@ set -e
set -u
if command -v fish > /dev/null; then
if ! test -e ~/.config/fish/config.fish; then
mkdir -p ~/.config/fish
touch ~/.config/fish/config.fish
chmod 0600 ~/.config/fish/config.fish
fi
if [ ! -e ~/.config/fish/config.fish ]; then
mkdir -p ~/.config/fish
touch ~/.config/fish/config.fish
chmod 0600 ~/.config/fish/config.fish
fi
fi
if [ "Darwin" != "$(uname -s)" ]; then
echo "No fish installer for Linux yet. Try this instead:"
echo " sudo apt install -y fish"
exit 1
fi
################
# Install fish #
################
my_os=$(uname -s)
# Every package should define these 6 variables
# shellcheck disable=2034
pkg_cmd_name="fish"
@@ -30,109 +34,70 @@ pkg_src_dir="$HOME/.local/opt/fish-v$WEBI_VERSION"
# shellcheck disable=2034
pkg_src="$pkg_src_cmd"
if test "Darwin" = "${my_os}"; then
pkg_src_cmd="/Applications/fish.app/Contents/Resources/base/usr/local/bin/fish"
# shellcheck disable=2034
pkg_src="${pkg_src_cmd}"
fi
_linux_post_install() {
if ! test -e "$HOME/.local/bin/fish"; then
return 0
fi
echo ""
echo "To set fish as your default shell, run:"
echo " chsh -s $HOME/.local/bin/fish"
echo ""
}
# pkg_install must be defined by every package
_macos_post_install() {
if ! test -e "$HOME/.local/bin/fish"; then
return 0
fi
if ! [ -e "$HOME/.local/bin/fish" ]; then
return 0
fi
echo ""
echo "Trying to set fish as the default shell..."
echo ""
# stop the caching of preferences
killall cfprefsd
echo ""
echo "Trying to set fish as the default shell..."
echo ""
# stop the caching of preferences
killall cfprefsd
# Set default Terminal.app shell to fish
defaults write com.apple.Terminal "Shell" -string "$HOME/.local/bin/fish"
echo "To set 'fish' as the default Terminal.app shell:"
echo " Terminal > Preferences > General > Shells open with:"
echo " $HOME/.local/bin/fish"
echo ""
# Set default Terminal.app shell to fish
defaults write com.apple.Terminal "Shell" -string "$HOME/.local/bin/fish"
echo "To set 'fish' as the default Terminal.app shell:"
echo " Terminal > Preferences > General > Shells open with:"
echo " $HOME/.local/bin/fish"
echo ""
# Set default iTerm2 shell to fish
if test -e "$HOME/Library/Preferences/com.googlecode.iterm2.plist"; then
/usr/libexec/PlistBuddy \
-c "SET ':New Bookmarks:0:Custom Command' 'Custom Shell'" \
"$HOME/Library/Preferences/com.googlecode.iterm2.plist"
/usr/libexec/PlistBuddy \
-c "SET ':New Bookmarks:0:Command' $HOME/.local/bin/fish" \
"$HOME/Library/Preferences/com.googlecode.iterm2.plist"
echo "To set 'fish' as the default iTerm2 shell:"
echo " iTerm2 > Preferences > Profiles > General > Command >"
echo " Custom Shell: $HOME/.local/bin/fish"
echo ""
fi
# Set default iTerm2 shell to fish
if [ -e "$HOME/Library/Preferences/com.googlecode.iterm2.plist" ]; then
/usr/libexec/PlistBuddy \
-c "SET ':New Bookmarks:0:Custom Command' 'Custom Shell'" \
"$HOME/Library/Preferences/com.googlecode.iterm2.plist"
/usr/libexec/PlistBuddy \
-c "SET ':New Bookmarks:0:Command' $HOME/.local/bin/fish" \
"$HOME/Library/Preferences/com.googlecode.iterm2.plist"
echo "To set 'fish' as the default iTerm2 shell:"
echo " iTerm2 > Preferences > Profiles > General > Command >"
echo " Custom Shell: $HOME/.local/bin/fish"
echo ""
fi
killall cfprefsd
killall cfprefsd
}
# always try to reset the default shells
if test "Darwin" = "${my_os}"; then
_macos_post_install
fi
_macos_post_install
pkg_install() {
if test "Darwin" = "${my_os}"; then
rm -rf "/Applications/fish-v${WEBI_VERSION}.app"
mv -f fish*.app "/Applications/fish-v${WEBI_VERSION}.app"
rm -rf /Applications/fish.app
mv "/Applications/fish-v${WEBI_VERSION}.app" "/Applications/fish.app"
return 0
fi
mv fish.app/Contents/Resources/base/usr/local "$HOME/.local/opt/fish-v${WEBI_VERSION}"
mkdir -p "$pkg_src_dir/bin"
mv fish "$pkg_src_dir/bin/"
}
pkg_link() {
if test "Darwin" = "${my_os}"; then
mkdir -p "$HOME/.local/bin"
ln -sf /Applications/fish.app/Contents/Resources/base/usr/local/bin/fish "$HOME/.local/bin/fish"
return 0
fi
rm -rf "$pkg_dst_cmd"
ln -s "$pkg_src_cmd" "$pkg_dst_cmd"
}
pkg_post_install() {
# don't skip what webi would do automatically
webi_post_install
# don't skip what webi would do automatically
webi_post_install
# try again to update default shells, now that all files should exist
if test "Darwin" = "${my_os}"; then
_macos_post_install
else
_linux_post_install
fi
if ! test -e ~/.config/fish/config.fish; then
mkdir -p ~/.config/fish
touch ~/.config/fish/config.fish
chmod 0600 ~/.config/fish/config.fish
fi
# try again to update default shells, now that all files should exist
_macos_post_install
if [ ! -e ~/.config/fish/config.fish ]; then
mkdir -p ~/.config/fish
touch ~/.config/fish/config.fish
chmod 0600 ~/.config/fish/config.fish
fi
}
# pkg_get_current_version is recommended, but (soon) not required
pkg_get_current_version() {
# 'fish --version' has output in this format:
# fish, version 4.3.3
# This trims it down to just the version number:
# 4.3.3
fish --version 2> /dev/null | head -n 1 | cut -d ' ' -f 3
# 'fish --version' has output in this format:
# fish, version 3.1.2
# This trims it down to just the version number:
# 3.1.2
fish --version 2> /dev/null | head -n 1 | cut -d ' ' -f 3
}

View File

@@ -1,2 +0,0 @@
github_releases = fish-shell/fish-shell
exclude = bundledpcre fish-static OpenBeta

View File

@@ -1 +0,0 @@
source = flutterdist

View File

@@ -1 +0,0 @@
github_releases = junegunn/fzf

View File

@@ -23,8 +23,14 @@ __init_gh() {
# ~/.local/opt/gh-v0.99.9/bin
mkdir -p "$(dirname "$pkg_src_cmd")"
# mv ./gh_*/bin/gh ~/.local/opt/gh-v0.99.9/bin/gh
# mv ./gh-*/gh ~/.local/opt/gh-v0.99.9/bin/gh
mv ./"$pkg_cmd_name"*/bin/gh "$pkg_src_cmd"
# install man pages if present
if test -d ./"$pkg_cmd_name"*/share/man; then
mkdir -p "$pkg_src_dir/share"
mv ./"$pkg_cmd_name"*/share/man "$pkg_src_dir/share/man"
fi
}
# pkg_get_current_version is recommended, but (soon) not required

View File

@@ -1 +0,0 @@
github_releases = cli/cli

View File

@@ -1,4 +0,0 @@
github_releases = git-for-windows/git
asset_filter = MinGit
exclude = busybox
variants = installer

View File

@@ -1 +0,0 @@
github_releases = therootcompany/gitdeploy

View File

@@ -1,3 +0,0 @@
github_releases = go-gitea/gitea
exclude = -src- -docs-
variants = gogit

View File

@@ -1 +0,0 @@
alias_of = gpg

5
go.mod
View File

@@ -1,5 +0,0 @@
module github.com/webinstall/webi-installers
go 1.26.1
require github.com/joho/godotenv v1.5.1

2
go.sum
View File

@@ -1,2 +0,0 @@
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=

View File

@@ -1 +0,0 @@
source = golang

View File

@@ -1 +0,0 @@
alias_of = go

View File

@@ -23,8 +23,25 @@ __init_goreleaser() {
# ~/.local/opt/goreleaser-v1.21.2/bin
mkdir -p "$(dirname "$pkg_src_cmd")"
# mv ./goreleaser ~/.local/opt/goreleaser-v1.21.2/bin/goreleaser
# mv ./goreleaser-*/goreleaser ~/.local/opt/goreleaser-v1.21.2/bin/goreleaser
mv ./goreleaser "$pkg_src_cmd"
# install completions if present (completions/{goreleaser.bash,.fish,.zsh})
if test -d ./completions; then
mkdir -p "$pkg_src_dir/share/bash-completion/completions"
mkdir -p "$pkg_src_dir/share/fish/vendor_completions.d"
mkdir -p "$pkg_src_dir/share/zsh/site-functions"
mv ./completions/goreleaser.bash "$pkg_src_dir/share/bash-completion/completions/goreleaser" 2>/dev/null || true
mv ./completions/goreleaser.fish "$pkg_src_dir/share/fish/vendor_completions.d/goreleaser.fish" 2>/dev/null || true
mv ./completions/goreleaser.zsh "$pkg_src_dir/share/zsh/site-functions/_goreleaser" 2>/dev/null || true
fi
# install man page if present (manpages/goreleaser.1.gz)
if test -d ./manpages; then
mkdir -p "$pkg_src_dir/share/man/man1"
mv ./manpages/*.1.gz "$pkg_src_dir/share/man/man1/" 2>/dev/null || true
mv ./manpages/*.1 "$pkg_src_dir/share/man/man1/" 2>/dev/null || true
fi
}
# pkg_get_current_version is recommended, but (soon) not required

View File

@@ -1 +0,0 @@
github_releases = goreleaser/goreleaser

View File

@@ -1 +0,0 @@
source = gpgdist

View File

@@ -1 +0,0 @@
github_releases = creedasaurus/gprox

View File

@@ -1 +0,0 @@
github_releases = anchore/grype

View File

@@ -1 +0,0 @@
github_releases = sharkdp/hexyl

View File

@@ -1,3 +0,0 @@
github_releases = gohugoio/hugo
asset_filter = extended
exclude = Linux-64bit

View File

@@ -1,2 +0,0 @@
github_releases = gohugoio/hugo
exclude = extended Linux-64bit

View File

@@ -1,168 +0,0 @@
// Package buildmeta is the shared vocabulary for Webi's build targets.
//
// Every package that deals with OS, architecture, libc, archive format, or
// release channel imports these types instead of passing raw strings. This
// prevents typos like "darwn" from compiling and gives a single place to
// enumerate what Webi supports.
package buildmeta
// OS represents a target operating system.
type OS string
const (
OSAny OS = "ANYOS"
OSDarwin OS = "darwin"
OSLinux OS = "linux"
OSWindows OS = "windows"
OSFreeBSD OS = "freebsd"
OSOpenBSD OS = "openbsd"
OSNetBSD OS = "netbsd"
OSDragonFly OS = "dragonfly"
OSSunOS OS = "sunos"
OSIllumos OS = "illumos"
OSSolaris OS = "solaris"
OSAIX OS = "aix"
OSAndroid OS = "android"
OSPlan9 OS = "plan9"
// POSIX compatibility levels — used when a package is a shell script
// or otherwise OS-independent for POSIX systems.
OSPosix2017 OS = "posix_2017"
OSPosix2024 OS = "posix_2024"
)
// Arch represents a target CPU architecture.
type Arch string
const (
ArchAny Arch = "ANYARCH"
ArchAMD64 Arch = "x86_64" // baseline (v1)
ArchAMD64v2 Arch = "x86_64_v2" // +SSE4, +POPCNT, etc.
ArchAMD64v3 Arch = "x86_64_v3" // +AVX2, +BMI, etc.
ArchAMD64v4 Arch = "x86_64_v4" // +AVX-512
ArchARM64 Arch = "aarch64"
ArchARMv7 Arch = "armv7"
ArchARMv6 Arch = "armv6"
ArchARMv5 Arch = "armv5"
ArchX86 Arch = "x86"
ArchPPC64LE Arch = "ppc64le"
ArchPPC64 Arch = "ppc64"
ArchPPC Arch = "powerpc" // 32-bit PowerPC (unsupported by webi, used to prevent gnueabihf over-matching)
ArchRISCV64 Arch = "riscv64"
ArchS390X Arch = "s390x"
ArchLoong64 Arch = "loong64"
ArchMIPS64LE Arch = "mips64le"
ArchMIPS64 Arch = "mips64"
ArchMIPS64R6EL Arch = "mips64r6el"
ArchMIPS64R6 Arch = "mips64r6"
ArchMIPSLE Arch = "mipsle"
ArchMIPS Arch = "mips"
// Universal (fat) binary architectures for macOS.
ArchUniversal1 Arch = "universal1" // PPC + x86 (Rosetta 1 era)
ArchUniversal2 Arch = "universal2" // x86_64 + ARM64 (Rosetta 2 era)
)
// Libc represents the C library a binary is linked against.
type Libc string
const (
LibcNone Libc = "none" // statically linked or no libc dependency (Go, Zig, etc.)
LibcGNU Libc = "gnu" // requires glibc (most Linux distros)
LibcMusl Libc = "musl" // requires musl (Alpine, some Docker images)
LibcMSVC Libc = "msvc" // Microsoft Visual C++ runtime
)
// Format represents an archive or package format.
type Format string
const (
FormatTarGz Format = ".tar.gz"
FormatTarXz Format = ".tar.xz"
FormatTarZst Format = ".tar.zst"
FormatTarBz2 Format = ".tar.bz2"
FormatZip Format = ".zip"
FormatGz Format = ".gz"
FormatXz Format = ".xz"
FormatZst Format = ".zst"
FormatExe Format = ".exe"
FormatExeXz Format = ".exe.xz"
FormatMSI Format = ".msi"
FormatDMG Format = ".dmg"
FormatPkg Format = ".pkg"
FormatAppZip Format = ".app.zip"
Format7z Format = ".7z"
FormatDeb Format = ".deb"
FormatRPM Format = ".rpm"
FormatSnap Format = ".snap"
FormatAppx Format = ".appx"
FormatAPK Format = ".apk"
FormatAppImage Format = ".AppImage"
FormatSh Format = ".sh"
FormatGit Format = ".git"
)
// Channel represents a release stability channel.
type Channel string
const (
ChannelStable Channel = "stable"
ChannelLatest Channel = "latest"
ChannelRC Channel = "rc"
ChannelPreview Channel = "preview"
ChannelBeta Channel = "beta"
ChannelAlpha Channel = "alpha"
ChannelDev Channel = "dev"
)
// Target represents a fully resolved build target.
type Target struct {
OS OS
Arch Arch
Libc Libc
}
// Triplet returns the canonical "os-arch-libc" string.
func (t Target) Triplet() string {
return string(t.OS) + "-" + string(t.Arch) + "-" + string(t.Libc)
}
// CompatArches returns the architectures that the given OS+arch
// combination can execute, ordered from most specific to least.
// The input arch is always first.
//
// These are OS-level facts (hardware + translation layer), not
// package-specific. Per-package overrides belong in installer config.
func CompatArches(os OS, arch Arch) []Arch {
switch os {
case OSDarwin:
switch arch {
case ArchARM64:
// Rosetta 2: Apple Silicon runs x86_64 binaries.
return []Arch{ArchARM64, ArchUniversal2, ArchAMD64}
case ArchAMD64:
return []Arch{ArchAMD64, ArchUniversal2, ArchX86}
}
case OSWindows:
switch arch {
case ArchARM64:
// Windows on ARM emulates x86_64 and x86.
return []Arch{ArchARM64, ArchAMD64, ArchX86}
}
}
// Micro-architecture fallbacks (universal across all OSes).
switch arch {
case ArchAMD64v4:
return []Arch{ArchAMD64v4, ArchAMD64v3, ArchAMD64v2, ArchAMD64}
case ArchAMD64v3:
return []Arch{ArchAMD64v3, ArchAMD64v2, ArchAMD64}
case ArchAMD64v2:
return []Arch{ArchAMD64v2, ArchAMD64}
case ArchARMv7:
return []Arch{ArchARMv7, ArchARMv6}
}
return []Arch{arch}
}

View File

@@ -1,279 +0,0 @@
// Package classify extracts build targets from release asset filenames.
//
// Standard toolchains (goreleaser, cargo-dist, zig build) produce predictable
// filenames like "tool_0.1.0_linux_amd64.tar.gz" or
// "tool-0.1.0-x86_64-unknown-linux-musl.tar.gz". This package matches those
// patterns directly using regex, avoiding heuristic guessing.
//
// Detection order matters: architectures are checked longest-first to prevent
// "x86" from matching inside "x86_64", and OS checks use word boundaries.
package classify
import (
"path"
"regexp"
"slices"
"strings"
"github.com/webinstall/webi-installers/internal/buildmeta"
)
// Result holds the classification of an asset filename.
type Result struct {
OS buildmeta.OS
Arch buildmeta.Arch
Libc buildmeta.Libc
Format buildmeta.Format
}
// Target returns the build target (OS + Arch + Libc).
func (r Result) Target() buildmeta.Target {
return buildmeta.Target{OS: r.OS, Arch: r.Arch, Libc: r.Libc}
}
// Filename classifies a release asset filename, returning the detected
// OS, architecture, libc, and archive format. Undetected fields are empty.
//
// OS is detected first because it can influence arch interpretation.
// For example, "windows-arm" in modern releases means ARM64, while
// bare "arm" on Linux historically means ARMv6.
func Filename(name string) Result {
lower := strings.ToLower(name)
os := detectOS(lower)
arch := detectArch(lower)
format := detectFormat(lower)
// .deb, .rpm, .snap are Linux-only package formats.
if os == "" && (format == buildmeta.FormatDeb || format == buildmeta.FormatRPM || format == buildmeta.FormatSnap) {
os = buildmeta.OSLinux
}
// .app.zip and .dmg are macOS-only formats.
if os == "" && (format == buildmeta.FormatAppZip || format == buildmeta.FormatDMG) {
os = buildmeta.OSDarwin
}
return Result{
OS: os,
Arch: arch,
Libc: detectLibc(lower),
Format: format,
}
}
// b is a boundary: start/end of string or a non-alphanumeric separator.
// Go's RE2 doesn't support \b, so we use this instead.
const b = `(?:^|[^a-zA-Z0-9])`
const bEnd = `(?:[^a-zA-Z0-9]|$)`
// --- OS detection ---
var osPatterns = []struct {
os buildmeta.OS
pattern *regexp.Regexp
}{
// macos[\d.]* matches versioned names like "macos10.10", "macos11", "macos12.0" (cmake naming).
{buildmeta.OSDarwin, regexp.MustCompile(`(?i)(?:` + b + `(?:darwin|macos[\d.]*|macosx[\d.]*|osx[\d.]*|os-x|apple)` + bEnd + `|` + b + `mac` + bEnd + `)`)},
// linux[\d.]* matches versioned names like "linux64", "linux32" (chromedriver/dashcore naming).
{buildmeta.OSLinux, regexp.MustCompile(`(?i)` + b + `linux[\d.]*` + bEnd)},
{buildmeta.OSWindows, regexp.MustCompile(`(?i)` + b + `(?:windows|win(?:32|64|x64|dows)?)` + bEnd + `|\.exe(?:\.xz)?$|\.msi$`)},
// freebsd\d* matches versioned names like "freebsd13", "freebsd14" (Gitea naming).
{buildmeta.OSFreeBSD, regexp.MustCompile(`(?i)` + b + `freebsd\d*` + bEnd)},
{buildmeta.OSOpenBSD, regexp.MustCompile(`(?i)` + b + `openbsd` + bEnd)},
{buildmeta.OSNetBSD, regexp.MustCompile(`(?i)` + b + `netbsd` + bEnd)},
{buildmeta.OSDragonFly, regexp.MustCompile(`(?i)` + b + `dragonfly(?:bsd)?` + bEnd)},
// solaris, illumos, and sunos are distinct OS values in the Node build-classifier.
// Keep them separate so the legacy cache matches what the classifier extracts.
{buildmeta.OSSolaris, regexp.MustCompile(`(?i)` + b + `solaris` + bEnd)},
{buildmeta.OSIllumos, regexp.MustCompile(`(?i)` + b + `illumos` + bEnd)},
{buildmeta.OSSunOS, regexp.MustCompile(`(?i)` + b + `sunos` + bEnd)},
{buildmeta.OSAIX, regexp.MustCompile(`(?i)` + b + `aix` + bEnd)},
{buildmeta.OSAndroid, regexp.MustCompile(`(?i)` + b + `android` + bEnd)},
{buildmeta.OSPlan9, regexp.MustCompile(`(?i)` + b + `plan9` + bEnd)},
}
func detectOS(lower string) buildmeta.OS {
for _, p := range osPatterns {
if p.pattern.MatchString(lower) {
return p.os
}
}
return ""
}
// --- Arch detection ---
// Order matters: check longer/more-specific patterns first.
var archPatterns = []struct {
arch buildmeta.Arch
pattern *regexp.Regexp
}{
// Universal/fat binaries before specific arches.
{buildmeta.ArchUniversal2, regexp.MustCompile(`(?i)` + b + `(?:universal2?|fat)` + bEnd)},
// amd64 micro-levels before baseline — "amd64v3" must not fall through to amd64.
// amd64_?vN: underscore optional but no dash — dash is ambiguous with version numbers
// (e.g. syncthing "amd64-v2.0.5" where v2 is the release version, not an arch level).
{buildmeta.ArchAMD64v4, regexp.MustCompile(`(?i)(?:x86[_-]64[_-]v4|amd64_?v4|v4-amd64)`)},
{buildmeta.ArchAMD64v3, regexp.MustCompile(`(?i)(?:x86[_-]64[_-]v3|amd64_?v3|v3-amd64)`)},
{buildmeta.ArchAMD64v2, regexp.MustCompile(`(?i)(?:x86[_-]64[_-]v2|amd64_?v2|v2-amd64)`)},
// amd64 baseline before x86 — "x86_64" must not match as x86.
{buildmeta.ArchAMD64, regexp.MustCompile(`(?i)(?:x86[_-]64|amd64|x64|win64)`)},
// arm64 before armv7/armv6 — "aarch64" must not match as arm.
{buildmeta.ArchARM64, regexp.MustCompile(`(?i)(?:aarch64|arm64|armv8)`)},
{buildmeta.ArchARMv7, regexp.MustCompile(`(?i)(?:armv7l?|arm-?v7|arm7|arm32|armhf)`)},
// armel and gnueabihf are ARMv6 soft/hard-float ABI names used in Debian and Rust triplets.
{buildmeta.ArchARMv6, regexp.MustCompile(`(?i)(?:armv6l?|arm-?v6|aarch32|armel|gnueabihf|` + b + `arm` + bEnd + `)`)},
{buildmeta.ArchARMv5, regexp.MustCompile(`(?i)(?:armv5)`)},
// powerpc64le/ppc64le before powerpc64/ppc64 before powerpc32.
// The longer powerpc* forms must come first to prevent shorter matches from
// winning. All powerpc entries must appear BEFORE ARM patterns — otherwise
// "powerpc-linux-gnueabihf" would match gnueabihf → ARMv6.
// ppc64el is an alternative spelling used in Debian/Ubuntu.
{buildmeta.ArchPPC64LE, regexp.MustCompile(`(?i)(?:powerpc64le|ppc64le|ppc64el)`)},
{buildmeta.ArchPPC64, regexp.MustCompile(`(?i)(?:powerpc64|ppc64)`)},
// powerpc (32-bit): webi does not serve powerpc32, but we must classify it
// here to prevent the gnueabihf suffix from matching the ARMv6 pattern.
{buildmeta.ArchPPC, regexp.MustCompile(`(?i)` + b + `powerpc` + bEnd)},
{buildmeta.ArchRISCV64, regexp.MustCompile(`(?i)riscv64`)},
{buildmeta.ArchS390X, regexp.MustCompile(`(?i)s390x`)},
{buildmeta.ArchLoong64, regexp.MustCompile(`(?i)loong(?:arch)?64`)},
// mips64r6 before mips64 — "mips64r6" contains "mips64" as a prefix.
{buildmeta.ArchMIPS64R6EL, regexp.MustCompile(`(?i)mips64r6e(?:l|le)`)},
{buildmeta.ArchMIPS64R6, regexp.MustCompile(`(?i)mips64r6`)},
{buildmeta.ArchMIPS64LE, regexp.MustCompile(`(?i)mips64(?:el|le)`)},
{buildmeta.ArchMIPS64, regexp.MustCompile(`(?i)mips64`)},
{buildmeta.ArchMIPSLE, regexp.MustCompile(`(?i)mips(?:el|le)`)},
{buildmeta.ArchMIPS, regexp.MustCompile(`(?i)` + b + `mips` + bEnd)},
// x86 last — must not steal x86_64.
{buildmeta.ArchX86, regexp.MustCompile(`(?i)(?:` + b + `x86` + bEnd + `|i[3-6]86|ia32|win32|` + b + `386` + bEnd + `)`)},
}
func detectArch(lower string) buildmeta.Arch {
for _, p := range archPatterns {
if p.pattern.MatchString(lower) {
return p.arch
}
}
return ""
}
// --- Libc detection ---
var (
reMusl = regexp.MustCompile(`(?i)` + b + `musl` + bEnd)
reGNU = regexp.MustCompile(`(?i)` + b + `(?:gnu|glibc)` + bEnd)
reMSVC = regexp.MustCompile(`(?i)` + b + `msvc` + bEnd)
reStatic = regexp.MustCompile(`(?i)` + b + `static` + bEnd)
)
func detectLibc(lower string) buildmeta.Libc {
switch {
case reMusl.MatchString(lower):
return buildmeta.LibcMusl
case reGNU.MatchString(lower):
return buildmeta.LibcGNU
case reMSVC.MatchString(lower):
return buildmeta.LibcMSVC
case reStatic.MatchString(lower):
return buildmeta.LibcNone
}
return ""
}
// --- Format detection ---
// formatSuffixes maps file extensions to formats, longest first.
var formatSuffixes = []struct {
suffix string
format buildmeta.Format
}{
{".tar.gz", buildmeta.FormatTarGz},
{".tar.xz", buildmeta.FormatTarXz},
{".tar.zst", buildmeta.FormatTarZst},
{".tar.bz2", buildmeta.FormatTarBz2},
{".exe.xz", buildmeta.FormatExeXz},
{".app.zip", buildmeta.FormatAppZip},
{".tgz", buildmeta.FormatTarGz},
{".zip", buildmeta.FormatZip},
{".gz", buildmeta.FormatGz},
{".xz", buildmeta.FormatXz},
{".zst", buildmeta.FormatZst},
{".7z", buildmeta.Format7z},
{".exe", buildmeta.FormatExe},
{".msi", buildmeta.FormatMSI},
{".dmg", buildmeta.FormatDMG},
{".deb", buildmeta.FormatDeb},
{".rpm", buildmeta.FormatRPM},
{".snap", buildmeta.FormatSnap},
{".appx", buildmeta.FormatAppx},
{".apk", buildmeta.FormatAPK},
{".AppImage", buildmeta.FormatAppImage},
{".pkg", buildmeta.FormatPkg},
}
func detectFormat(lower string) buildmeta.Format {
// Use the base name to avoid directory separators confusing suffix matching.
base := path.Base(lower)
for _, s := range formatSuffixes {
if strings.HasSuffix(base, s.suffix) {
return s.format
}
}
return ""
}
// IsMetaAsset returns true if the filename is a non-installable meta file
// (checksums, signatures, source tarballs, documentation, etc.).
func IsMetaAsset(name string) bool {
lower := strings.ToLower(name)
for _, suffix := range []string{
".txt",
".sha256",
".sha256sum",
".sha512",
".sha512sum",
".md5",
".md5sum",
".sig",
".asc",
".pem",
".sbom",
".spdx",
".json.sig",
".sigstore",
".minisig",
"_src.tar.gz",
"_src.tar.xz",
"_src.zip",
"-src.tar.gz",
".src.tar.gz",
"-src.tar.xz",
"-src.zip",
".d.ts",
".pub",
".bsdiff",
".flatpak",
} {
if strings.HasSuffix(lower, suffix) {
return true
}
}
for _, substr := range []string{
"checksums",
"sha256sum",
"sha512sum",
"buildable-artifact",
".LICENSE",
".README",
} {
if strings.Contains(lower, substr) {
return true
}
}
return slices.Contains([]string{
"install.sh",
"install.ps1",
"compat.json",
"b3sums",
"dist-manifest.json",
}, lower)
}

View File

@@ -1,352 +0,0 @@
package classify_test
import (
"testing"
"github.com/webinstall/webi-installers/internal/buildmeta"
"github.com/webinstall/webi-installers/internal/classify"
)
func TestFilename(t *testing.T) {
tests := []struct {
name string
input string
wantOS buildmeta.OS
arch buildmeta.Arch
libc buildmeta.Libc
format buildmeta.Format
}{
// Goreleaser-style
{
name: "goreleaser linux amd64 tar.gz",
input: "hugo_0.145.0_linux-amd64.tar.gz",
wantOS: buildmeta.OSLinux,
arch: buildmeta.ArchAMD64,
format: buildmeta.FormatTarGz,
},
{
name: "goreleaser darwin arm64 tar.gz",
input: "hugo_0.145.0_darwin-arm64.tar.gz",
wantOS: buildmeta.OSDarwin,
arch: buildmeta.ArchARM64,
format: buildmeta.FormatTarGz,
},
{
name: "goreleaser windows amd64 zip",
input: "hugo_0.145.0_windows-amd64.zip",
wantOS: buildmeta.OSWindows,
arch: buildmeta.ArchAMD64,
format: buildmeta.FormatZip,
},
{
name: "goreleaser freebsd",
input: "hugo_0.145.0_freebsd-amd64.tar.gz",
wantOS: buildmeta.OSFreeBSD,
arch: buildmeta.ArchAMD64,
format: buildmeta.FormatTarGz,
},
// Rust/cargo-dist style
{
name: "rust linux musl",
input: "ripgrep-14.1.1-x86_64-unknown-linux-musl.tar.gz",
wantOS: buildmeta.OSLinux,
arch: buildmeta.ArchAMD64,
libc: buildmeta.LibcMusl,
format: buildmeta.FormatTarGz,
},
{
name: "rust linux gnu",
input: "bat-v0.24.0-x86_64-unknown-linux-gnu.tar.gz",
wantOS: buildmeta.OSLinux,
arch: buildmeta.ArchAMD64,
libc: buildmeta.LibcGNU,
format: buildmeta.FormatTarGz,
},
{
name: "rust apple darwin",
input: "ripgrep-14.1.1-x86_64-apple-darwin.tar.gz",
wantOS: buildmeta.OSDarwin,
arch: buildmeta.ArchAMD64,
format: buildmeta.FormatTarGz,
},
{
name: "rust windows msvc",
input: "bat-v0.24.0-x86_64-pc-windows-msvc.zip",
wantOS: buildmeta.OSWindows,
arch: buildmeta.ArchAMD64,
libc: buildmeta.LibcMSVC,
format: buildmeta.FormatZip,
},
{
name: "rust aarch64 linux",
input: "ripgrep-14.1.1-aarch64-unknown-linux-gnu.tar.gz",
wantOS: buildmeta.OSLinux,
arch: buildmeta.ArchARM64,
libc: buildmeta.LibcGNU,
format: buildmeta.FormatTarGz,
},
// Zig-style
{
name: "zig linux x86_64",
input: "zig-linux-x86_64-0.14.0.tar.xz",
wantOS: buildmeta.OSLinux,
arch: buildmeta.ArchAMD64,
format: buildmeta.FormatTarXz,
},
{
name: "zig macos aarch64",
input: "zig-macos-aarch64-0.14.0.tar.xz",
wantOS: buildmeta.OSDarwin,
arch: buildmeta.ArchARM64,
format: buildmeta.FormatTarXz,
},
// Windows executables
{
name: "bare exe",
input: "jq-windows-amd64.exe",
wantOS: buildmeta.OSWindows,
arch: buildmeta.ArchAMD64,
format: buildmeta.FormatExe,
},
{
name: "msi installer",
input: "caddy_2.9.0_windows_amd64.msi",
wantOS: buildmeta.OSWindows,
arch: buildmeta.ArchAMD64,
format: buildmeta.FormatMSI,
},
// macOS formats
{
name: "dmg installer",
input: "MyApp-1.0.0-darwin-arm64.dmg",
wantOS: buildmeta.OSDarwin,
arch: buildmeta.ArchARM64,
format: buildmeta.FormatDMG,
},
// Arch priority: x86_64 must not match x86
{
name: "x86_64 not x86",
input: "tool-x86_64-linux.tar.gz",
wantOS: buildmeta.OSLinux,
arch: buildmeta.ArchAMD64,
format: buildmeta.FormatTarGz,
},
{
name: "actual x86",
input: "tool-x86-linux.tar.gz",
wantOS: buildmeta.OSLinux,
arch: buildmeta.ArchX86,
format: buildmeta.FormatTarGz,
},
{
name: "i386",
input: "tool-linux-i386.tar.gz",
wantOS: buildmeta.OSLinux,
arch: buildmeta.ArchX86,
format: buildmeta.FormatTarGz,
},
// Windows ARM: bare "arm" is armv6 (some tools ship genuine arm32 Windows builds).
// Explicit "arm64" is always aarch64 regardless of OS.
{
name: "windows bare arm stays armv6",
input: "tool-1.0.0-windows-arm.zip",
wantOS: buildmeta.OSWindows,
arch: buildmeta.ArchARMv6,
format: buildmeta.FormatZip,
},
{
name: "windows armv6 stays armv6",
input: "tool-2.0.0-windows-armv6.zip",
wantOS: buildmeta.OSWindows,
arch: buildmeta.ArchARMv6,
format: buildmeta.FormatZip,
},
{
name: "windows arm64 stays arm64",
input: "tool-1.0.0-windows-arm64.zip",
wantOS: buildmeta.OSWindows,
arch: buildmeta.ArchARM64,
format: buildmeta.FormatZip,
},
// armel and gnueabihf are ARMv6 ABI names
{
name: "armel is armv6",
input: "jq-linux-armel",
wantOS: buildmeta.OSLinux,
arch: buildmeta.ArchARMv6,
},
{
name: "gnueabihf is armv6",
input: "tool-arm-unknown-linux-gnueabihf.tar.gz",
wantOS: buildmeta.OSLinux,
arch: buildmeta.ArchARMv6,
format: buildmeta.FormatTarGz,
},
// winx64 is a Windows x86_64 naming used by MariaDB
{
name: "winx64 is windows x86_64",
input: "mariadb-11.4.5-winx64.zip",
wantOS: buildmeta.OSWindows,
arch: buildmeta.ArchAMD64,
format: buildmeta.FormatZip,
},
// win32/win64 naming used by chromedriver, dashcore, etc.
{
name: "win32 is windows x86",
input: "chromedriver-win32.zip",
wantOS: buildmeta.OSWindows,
arch: buildmeta.ArchX86,
format: buildmeta.FormatZip,
},
{
name: "win64 is windows amd64",
input: "dashcore-23.1.2-win64-setup.exe",
wantOS: buildmeta.OSWindows,
arch: buildmeta.ArchAMD64,
format: buildmeta.FormatExe,
},
// ppc64el is a Debian/Ubuntu alias for ppc64le
{
name: "ppc64el is ppc64le",
input: "jq-linux-ppc64el",
arch: buildmeta.ArchPPC64LE,
},
// amd64 micro-architecture levels
{
name: "amd64v2",
input: "tool-linux-amd64v2.tar.gz",
arch: buildmeta.ArchAMD64v2,
},
{
name: "amd64v3",
input: "tool-linux-x86_64_v3.tar.gz",
arch: buildmeta.ArchAMD64v3,
},
{
name: "amd64v4",
input: "tool-linux-amd64v4.tar.gz",
arch: buildmeta.ArchAMD64v4,
},
{
name: "amd64v3 not baseline",
input: "tool-1.0.0-linux-amd64v3.tar.gz",
wantOS: buildmeta.OSLinux,
arch: buildmeta.ArchAMD64v3,
format: buildmeta.FormatTarGz,
},
// ARM variants: arm64 must not match armv7/armv6
{
name: "aarch64 not armv7",
input: "tool-aarch64-linux.tar.gz",
arch: buildmeta.ArchARM64,
},
{
name: "armv7",
input: "tool-armv7l-linux.tar.gz",
arch: buildmeta.ArchARMv7,
},
{
name: "armv6",
input: "tool-armv6l-linux.tar.gz",
arch: buildmeta.ArchARMv6,
},
// ppc64le before ppc64
{
name: "ppc64le",
input: "tool-linux-ppc64le.tar.gz",
arch: buildmeta.ArchPPC64LE,
},
{
name: "ppc64",
input: "tool-linux-ppc64.tar.gz",
arch: buildmeta.ArchPPC64,
},
// Static linking
{
name: "static binary",
input: "tool-linux-amd64-static.tar.gz",
libc: buildmeta.LibcNone,
},
// .exe implies Windows
{
name: "exe implies windows",
input: "tool-amd64.exe",
wantOS: buildmeta.OSWindows,
arch: buildmeta.ArchAMD64,
format: buildmeta.FormatExe,
},
// Compound extensions
{
name: "tar.zst",
input: "tool-linux-amd64.tar.zst",
format: buildmeta.FormatTarZst,
},
{
name: "exe.xz",
input: "tool-windows-amd64.exe.xz",
format: buildmeta.FormatExeXz,
},
{
name: "app.zip",
input: "MyApp-1.0.0.app.zip",
format: buildmeta.FormatAppZip,
},
{
name: "tgz alias",
input: "tool-linux-amd64.tgz",
format: buildmeta.FormatTarGz,
},
// s390x, mips
{
name: "s390x",
input: "tool-linux-s390x.tar.gz",
arch: buildmeta.ArchS390X,
},
{
name: "mips64",
input: "tool-linux-mips64.tar.gz",
arch: buildmeta.ArchMIPS64,
},
// Unknown / no match
{
name: "checksum file",
input: "checksums.txt",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := classify.Filename(tt.input)
if tt.wantOS != "" && got.OS != tt.wantOS {
t.Errorf("OS = %q, want %q", got.OS, tt.wantOS)
}
if tt.arch != "" && got.Arch != tt.arch {
t.Errorf("Arch = %q, want %q", got.Arch, tt.arch)
}
if tt.libc != "" && got.Libc != tt.libc {
t.Errorf("Libc = %q, want %q", got.Libc, tt.libc)
}
if tt.format != "" && got.Format != tt.format {
t.Errorf("Format = %q, want %q", got.Format, tt.format)
}
})
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,154 +0,0 @@
// Package httpclient provides a well-configured [http.Client] for upstream
// API calls. It exists because [http.DefaultClient] has no timeouts, no TLS
// minimum, and follows redirects from HTTPS to HTTP — none of which are
// acceptable for a server calling GitHub, Gitea, etc. on behalf of users.
//
// Use [New] to create a configured client. Use [Do] to execute a request
// with automatic retries for transient failures.
package httpclient
import (
"context"
"crypto/tls"
"errors"
"fmt"
"math/rand/v2"
"net"
"net/http"
"strconv"
"time"
)
const userAgent = "Webi/2.0 (+https://webinstall.dev)"
// New returns an [http.Client] with secure, production-ready defaults:
// TLS 1.2+, timeouts at every level, connection pooling, no HTTPS→HTTP
// redirect, and a Webi User-Agent.
func New() *http.Client {
return &http.Client{
Transport: &http.Transport{
DialContext: (&net.Dialer{
Timeout: 10 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
TLSClientConfig: &tls.Config{
MinVersion: tls.VersionTLS12,
},
TLSHandshakeTimeout: 10 * time.Second,
ResponseHeaderTimeout: 30 * time.Second,
MaxIdleConns: 100,
MaxIdleConnsPerHost: 10,
IdleConnTimeout: 90 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
ForceAttemptHTTP2: true,
},
Timeout: 60 * time.Second,
CheckRedirect: checkRedirect,
}
}
// checkRedirect prevents HTTPS→HTTP downgrades and limits redirect depth.
func checkRedirect(req *http.Request, via []*http.Request) error {
if len(via) >= 10 {
return fmt.Errorf("stopped after %d redirects", len(via))
}
if len(via) > 0 && via[0].URL.Scheme == "https" && req.URL.Scheme == "http" {
return errors.New("refused redirect from https to http")
}
return nil
}
// Get performs a GET request with the Webi User-Agent header.
func Get(ctx context.Context, client *http.Client, url string) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", userAgent)
return client.Do(req)
}
// Do executes a request with automatic retries for transient errors (429,
// 502, 503, 504). Retries up to 3 times with exponential backoff and jitter.
// Respects Retry-After headers. Only retries GET and HEAD (idempotent).
//
// Sets the Webi User-Agent header if not already present.
func Do(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
if req.Header.Get("User-Agent") == "" {
req.Header.Set("User-Agent", userAgent)
}
// Only retry idempotent methods.
idempotent := req.Method == http.MethodGet || req.Method == http.MethodHead
const maxRetries = 3
var resp *http.Response
var err error
for attempt := range maxRetries + 1 {
if attempt > 0 {
if !idempotent {
break
}
delay := backoff(attempt, resp)
timer := time.NewTimer(delay)
select {
case <-ctx.Done():
timer.Stop()
return nil, ctx.Err()
case <-timer.C:
}
if resp != nil {
resp.Body.Close()
}
}
resp, err = client.Do(req)
if err != nil {
if ctx.Err() != nil {
return nil, ctx.Err()
}
continue
}
if !isRetryable(resp.StatusCode) {
return resp, nil
}
}
if err != nil {
return nil, fmt.Errorf("after %d retries: %w", maxRetries, err)
}
return resp, nil
}
func isRetryable(status int) bool {
return status == http.StatusTooManyRequests ||
status == http.StatusBadGateway ||
status == http.StatusServiceUnavailable ||
status == http.StatusGatewayTimeout
}
// backoff returns a delay before the next retry. Respects Retry-After,
// otherwise uses exponential backoff with jitter.
func backoff(attempt int, resp *http.Response) time.Duration {
if resp != nil {
if ra := resp.Header.Get("Retry-After"); ra != "" {
if seconds, err := strconv.Atoi(ra); err == nil && seconds > 0 && seconds < 300 {
return time.Duration(seconds) * time.Second
}
}
}
// 1s, 2s, 4s base delays
base := time.Second << (attempt - 1)
if base > 30*time.Second {
base = 30 * time.Second
}
// Add jitter: 75% to 125% of base
jitter := float64(base) * (0.75 + 0.5*rand.Float64())
return time.Duration(jitter)
}

View File

@@ -1,286 +0,0 @@
// Package installerconf reads per-package releases.conf files.
//
// The format is simple key=value, one per line. Blank lines and lines
// starting with # are ignored. Keys and values are trimmed of whitespace.
// Multi-value keys are whitespace-delimited.
//
// The source type is inferred from the primary key:
//
// GitHub binary releases:
//
// github_releases = sharkdp/bat
// github_releases = https://github.com/sharkdp/bat
//
// GitHub source archives (for source-installable packages):
//
// github_sources = BeyondCodeBootcamp/aliasman
// git_url = https://github.com/BeyondCodeBootcamp/aliasman.git
//
// Gitea binary releases (self-hosted, requires full URL or base_url):
//
// gitea_releases = https://git.rootprojects.org/root/pathman
//
// GitLab binary releases (defaults to gitlab.com):
//
// gitlab_releases = owner/repo
// gitlab_releases = https://gitlab.example.com/owner/repo
//
// Git tag enumeration (vim plugins, etc.):
//
// git_url = https://github.com/tpope/vim-commentary.git
//
// HashiCorp releases:
//
// hashicorp_product = terraform
//
// Other sources (one-off scrapers):
//
// source = nodedist
// url = https://nodejs.org/download/release
//
// Complex packages that need custom logic beyond what the classifier
// auto-detects (e.g. ollama's universal binaries, ffmpeg's non-standard
// naming) should put that logic in Go code, not in the config.
// The variants key documents known build variants for human readers;
// actual variant detection logic lives in Go.
package installerconf
import (
"bufio"
"fmt"
"net/url"
"os"
"strings"
)
// Conf holds the parsed per-package release configuration.
type Conf struct {
// Source is the fetch source type: "github", "githubsource",
// "gitea", "giteasource", "gitlab", "gitlabsource",
// "gittag", "nodedist", etc.
Source string
// Owner is the repository owner (org or user).
Owner string
// Repo is the repository name.
Repo string
// BaseURL is a custom base URL for non-GitHub sources
// (e.g. a Gitea instance or nodedist index URL).
BaseURL string
// GitURL is the git clone URL for source-installable packages.
// Present alongside github_sources/gitea_sources to provide a
// git clone fallback in addition to release tarballs.
GitURL string
// TagPrefix filters releases in monorepos. Only tags starting with
// this prefix are included, and the prefix is stripped from the
// version string. Example: "tools/monorel/"
TagPrefix string
// VersionPrefixes are stripped from version/tag strings.
// Whitespace-delimited. Each release tag is checked against these
// in order; the first match is stripped. Projects may change tag
// conventions across versions (e.g. "jq-1.7.1" older, "1.8.0" later).
VersionPrefixes []string
// Exclude lists filename substrings to filter out.
// Whitespace-delimited. Assets whose name contains any of these
// are skipped entirely (not stored).
Exclude []string
// AssetFilter is a substring that asset filenames must contain.
// Used when multiple packages share a GitHub release (e.g.
// kubectx/kubens) to select only the relevant assets.
AssetFilter string
// Variants documents known build variant names for this package.
// Whitespace-delimited. This is a human-readable cue — actual
// variant detection logic lives in Go code per-package.
Variants []string
// OS restricts all assets to this OS value when set.
// Use "posix_2017" for POSIX-only shell packages that don't
// support Windows.
OS string
// AliasOf names another package that this one mirrors.
// When set, the package has no releases of its own — it shares
// the cache output of the named target (e.g. dashd → dashcore).
AliasOf string
// Extra holds any unrecognized keys for forward compatibility.
Extra map[string]string
}
// parseRepoRef parses a value that is either "owner/repo" or a full URL
// like "https://github.com/owner/repo". Returns baseURL, owner, repo.
// For short form, baseURL is empty (caller uses the default for the forge).
// For full URL form, baseURL is the scheme+host (e.g. "https://github.com").
func parseRepoRef(val, defaultBase string) (baseURL, owner, repo string) {
if strings.Contains(val, "://") {
u, err := url.Parse(val)
if err == nil {
baseURL = u.Scheme + "://" + u.Host
path := strings.Trim(u.Path, "/")
owner, repo, _ = strings.Cut(path, "/")
return baseURL, owner, repo
}
}
// Short form: "owner/repo"
owner, repo, _ = strings.Cut(val, "/")
return defaultBase, owner, repo
}
// Read parses a releases.conf file.
func Read(path string) (*Conf, error) {
f, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("installerconf: %w", err)
}
defer f.Close()
raw := make(map[string]string)
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || line[0] == '#' {
continue
}
key, val, ok := strings.Cut(line, "=")
if !ok {
continue
}
raw[strings.TrimSpace(key)] = strings.TrimSpace(val)
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("installerconf: read %s: %w", path, err)
}
c := &Conf{}
// Infer source from primary key, falling back to explicit "source".
// When both github_releases and source are set, parse the repo ref
// from github_releases but use the explicit source for classification.
switch {
// GitHub binary releases.
case raw["github_releases"] != "":
c.Source = "github"
c.BaseURL, c.Owner, c.Repo = parseRepoRef(raw["github_releases"], "https://github.com")
// GitHub source tarballs.
case raw["github_sources"] != "":
c.Source = "githubsource"
c.BaseURL, c.Owner, c.Repo = parseRepoRef(raw["github_sources"], "https://github.com")
// Gitea binary releases (self-hosted only — requires full URL or base_url).
case raw["gitea_releases"] != "":
c.Source = "gitea"
c.BaseURL, c.Owner, c.Repo = parseRepoRef(raw["gitea_releases"], raw["base_url"])
// Gitea source tarballs (self-hosted only).
case raw["gitea_sources"] != "":
c.Source = "giteasource"
c.BaseURL, c.Owner, c.Repo = parseRepoRef(raw["gitea_sources"], raw["base_url"])
// GitLab binary releases (defaults to gitlab.com).
case raw["gitlab_releases"] != "":
c.Source = "gitlab"
c.BaseURL, c.Owner, c.Repo = parseRepoRef(raw["gitlab_releases"], "https://gitlab.com")
// GitLab source tarballs (defaults to gitlab.com).
case raw["gitlab_sources"] != "":
c.Source = "gitlabsource"
c.BaseURL, c.Owner, c.Repo = parseRepoRef(raw["gitlab_sources"], "https://gitlab.com")
// Explicit source type (servicemandist, nodedist, zigdist, etc.).
// Must come before git_url so that "source = X" + "git_url = ..."
// uses X as the primary source, not gittag.
case raw["source"] != "":
c.Source = raw["source"]
c.BaseURL = raw["url"]
// Git tag enumeration (only when no explicit source is set).
case raw["git_url"] != "":
c.Source = "gittag"
c.BaseURL = raw["git_url"]
// HashiCorp.
case raw["hashicorp_product"] != "":
c.Source = "hashicorp"
c.Repo = raw["hashicorp_product"]
default:
}
// Explicit "source" overrides the inferred source when both are present.
// This lets packages like ffmpeg use github_releases for fetching but
// a custom classifier for classification.
if raw["source"] != "" && c.Source != "" {
c.Source = raw["source"]
}
// git_url can appear alongside any source type (e.g. github_sources)
// to provide a git clone fallback. When it's the only key, it's the
// primary source (gittag).
c.GitURL = raw["git_url"]
c.TagPrefix = raw["tag_prefix"]
if v := raw["version_prefixes"]; v != "" {
c.VersionPrefixes = strings.Fields(v)
} else if v := raw["version_prefix"]; v != "" {
c.VersionPrefixes = strings.Fields(v)
}
// Accept both "exclude" and "asset_exclude" (back-compat).
if v := raw["exclude"]; v != "" {
c.Exclude = strings.Fields(v)
} else if v := raw["asset_exclude"]; v != "" {
c.Exclude = strings.Fields(v)
}
c.AssetFilter = raw["asset_filter"]
c.OS = raw["os"]
c.AliasOf = raw["alias_of"]
if v := raw["variants"]; v != "" {
c.Variants = strings.Fields(v)
}
// Collect unrecognized keys.
known := map[string]bool{
"source": true,
"github_releases": true,
"github_sources": true,
"gitea_releases": true,
"gitea_sources": true,
"gitlab_releases": true,
"gitlab_sources": true,
"git_url": true,
"hashicorp_product": true,
"base_url": true,
"url": true,
"tag_prefix": true,
"version_prefix": true,
"version_prefixes": true,
"exclude": true,
"asset_exclude": true,
"asset_filter": true,
"os": true,
"variants": true,
"alias_of": true,
}
for k, v := range raw {
if !known[k] {
if c.Extra == nil {
c.Extra = make(map[string]string)
}
c.Extra[k] = v
}
}
return c, nil
}

View File

@@ -1,217 +0,0 @@
package installerconf_test
import (
"os"
"path/filepath"
"testing"
"github.com/webinstall/webi-installers/internal/installerconf"
)
func TestGitHubReleases(t *testing.T) {
c := confFromString(t, `
github_releases = sharkdp/bat
`)
assertEqual(t, "Source", c.Source, "github")
assertEqual(t, "Owner", c.Owner, "sharkdp")
assertEqual(t, "Repo", c.Repo, "bat")
assertEqual(t, "BaseURL", c.BaseURL, "https://github.com")
assertEqual(t, "TagPrefix", c.TagPrefix, "")
if len(c.VersionPrefixes) != 0 {
t.Errorf("VersionPrefixes = %v, want empty", c.VersionPrefixes)
}
if len(c.Exclude) != 0 {
t.Errorf("Exclude = %v, want empty", c.Exclude)
}
}
func TestGitHubReleasesFullURL(t *testing.T) {
c := confFromString(t, `
github_releases = https://github.com/sharkdp/bat
`)
assertEqual(t, "Source", c.Source, "github")
assertEqual(t, "BaseURL", c.BaseURL, "https://github.com")
assertEqual(t, "Owner", c.Owner, "sharkdp")
assertEqual(t, "Repo", c.Repo, "bat")
}
func TestGitHubSources(t *testing.T) {
c := confFromString(t, `
github_sources = BeyondCodeBootcamp/aliasman
git_url = https://github.com/BeyondCodeBootcamp/aliasman.git
`)
assertEqual(t, "Source", c.Source, "githubsource")
assertEqual(t, "Owner", c.Owner, "BeyondCodeBootcamp")
assertEqual(t, "Repo", c.Repo, "aliasman")
assertEqual(t, "GitURL", c.GitURL, "https://github.com/BeyondCodeBootcamp/aliasman.git")
}
func TestGitHubSourcesFullURL(t *testing.T) {
c := confFromString(t, `
github_sources = https://github.com/BeyondCodeBootcamp/aliasman
git_url = https://github.com/BeyondCodeBootcamp/aliasman.git
`)
assertEqual(t, "Source", c.Source, "githubsource")
assertEqual(t, "BaseURL", c.BaseURL, "https://github.com")
assertEqual(t, "Owner", c.Owner, "BeyondCodeBootcamp")
assertEqual(t, "Repo", c.Repo, "aliasman")
}
func TestVersionPrefixes(t *testing.T) {
c := confFromString(t, `
github_releases = jqlang/jq
version_prefixes = jq- cli-
`)
if len(c.VersionPrefixes) != 2 {
t.Fatalf("VersionPrefixes has %d items, want 2: %v", len(c.VersionPrefixes), c.VersionPrefixes)
}
assertEqual(t, "VersionPrefixes[0]", c.VersionPrefixes[0], "jq-")
assertEqual(t, "VersionPrefixes[1]", c.VersionPrefixes[1], "cli-")
}
func TestExclude(t *testing.T) {
c := confFromString(t, `
github_releases = gohugoio/hugo
exclude = _extended_ Linux-64bit
`)
if len(c.Exclude) != 2 {
t.Fatalf("Exclude has %d items, want 2: %v", len(c.Exclude), c.Exclude)
}
assertEqual(t, "Exclude[0]", c.Exclude[0], "_extended_")
assertEqual(t, "Exclude[1]", c.Exclude[1], "Linux-64bit")
}
func TestMonorepoTagPrefix(t *testing.T) {
c := confFromString(t, `
github_releases = therootcompany/golib
tag_prefix = tools/monorel/
`)
assertEqual(t, "TagPrefix", c.TagPrefix, "tools/monorel/")
}
func TestNodeDist(t *testing.T) {
c := confFromString(t, `
source = nodedist
url = https://nodejs.org/download/release
`)
assertEqual(t, "Source", c.Source, "nodedist")
assertEqual(t, "BaseURL", c.BaseURL, "https://nodejs.org/download/release")
}
func TestGiteaReleases(t *testing.T) {
c := confFromString(t, `
gitea_releases = https://git.rootprojects.org/root/pathman
`)
assertEqual(t, "Source", c.Source, "gitea")
assertEqual(t, "BaseURL", c.BaseURL, "https://git.rootprojects.org")
assertEqual(t, "Owner", c.Owner, "root")
assertEqual(t, "Repo", c.Repo, "pathman")
}
func TestGiteaReleasesWithBaseURL(t *testing.T) {
c := confFromString(t, `
gitea_releases = root/pathman
base_url = https://git.rootprojects.org
`)
assertEqual(t, "Source", c.Source, "gitea")
assertEqual(t, "BaseURL", c.BaseURL, "https://git.rootprojects.org")
assertEqual(t, "Owner", c.Owner, "root")
assertEqual(t, "Repo", c.Repo, "pathman")
}
func TestGitLabReleases(t *testing.T) {
c := confFromString(t, `
gitlab_releases = owner/repo
`)
assertEqual(t, "Source", c.Source, "gitlab")
assertEqual(t, "BaseURL", c.BaseURL, "https://gitlab.com")
assertEqual(t, "Owner", c.Owner, "owner")
assertEqual(t, "Repo", c.Repo, "repo")
}
func TestGitLabReleasesFullURL(t *testing.T) {
c := confFromString(t, `
gitlab_releases = https://gitlab.example.com/myorg/myrepo
`)
assertEqual(t, "Source", c.Source, "gitlab")
assertEqual(t, "BaseURL", c.BaseURL, "https://gitlab.example.com")
assertEqual(t, "Owner", c.Owner, "myorg")
assertEqual(t, "Repo", c.Repo, "myrepo")
}
func TestBlanksAndComments(t *testing.T) {
c := confFromString(t, `
# Hugo config
github_releases = foo/bar
# exclude line
exclude = extended
`)
assertEqual(t, "Source", c.Source, "github")
assertEqual(t, "Owner", c.Owner, "foo")
assertEqual(t, "Repo", c.Repo, "bar")
}
func TestExtraKeys(t *testing.T) {
c := confFromString(t, `
github_releases = foo/bar
custom_thing = hello
`)
if c.Extra == nil || c.Extra["custom_thing"] != "hello" {
t.Errorf("Extra[custom_thing] = %q, want hello", c.Extra["custom_thing"])
}
}
func TestAssetExcludeAlias(t *testing.T) {
c := confFromString(t, `
github_releases = gohugoio/hugo
asset_exclude = extended
`)
if len(c.Exclude) != 1 {
t.Fatalf("Exclude has %d items, want 1: %v", len(c.Exclude), c.Exclude)
}
assertEqual(t, "Exclude[0]", c.Exclude[0], "extended")
}
func TestVariants(t *testing.T) {
c := confFromString(t, `
github_releases = jmorganca/ollama
variants = rocm jetpack5 jetpack6
`)
if len(c.Variants) != 3 {
t.Fatalf("Variants has %d items, want 3: %v", len(c.Variants), c.Variants)
}
assertEqual(t, "Variants[0]", c.Variants[0], "rocm")
assertEqual(t, "Variants[1]", c.Variants[1], "jetpack5")
assertEqual(t, "Variants[2]", c.Variants[2], "jetpack6")
}
func TestEmptyExclude(t *testing.T) {
c := confFromString(t, "github_releases = foo/bar\n")
if c.Exclude != nil {
t.Errorf("Exclude = %v, want nil", c.Exclude)
}
}
// helpers
func confFromString(t *testing.T, content string) *installerconf.Conf {
t.Helper()
dir := t.TempDir()
path := filepath.Join(dir, "releases.conf")
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
c, err := installerconf.Read(path)
if err != nil {
t.Fatal(err)
}
return c
}
func assertEqual(t *testing.T, name, got, want string) {
t.Helper()
if got != want {
t.Errorf("%s = %q, want %q", name, got, want)
}
}

View File

@@ -1,189 +0,0 @@
// Package lexver makes version strings comparable and sortable.
//
// Not all version strings are semver. Webi handles 4-part versions
// (chromedriver 121.0.6120.0), date-based versions (atomicparsley),
// and pre-releases with extra dots (flutter 2.3.0-16.0.pre). Lexver
// parses these into a struct with an arbitrary-depth numeric segment
// list and provides a comparison function for use with [slices.SortFunc].
//
// Pre-releases sort before their corresponding stable release:
//
// 1.0.0-alpha1 < 1.0.0-beta1 < 1.0.0-rc1 < 1.0.0
//
// When release dates are known, they break ties between versions with
// identical numeric segments.
package lexver
import (
"cmp"
"strconv"
"strings"
"time"
"unicode"
)
// Version is a parsed version with comparable fields.
type Version struct {
// Nums holds the dotted numeric segments in order.
// "1.20.3" → [1, 20, 3], "121.0.6120.0" → [121, 0, 6120, 0].
Nums []int
Channel string // "" for stable, or "alpha", "beta", "dev", "pre", "preview", "rc"
ChannelNum int // e.g. 2 in "rc2"
Date time.Time // release date/time, if known; breaks ties between same-numbered versions
Original string // version string exactly as the releaser published it (e.g. "REL_17_0", "r21")
Raw string // version string after Webi's normalization (e.g. "17.0", "0.21.0")
// ExtraSort is an optional opaque string for package-specific ordering.
// Set by release-fetcher code for packages where Nums alone can't capture
// the sort order (e.g. flutter's "2.3.0-16.0.pre"). Compared as a plain
// string, only consulted when Nums and Channel are equal.
ExtraSort string
}
// Parse breaks a version string into its components.
// Both Original and Raw are set to s; callers that normalize versions
// (e.g. "REL_17_0" → "17.0") should set Original to the upstream tag
// and pass the normalized string to Parse.
func Parse(s string) Version {
v := Version{Original: s, Raw: s}
s = strings.TrimLeft(s, "vV")
numStr, prerelease := splitAtPrerelease(s)
v.Nums = splitNums(numStr)
if prerelease != "" {
v.Channel, v.ChannelNum = splitChannel(prerelease)
}
return v
}
// Major returns the first numeric segment, or 0 if none.
func (v Version) Major() int { return v.num(0) }
// Minor returns the second numeric segment, or 0 if none.
func (v Version) Minor() int { return v.num(1) }
// Patch returns the third numeric segment, or 0 if none.
func (v Version) Patch() int { return v.num(2) }
func (v Version) num(i int) int {
if i < len(v.Nums) {
return v.Nums[i]
}
return 0
}
// IsStable reports whether this is a stable (non-pre-release) version.
func (v Version) IsStable() bool {
return v.Channel == ""
}
// Compare returns -1, 0, or 1 for ordering two versions.
// Stable releases sort after pre-releases of the same numeric version.
func Compare(a, b Version) int {
// Compare numeric segments pairwise, treating missing segments as 0.
n := max(len(a.Nums), len(b.Nums))
for i := range n {
an, bn := a.num(i), b.num(i)
if c := cmp.Compare(an, bn); c != 0 {
return c
}
}
// Break ties with release date when both are known.
if !a.Date.IsZero() && !b.Date.IsZero() {
if c := a.Date.Compare(b.Date); c != 0 {
return c
}
}
// ExtraSort: package-specific tiebreaker set by release-fetcher code.
if a.ExtraSort != "" && b.ExtraSort != "" {
if c := cmp.Compare(a.ExtraSort, b.ExtraSort); c != 0 {
return c
}
}
// Both stable → equal.
if a.Channel == "" && b.Channel == "" {
return 0
}
// Stable beats any pre-release.
if a.Channel == "" {
return 1
}
if b.Channel == "" {
return -1
}
// Both pre-release: alphabetical channel, then number.
if c := cmp.Compare(a.Channel, b.Channel); c != 0 {
return c
}
return cmp.Compare(a.ChannelNum, b.ChannelNum)
}
// HasPrefix reports whether v matches a partial version prefix.
// A prefix with Nums [1, 20] matches any version starting with 1.20
// (e.g. 1.20.0, 1.20.3, 1.20.3.1).
func (v Version) HasPrefix(prefix Version) bool {
for i, pn := range prefix.Nums {
if i >= len(v.Nums) || v.Nums[i] != pn {
return false
}
}
return true
}
// splitAtPrerelease splits "1.20.3-beta1" into ("1.20.3", "beta1").
// Also handles "1.2beta3" (no separator).
func splitAtPrerelease(s string) (string, string) {
for _, sep := range []byte{'-', '+'} {
if idx := strings.IndexByte(s, sep); idx >= 0 {
return s[:idx], s[idx+1:]
}
}
// "1.2beta3": letter following a digit
for i := 1; i < len(s); i++ {
if unicode.IsLetter(rune(s[i])) && unicode.IsDigit(rune(s[i-1])) {
return s[:i], s[i:]
}
}
return s, ""
}
// splitNums parses "1.20.3" into [1, 20, 3].
// Handles any number of dot-separated segments.
func splitNums(s string) []int {
var nums []int
for _, seg := range strings.Split(s, ".") {
n, err := strconv.Atoi(seg)
if err != nil {
break
}
nums = append(nums, n)
}
return nums
}
// splitChannel separates "beta1" into ("beta", 1) or "rc" into ("rc", 0).
func splitChannel(s string) (string, int) {
s = strings.ToLower(s)
s = strings.NewReplacer("-", "", ".", "", "_", "").Replace(s)
i := len(s)
for i > 0 && unicode.IsDigit(rune(s[i-1])) {
i--
}
name := s[:i]
num := 0
if i < len(s) {
num, _ = strconv.Atoi(s[i:])
}
return name, num
}

View File

@@ -1,270 +0,0 @@
package lexver_test
import (
"slices"
"testing"
"time"
"github.com/webinstall/webi-installers/internal/lexver"
)
func TestParse(t *testing.T) {
tests := []struct {
input string
nums []int
channel string
chanNum int
}{
// Standard semver
{"1.0.0", []int{1, 0, 0}, "", 0},
{"v1.2.3", []int{1, 2, 3}, "", 0},
{"1.20.156", []int{1, 20, 156}, "", 0},
// Partial
{"1.20", []int{1, 20}, "", 0},
{"1", []int{1}, "", 0},
// 4-part (chromedriver, gpg)
{"121.0.6120.0", []int{121, 0, 6120, 0}, "", 0},
{"2.2.19.0", []int{2, 2, 19, 0}, "", 0},
// Pre-release
{"1.0.0-beta1", []int{1, 0, 0}, "beta", 1},
{"1.0.0-rc2", []int{1, 0, 0}, "rc", 2},
{"2.0.0-alpha3", []int{2, 0, 0}, "alpha", 3},
{"1.0.0-dev", []int{1, 0, 0}, "dev", 0},
// No separator before channel
{"1.2beta3", []int{1, 2}, "beta", 3},
{"1.0rc1", []int{1, 0}, "rc", 1},
}
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
v := lexver.Parse(tt.input)
if !slices.Equal(v.Nums, tt.nums) {
t.Errorf("Parse(%q).Nums = %v, want %v", tt.input, v.Nums, tt.nums)
}
if v.Channel != tt.channel || v.ChannelNum != tt.chanNum {
t.Errorf("Parse(%q) channel = %q/%d, want %q/%d",
tt.input, v.Channel, v.ChannelNum, tt.channel, tt.chanNum)
}
})
}
}
func TestAccessors(t *testing.T) {
v := lexver.Parse("121.0.6120.0")
if v.Major() != 121 || v.Minor() != 0 || v.Patch() != 6120 {
t.Errorf("got %d.%d.%d, want 121.0.6120", v.Major(), v.Minor(), v.Patch())
}
short := lexver.Parse("1")
if short.Minor() != 0 || short.Patch() != 0 {
t.Error("missing segments should return 0")
}
}
func TestSortOrder(t *testing.T) {
// Must be in ascending order.
ordered := []string{
"0.1.0",
"1.0.0-alpha1",
"1.0.0-alpha2",
"1.0.0-beta1",
"1.0.0-rc1",
"1.0.0-rc2",
"1.0.0",
"1.0.1",
"1.1.0",
"1.2.0",
"1.20.0",
"2.0.0-beta1",
"2.0.0",
}
for i := 1; i < len(ordered); i++ {
a := lexver.Parse(ordered[i-1])
b := lexver.Parse(ordered[i])
if lexver.Compare(a, b) >= 0 {
t.Errorf("expected %q < %q", ordered[i-1], ordered[i])
}
}
}
func TestSortOrder4Part(t *testing.T) {
ordered := []string{
"121.0.6120.0",
"121.0.6120.1",
"121.0.6121.0",
"122.0.6100.0",
}
for i := 1; i < len(ordered); i++ {
a := lexver.Parse(ordered[i-1])
b := lexver.Parse(ordered[i])
if lexver.Compare(a, b) >= 0 {
t.Errorf("expected %q < %q", ordered[i-1], ordered[i])
}
}
}
func TestMismatchedDepth(t *testing.T) {
// "1.0" and "1.0.0" should be equal (trailing zeros).
a := lexver.Parse("1.0")
b := lexver.Parse("1.0.0")
if lexver.Compare(a, b) != 0 {
t.Error("1.0 and 1.0.0 should be equal")
}
// "1.0.0.1" should be greater than "1.0.0".
c := lexver.Parse("1.0.0.1")
d := lexver.Parse("1.0.0")
if lexver.Compare(c, d) <= 0 {
t.Error("1.0.0.1 should be greater than 1.0.0")
}
}
func TestSortFunc(t *testing.T) {
versions := []string{"1.0.0", "2.0.0-rc1", "1.20.3", "1.20.2", "1.19.5", "2.0.0"}
parsed := make([]lexver.Version, len(versions))
for i, s := range versions {
parsed[i] = lexver.Parse(s)
}
// Sort descending (newest first).
slices.SortFunc(parsed, func(a, b lexver.Version) int {
return lexver.Compare(b, a)
})
want := []string{"2.0.0", "2.0.0-rc1", "1.20.3", "1.20.2", "1.19.5", "1.0.0"}
for i, v := range parsed {
if v.Raw != want[i] {
t.Errorf("index %d: got %q, want %q", i, v.Raw, want[i])
}
}
}
func TestIsStable(t *testing.T) {
tests := []struct {
input string
want bool
}{
{"1.0.0", true},
{"121.0.6120.0", true},
{"1.0.0-beta1", false},
{"v2.0.0-dev", false},
}
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
v := lexver.Parse(tt.input)
if v.IsStable() != tt.want {
t.Errorf("Parse(%q).IsStable() = %v, want %v", tt.input, v.IsStable(), tt.want)
}
})
}
}
func TestDateTiebreaker(t *testing.T) {
a := lexver.Parse("1.0.0")
a.Date = time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC)
b := lexver.Parse("1.0.0")
b.Date = time.Date(2024, 6, 1, 14, 30, 0, 0, time.UTC)
if lexver.Compare(a, b) >= 0 {
t.Error("earlier date should sort before later date at same version")
}
// Without dates, same version is equal.
c := lexver.Parse("1.0.0")
d := lexver.Parse("1.0.0")
if lexver.Compare(c, d) != 0 {
t.Error("same version without dates should be equal")
}
// Date only matters when both have it.
e := lexver.Parse("1.0.0")
e.Date = time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC)
f := lexver.Parse("1.0.0")
if lexver.Compare(e, f) != 0 {
t.Error("date should be ignored when only one side has it")
}
}
func TestDateMinutePrecision(t *testing.T) {
a := lexver.Parse("1.0.0")
a.Date = time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
b := lexver.Parse("1.0.0")
b.Date = time.Date(2024, 1, 15, 10, 30, 0, 0, time.UTC)
if lexver.Compare(a, b) >= 0 {
t.Error("same date, later time should sort after")
}
}
func TestOriginal(t *testing.T) {
// Parse sets both Original and Raw to the input.
v := lexver.Parse("17.0")
if v.Original != "17.0" {
t.Errorf("Original = %q, want %q", v.Original, "17.0")
}
// Release fetcher would do:
// v := lexver.Parse("17.0")
// v.Original = "REL_17_0"
v.Original = "REL_17_0"
if v.Raw != "17.0" {
t.Errorf("Raw should remain %q after setting Original, got %q", "17.0", v.Raw)
}
}
func TestExtraSort(t *testing.T) {
// Flutter example: 2.3.0-16.0.pre and 2.3.0-16.1.pre
// Nums and Channel are the same; ExtraSort distinguishes them.
a := lexver.Parse("2.3.0-pre")
a.ExtraSort = "0016.0000"
b := lexver.Parse("2.3.0-pre")
b.ExtraSort = "0016.0001"
if lexver.Compare(a, b) >= 0 {
t.Error("ExtraSort 0016.0000 should sort before 0016.0001")
}
// ExtraSort ignored when only one side has it.
c := lexver.Parse("2.3.0-pre")
c.ExtraSort = "0016.0000"
d := lexver.Parse("2.3.0-pre")
if lexver.Compare(c, d) != 0 {
t.Error("ExtraSort should be ignored when only one side has it")
}
}
func TestHasPrefix(t *testing.T) {
v := lexver.Parse("1.20.3")
if !v.HasPrefix(lexver.Parse("1.20")) {
t.Error("1.20.3 should match prefix 1.20")
}
if !v.HasPrefix(lexver.Parse("1")) {
t.Error("1.20.3 should match prefix 1")
}
if v.HasPrefix(lexver.Parse("1.19")) {
t.Error("1.20.3 should not match prefix 1.19")
}
if v.HasPrefix(lexver.Parse("2")) {
t.Error("1.20.3 should not match prefix 2")
}
// 4-part prefix matching
v4 := lexver.Parse("121.0.6120.0")
if !v4.HasPrefix(lexver.Parse("121.0.6120")) {
t.Error("121.0.6120.0 should match prefix 121.0.6120")
}
if !v4.HasPrefix(lexver.Parse("121.0")) {
t.Error("121.0.6120.0 should match prefix 121.0")
}
}

View File

@@ -1,63 +0,0 @@
package rawcache
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"os"
"path/filepath"
"time"
)
// LogEntry records one event in the append-only audit log.
type LogEntry struct {
Time time.Time `json:"time"`
Tag string `json:"tag"`
Action string `json:"action"` // "added", "changed", "removed"
SHA256 string `json:"sha256,omitempty"`
}
// AuditLog is an append-only JSONL file that tracks when releases appear,
// change, or disappear from upstream. One file per package, lives alongside
// the double-buffer slots.
type AuditLog struct {
path string
}
// openLog returns the audit log for a Dir.
func (d *Dir) openLog() *AuditLog {
return &AuditLog{path: filepath.Join(d.root, "audit.jsonl")}
}
// Append writes one log entry.
func (l *AuditLog) Append(entry LogEntry) error {
if entry.Time.IsZero() {
entry.Time = time.Now().UTC()
}
data, err := json.Marshal(entry)
if err != nil {
return fmt.Errorf("rawcache: marshal log entry: %w", err)
}
data = append(data, '\n')
f, err := os.OpenFile(l.path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
if err != nil {
return fmt.Errorf("rawcache: open audit log: %w", err)
}
_, writeErr := f.Write(data)
closeErr := f.Close()
if writeErr != nil {
return fmt.Errorf("rawcache: write audit log: %w", writeErr)
}
if closeErr != nil {
return fmt.Errorf("rawcache: close audit log: %w", closeErr)
}
return nil
}
// ContentHash returns the SHA-256 hex digest of data.
func ContentHash(data []byte) string {
h := sha256.Sum256(data)
return hex.EncodeToString(h[:])
}

View File

@@ -1,265 +0,0 @@
// Package rawcache stores raw upstream API responses on disk, one file per
// release, with double-buffered full refreshes.
//
// Directory layout:
//
// {root}/
// active → a symlink to the current slot
// a/ slot A
// _latest one-line file: newest tag
// v0.145.0.json
// v0.144.1.json
// ...
// b/ slot B (standby)
//
// Incremental updates write directly to the active slot. Each file write
// is atomic (temp file + rename). Full refreshes write to the standby slot,
// then atomically swap the symlink.
package rawcache
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
)
// Dir manages a raw release cache for one package.
type Dir struct {
root string // e.g. "_cache/raw/github/gohugoio/hugo"
}
// Open returns a Dir for the given root path. Creates the directory
// structure (slots + symlink) if it doesn't exist.
func Open(root string) (*Dir, error) {
d := &Dir{root: root}
slotA := filepath.Join(root, "a")
slotB := filepath.Join(root, "b")
active := filepath.Join(root, "active")
// Create both slots.
for _, slot := range []string{slotA, slotB} {
if err := os.MkdirAll(slot, 0o755); err != nil {
return nil, fmt.Errorf("rawcache: create slot: %w", err)
}
}
// Create the active symlink if it doesn't exist.
if _, err := os.Lstat(active); errors.Is(err, os.ErrNotExist) {
if err := os.Symlink("a", active); err != nil {
return nil, fmt.Errorf("rawcache: create active symlink: %w", err)
}
}
return d, nil
}
// ActivePath returns the absolute path of the currently active slot.
func (d *Dir) ActivePath() (string, error) {
target, err := os.Readlink(filepath.Join(d.root, "active"))
if err != nil {
return "", fmt.Errorf("rawcache: read active symlink: %w", err)
}
return filepath.Join(d.root, target), nil
}
// standbySlot returns the name of the inactive slot ("a" or "b").
func (d *Dir) standbySlot() (string, error) {
target, err := os.Readlink(filepath.Join(d.root, "active"))
if err != nil {
return "", fmt.Errorf("rawcache: read active symlink: %w", err)
}
if target == "a" {
return "b", nil
}
return "a", nil
}
// Populated returns true if the active slot contains at least one release file.
func (d *Dir) Populated() bool {
active, err := d.ActivePath()
if err != nil {
return false
}
entries, err := os.ReadDir(active)
if err != nil {
return false
}
for _, e := range entries {
if !e.IsDir() && !strings.HasPrefix(e.Name(), "_") {
return true
}
}
return false
}
// Has reports whether a release file exists in the active slot.
func (d *Dir) Has(tag string) bool {
active, err := d.ActivePath()
if err != nil {
return false
}
_, err = os.Stat(filepath.Join(active, tagToFilename(tag)))
return err == nil
}
// Latest returns the newest tag from the active slot.
// Returns "" if no latest marker exists.
func (d *Dir) Latest() string {
active, err := d.ActivePath()
if err != nil {
return ""
}
data, err := os.ReadFile(filepath.Join(active, "_latest"))
if err != nil {
return ""
}
return strings.TrimSpace(string(data))
}
// Read returns the raw cached data for a tag from the active slot.
func (d *Dir) Read(tag string) ([]byte, error) {
active, err := d.ActivePath()
if err != nil {
return nil, err
}
return os.ReadFile(filepath.Join(active, tagToFilename(tag)))
}
// Put writes a release file to the active slot. The write is atomic
// (temp file + rename).
func (d *Dir) Put(tag string, data []byte) error {
active, err := d.ActivePath()
if err != nil {
return err
}
return atomicWrite(filepath.Join(active, tagToFilename(tag)), data)
}
// Merge writes a release to the active slot if it's new or changed.
// Returns the action taken: "added", "changed", or "" (unchanged).
// Logs the event to the audit log when something happens.
func (d *Dir) Merge(tag string, data []byte) (string, error) {
log := d.openLog()
hash := ContentHash(data)
if d.Has(tag) {
existing, err := d.Read(tag)
if err != nil {
return "", err
}
if ContentHash(existing) == hash {
return "", nil // unchanged
}
if err := d.Put(tag, data); err != nil {
return "", err
}
log.Append(LogEntry{Tag: tag, Action: "changed", SHA256: hash})
return "changed", nil
}
if err := d.Put(tag, data); err != nil {
return "", err
}
log.Append(LogEntry{Tag: tag, Action: "added", SHA256: hash})
return "added", nil
}
// SetLatest updates the _latest marker in the active slot.
func (d *Dir) SetLatest(tag string) error {
active, err := d.ActivePath()
if err != nil {
return err
}
return atomicWrite(filepath.Join(active, "_latest"), []byte(tag+"\n"))
}
// BeginRefresh starts a full refresh. Clears the standby slot and returns
// a Refresh handle for writing to it. Call Commit to atomically swap, or
// Abort to discard.
func (d *Dir) BeginRefresh() (*Refresh, error) {
standby, err := d.standbySlot()
if err != nil {
return nil, err
}
standbyPath := filepath.Join(d.root, standby)
// Clear the standby slot.
entries, _ := os.ReadDir(standbyPath)
for _, e := range entries {
os.Remove(filepath.Join(standbyPath, e.Name()))
}
return &Refresh{
dir: d,
slot: standby,
slotDir: standbyPath,
}, nil
}
// Refresh writes releases to the standby slot during a full refresh.
type Refresh struct {
dir *Dir
slot string // "a" or "b"
slotDir string
}
// Put writes a release file to the standby slot.
func (r *Refresh) Put(tag string, data []byte) error {
return atomicWrite(filepath.Join(r.slotDir, tagToFilename(tag)), data)
}
// SetLatest updates the _latest marker in the standby slot.
func (r *Refresh) SetLatest(tag string) error {
return atomicWrite(filepath.Join(r.slotDir, "_latest"), []byte(tag+"\n"))
}
// Commit atomically swaps the active symlink to point to the standby slot.
func (r *Refresh) Commit() error {
active := filepath.Join(r.dir.root, "active")
tmp := active + ".tmp"
// Remove stale temp symlink if it exists.
os.Remove(tmp)
if err := os.Symlink(r.slot, tmp); err != nil {
return fmt.Errorf("rawcache: create temp symlink: %w", err)
}
if err := os.Rename(tmp, active); err != nil {
os.Remove(tmp)
return fmt.Errorf("rawcache: swap active symlink: %w", err)
}
return nil
}
// Abort discards the standby slot contents.
func (r *Refresh) Abort() {
entries, _ := os.ReadDir(r.slotDir)
for _, e := range entries {
os.Remove(filepath.Join(r.slotDir, e.Name()))
}
}
// tagToFilename converts a tag to a safe filename.
// Tags like "v0.145.0" become "v0.145.0". The raw cache stores opaque
// bytes — no extension is assumed because upstream responses may be
// JSON, CSV, XML, or bespoke formats.
func tagToFilename(tag string) string {
// Replace path separators in case a tag contains slashes.
return strings.ReplaceAll(tag, "/", "_")
}
// atomicWrite writes data to path via a temp file + rename.
func atomicWrite(path string, data []byte) error {
tmp := path + ".tmp"
if err := os.WriteFile(tmp, data, 0o644); err != nil {
return fmt.Errorf("rawcache: write %s: %w", tmp, err)
}
if err := os.Rename(tmp, path); err != nil {
os.Remove(tmp)
return fmt.Errorf("rawcache: rename %s: %w", path, err)
}
return nil
}

View File

@@ -1,173 +0,0 @@
package rawcache_test
import (
"os"
"path/filepath"
"testing"
"github.com/webinstall/webi-installers/internal/rawcache"
)
func TestOpenCreatesStructure(t *testing.T) {
root := filepath.Join(t.TempDir(), "pkg")
d, err := rawcache.Open(root)
if err != nil {
t.Fatal(err)
}
_ = d
// Verify structure exists.
for _, name := range []string{"a", "b"} {
info, err := os.Stat(filepath.Join(root, name))
if err != nil {
t.Fatalf("slot %s: %v", name, err)
}
if !info.IsDir() {
t.Fatalf("slot %s is not a directory", name)
}
}
target, err := os.Readlink(filepath.Join(root, "active"))
if err != nil {
t.Fatal(err)
}
if target != "a" {
t.Errorf("active symlink = %q, want %q", target, "a")
}
}
func TestPutAndRead(t *testing.T) {
d, err := rawcache.Open(filepath.Join(t.TempDir(), "pkg"))
if err != nil {
t.Fatal(err)
}
data := []byte(`{"tag_name":"v1.0.0"}`)
if err := d.Put("v1.0.0", data); err != nil {
t.Fatal(err)
}
if !d.Has("v1.0.0") {
t.Error("Has(v1.0.0) = false after Put")
}
if d.Has("v2.0.0") {
t.Error("Has(v2.0.0) = true, should be false")
}
got, err := d.Read("v1.0.0")
if err != nil {
t.Fatal(err)
}
if string(got) != string(data) {
t.Errorf("Read = %q, want %q", got, data)
}
}
func TestLatest(t *testing.T) {
d, err := rawcache.Open(filepath.Join(t.TempDir(), "pkg"))
if err != nil {
t.Fatal(err)
}
if latest := d.Latest(); latest != "" {
t.Errorf("Latest() = %q before any writes, want empty", latest)
}
if err := d.SetLatest("v1.0.0"); err != nil {
t.Fatal(err)
}
if latest := d.Latest(); latest != "v1.0.0" {
t.Errorf("Latest() = %q, want %q", latest, "v1.0.0")
}
}
func TestRefreshDoubleBuffer(t *testing.T) {
root := filepath.Join(t.TempDir(), "pkg")
d, err := rawcache.Open(root)
if err != nil {
t.Fatal(err)
}
// Write to active slot (A).
d.Put("v1.0.0", []byte(`{"old":true}`))
d.SetLatest("v1.0.0")
// Start a full refresh — writes to standby (B).
r, err := d.BeginRefresh()
if err != nil {
t.Fatal(err)
}
r.Put("v1.0.0", []byte(`{"new":true}`))
r.Put("v2.0.0", []byte(`{"tag_name":"v2.0.0"}`))
r.SetLatest("v2.0.0")
// Before commit, active still points to A.
if d.Latest() != "v1.0.0" {
t.Error("latest should still be v1.0.0 before commit")
}
old, _ := d.Read("v1.0.0")
if string(old) != `{"old":true}` {
t.Errorf("active slot should still have old data, got %q", old)
}
// Commit swaps to B.
if err := r.Commit(); err != nil {
t.Fatal(err)
}
if d.Latest() != "v2.0.0" {
t.Errorf("Latest() = %q after commit, want %q", d.Latest(), "v2.0.0")
}
if !d.Has("v2.0.0") {
t.Error("v2.0.0 should exist after commit")
}
updated, _ := d.Read("v1.0.0")
if string(updated) != `{"new":true}` {
t.Errorf("v1.0.0 should be updated after commit, got %q", updated)
}
}
func TestRefreshAbort(t *testing.T) {
root := filepath.Join(t.TempDir(), "pkg")
d, err := rawcache.Open(root)
if err != nil {
t.Fatal(err)
}
d.Put("v1.0.0", []byte(`original`))
d.SetLatest("v1.0.0")
r, err := d.BeginRefresh()
if err != nil {
t.Fatal(err)
}
r.Put("v99.0.0", []byte(`aborted`))
r.Abort()
// Active slot should be unchanged.
if d.Latest() != "v1.0.0" {
t.Error("latest should still be v1.0.0 after abort")
}
if d.Has("v99.0.0") {
t.Error("v99.0.0 should not exist after abort")
}
}
func TestOpenIdempotent(t *testing.T) {
root := filepath.Join(t.TempDir(), "pkg")
d1, err := rawcache.Open(root)
if err != nil {
t.Fatal(err)
}
d1.Put("v1.0.0", []byte(`data`))
// Opening again should not lose data.
d2, err := rawcache.Open(root)
if err != nil {
t.Fatal(err)
}
if !d2.Has("v1.0.0") {
t.Error("data lost after re-open")
}
}

View File

@@ -1,50 +0,0 @@
// Package atomicparsley provides OS/arch classification for AtomicParsley releases.
//
// AtomicParsley uses non-standard filenames with no platform terms
// (e.g. "AtomicParsleyLinux.zip", "AtomicParsleyMacOS.zip"). The generic
// filename classifier can't extract OS or arch from these — this package
// applies the same hardcoded mapping that the production releases.js uses.
package atomicparsleydist
import (
"strings"
"github.com/webinstall/webi-installers/internal/storage"
)
// TagVariants sets OS, arch, and libc for AtomicParsley assets based on
// filename keyword matching. Replicates atomicparsley/releases.js mappings:
// - Alpine → linux/x86_64/musl
// - Linux → linux/x86_64/gnu
// - MacOS → darwin/x86_64
// - WindowsX86 → windows/x86/msvc
// - Windows → windows/x86_64/msvc
func TagVariants(assets []storage.Asset) {
for i := range assets {
if assets[i].OS != "" {
continue // already classified
}
lower := strings.ToLower(assets[i].Filename)
switch {
case strings.Contains(lower, "alpine"):
assets[i].OS = "linux"
assets[i].Arch = "x86_64"
assets[i].Libc = "musl"
case strings.Contains(lower, "linux"):
assets[i].OS = "linux"
assets[i].Arch = "x86_64"
assets[i].Libc = "gnu"
case strings.Contains(lower, "macos"):
assets[i].OS = "darwin"
assets[i].Arch = "x86_64"
case strings.Contains(lower, "windowsx86"):
assets[i].OS = "windows"
assets[i].Arch = "x86"
assets[i].Libc = "msvc"
case strings.Contains(lower, "windows"):
assets[i].OS = "windows"
assets[i].Arch = "x86_64"
assets[i].Libc = "msvc"
}
}
}

View File

@@ -1,36 +0,0 @@
// Package bun provides variant tagging for Bun releases.
//
// Bun publishes -profile (debug) builds and uses a non-standard arch
// convention: the default x86_64 build targets x86_64_v3 (AVX2+),
// while -baseline targets plain x86_64.
package bundist
import (
"strings"
"github.com/webinstall/webi-installers/internal/storage"
)
// TagVariants tags bun-specific build variants and remaps arch fields.
//
// Bun's default x86_64 build requires AVX2 (x86_64_v3). The -baseline
// build targets plain x86_64. For legacy export, baseline is the one
// we serve (matching Node.js behavior), so non-baseline gets a variant
// tag. The -baseline suffix is stripped from Filename (but not Download)
// so the legacy server sees a clean name.
func TagVariants(assets []storage.Asset) {
for i := range assets {
lower := strings.ToLower(assets[i].Filename)
if assets[i].Arch == "x86_64" {
if strings.Contains(lower, "-baseline") {
// Baseline is plain x86_64 — strip the suffix from
// Filename so the legacy server sees a clean name.
assets[i].Filename = strings.Replace(assets[i].Filename, "-baseline", "", 1)
} else {
// Non-baseline is v3 — tag as variant (excluded from legacy).
assets[i].Arch = "x86_64_v3"
assets[i].Variants = append(assets[i].Variants, "v3")
}
}
}
}

View File

@@ -1,72 +0,0 @@
// Package chromedist fetches Chrome for Testing release data.
//
// Google publishes a JSON index of known-good Chrome/ChromeDriver versions at:
//
// https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json
//
// Each version entry has per-platform download URLs for chrome, chromedriver,
// and chrome-headless-shell.
package chromedist
import (
"context"
"encoding/json"
"fmt"
"iter"
"net/http"
)
// Index is the top-level response.
type Index struct {
Timestamp string `json:"timestamp"`
Versions []Version `json:"versions"`
}
// Version is one Chrome for Testing version with its downloads.
type Version struct {
Version string `json:"version"` // "121.0.6120.0"
Revision string `json:"revision"` // "1222902"
Downloads map[string][]Download `json:"downloads"` // "chromedriver" → []Download
}
// Download is one platform-specific download URL.
type Download struct {
Platform string `json:"platform"` // "linux64", "mac-arm64", "mac-x64", "win32", "win64"
URL string `json:"url"`
}
// Fetch retrieves the Chrome for Testing release index.
//
// Yields one batch containing all versions.
func Fetch(ctx context.Context, client *http.Client) iter.Seq2[[]Version, error] {
return func(yield func([]Version, error) bool) {
url := "https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json"
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
yield(nil, fmt.Errorf("chromedist: %w", err))
return
}
req.Header.Set("Accept", "application/json")
resp, err := client.Do(req)
if err != nil {
yield(nil, fmt.Errorf("chromedist: fetch: %w", err))
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
yield(nil, fmt.Errorf("chromedist: fetch: %s", resp.Status))
return
}
var idx Index
if err := json.NewDecoder(resp.Body).Decode(&idx); err != nil {
yield(nil, fmt.Errorf("chromedist: decode: %w", err))
return
}
yield(idx.Versions, nil)
}
}

View File

@@ -1,60 +0,0 @@
package cmakedist
import (
"strings"
"github.com/webinstall/webi-installers/internal/storage"
)
// TagVariants tags cmake-specific build variants for exclusion from legacy export.
//
// cmake ships many formats and platforms that webi can't serve:
//
// - .sh self-extracting installer scripts: webi uses the .tar.gz archives.
//
// - .tar.Z files (old UNIX compress format): format not recognized by webi.
//
// - Darwin64 builds (pre-3.6 macOS naming): ancient format, superseded by
// the macos-universal builds.
//
// - sunos-sparc64 builds: unsupported platform (sparc64 arch not recognized).
//
// - AIX/powerpc builds: unsupported platform.
//
// - IRIX builds: unsupported platform.
//
// Note: macos10.N versioned builds (cmake-*-macos10.10-universal.tar.gz) are
// NOT dropped. Go correctly classifies them as os="darwin". The Node production
// classifier has a gap and can't parse "macos10.10" → that is a known prod bug,
// not a Go correctness issue. NODER should treat these as expected differences.
func TagVariants(assets []storage.Asset) {
for i := range assets {
lower := strings.ToLower(assets[i].Filename)
// Self-extracting installer scripts — webi uses .tar.gz archives.
if strings.HasSuffix(lower, ".sh") {
assets[i].Variants = append(assets[i].Variants, "installer")
continue
}
// Old UNIX compress format (.tar.Z) — not supported by webi.
if strings.HasSuffix(lower, ".tar.z") {
assets[i].Variants = append(assets[i].Variants, "legacy-archive")
continue
}
// Darwin64 builds: pre-cmake-3.6 macOS naming, superseded by macos-universal.
if strings.Contains(lower, "darwin64") {
assets[i].Variants = append(assets[i].Variants, "legacy-mac")
continue
}
// Unsupported platforms.
if strings.Contains(lower, "sunos") ||
strings.Contains(lower, "-aix-") ||
strings.Contains(lower, "irix") {
assets[i].Variants = append(assets[i].Variants, "unsupported-platform")
continue
}
}
}

View File

@@ -1,28 +0,0 @@
// Package fish provides variant tagging for fish shell releases.
//
// Fish publishes .pkg macOS installers alongside the standard archives.
// It also includes a source tarball (fish-{version}.tar.xz) as an
// uploaded release asset — no OS or arch in the name, indistinguishable
// from binaries by content_type. We tag it explicitly as "source".
package fishdist
import "github.com/webinstall/webi-installers/internal/storage"
// TagVariants tags fish-specific build variants.
func TagVariants(assets []storage.Asset) {
for i := range assets {
if assets[i].Format == ".pkg" {
assets[i].Variants = append(assets[i].Variants, "installer")
}
// Source tarball: no OS or arch detected by the classifier.
if assets[i].OS == "" && assets[i].Arch == "" {
assets[i].Variants = append(assets[i].Variants, "source")
}
// fish-*.app.zip is a macOS universal binary. Fish's naming puts
// arch in Linux filenames (e.g. fish-*-aarch64.tar.xz) but not in
// macOS .app.zip. Tag as x86_64; darwin waterfall serves arm64.
if assets[i].OS == "darwin" && assets[i].Arch == "" && assets[i].Format == ".app.zip" {
assets[i].Arch = "x86_64"
}
}
}

View File

@@ -1,94 +0,0 @@
// Package flutterdist fetches Flutter release data from Google Storage.
//
// Flutter publishes per-OS release indexes:
//
// https://storage.googleapis.com/flutter_infra_release/releases/releases_macos.json
// https://storage.googleapis.com/flutter_infra_release/releases/releases_linux.json
// https://storage.googleapis.com/flutter_infra_release/releases/releases_windows.json
//
// Each response has a base_url and a releases array with version, channel,
// release_date, archive path, and sha256.
package flutterdist
import (
"context"
"encoding/json"
"fmt"
"iter"
"net/http"
)
// index is the top-level JSON structure for one OS endpoint.
type index struct {
BaseURL string `json:"base_url"`
Releases []Release `json:"releases"`
}
// Release is one Flutter release entry.
type Release struct {
Hash string `json:"hash"` // git commit hash
Channel string `json:"channel"` // "stable", "beta", "dev"
Version string `json:"version"` // "3.29.2"
ReleaseDate string `json:"release_date"` // "2025-03-13T00:14:34.044690Z"
Archive string `json:"archive"` // "stable/macos/flutter_macos_arm64_3.29.2-stable.zip"
SHA256 string `json:"sha256"`
// DownloadURL is the fully-qualified URL, assembled from base_url + archive.
// Not in the upstream JSON — set by Fetch.
DownloadURL string `json:"download_url"`
// OS is the platform this entry came from ("macos", "linux", "windows").
// Not in the upstream JSON — set by Fetch.
OS string `json:"os"`
}
var defaultOSes = []string{"macos", "linux", "windows"}
// Fetch retrieves Flutter releases for all platforms.
//
// Yields one batch per OS. The iterator interface exists so callers use
// the same pattern as paginated sources.
func Fetch(ctx context.Context, client *http.Client) iter.Seq2[[]Release, error] {
return func(yield func([]Release, error) bool) {
for _, osName := range defaultOSes {
url := fmt.Sprintf(
"https://storage.googleapis.com/flutter_infra_release/releases/releases_%s.json",
osName,
)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
yield(nil, fmt.Errorf("flutterdist: %w", err))
return
}
req.Header.Set("Accept", "application/json")
resp, err := client.Do(req)
if err != nil {
yield(nil, fmt.Errorf("flutterdist: fetch %s: %w", osName, err))
return
}
var idx index
err = json.NewDecoder(resp.Body).Decode(&idx)
resp.Body.Close()
if err != nil {
yield(nil, fmt.Errorf("flutterdist: decode %s: %w", osName, err))
return
}
if resp.StatusCode != http.StatusOK {
yield(nil, fmt.Errorf("flutterdist: fetch %s: %s", osName, resp.Status))
return
}
for i := range idx.Releases {
idx.Releases[i].DownloadURL = idx.BaseURL + "/" + idx.Releases[i].Archive
idx.Releases[i].OS = osName
}
if !yield(idx.Releases, nil) {
return
}
}
}
}

View File

@@ -1,16 +0,0 @@
package flutterdist
import "github.com/webinstall/webi-installers/internal/storage"
// TagVariants handles flutter-specific arch defaults.
//
// Flutter's naming convention: flutter_{os}_{version} for x86_64 builds,
// flutter_{os}_arm64_{version} for arm64. The absence of an arch token
// means x86_64 — arm64 is always explicit.
func TagVariants(assets []storage.Asset) {
for i := range assets {
if assets[i].Arch == "" && assets[i].OS != "" {
assets[i].Arch = "x86_64"
}
}
}

View File

@@ -1,52 +0,0 @@
// Package git provides variant tagging for Git for Windows releases.
//
// Git for Windows publishes GUI installer .exe files (Git-*-bit.exe),
// self-extracting PortableGit archives, and .pdb debug symbol packages
// alongside the MinGit .zip that webi installs.
package gitdist
import (
"strings"
"github.com/webinstall/webi-installers/internal/storage"
)
// TagVariants tags git-specific build variants and fixes OS/arch classification.
// All git-for-windows releases are Windows-only, but MinGit filenames like
// "MinGit-2.33.0-64-bit.zip" have no "windows" indicator — force OS=windows.
// MinGit uses "64-bit"/"32-bit" for arch — a convention specific to this project
// that the general classifier intentionally does not handle.
func TagVariants(assets []storage.Asset) {
for i := range assets {
// All git-for-windows assets are Windows. Filenames like
// "MinGit-2.33.0-64-bit.zip" have no OS term; set it explicitly.
if assets[i].OS == "" {
assets[i].OS = "windows"
}
// MinGit uses "64-bit"→x86_64, "32-bit"→x86 naming.
// "arm64" is already handled by the general classifier.
if assets[i].Arch == "" {
lower := strings.ToLower(assets[i].Filename)
if strings.Contains(lower, "64-bit") {
assets[i].Arch = "x86_64"
} else if strings.Contains(lower, "32-bit") {
assets[i].Arch = "x86"
}
}
lower := strings.ToLower(assets[i].Filename)
if assets[i].Format == ".exe" {
assets[i].Variants = append(assets[i].Variants, "installer")
}
if strings.Contains(lower, "portablegit") {
assets[i].Variants = append(assets[i].Variants, "installer")
}
if strings.Contains(lower, "-pdb") || strings.Contains(lower, "pdbs-for-") {
assets[i].Variants = append(assets[i].Variants, "pdb")
}
if strings.Contains(lower, "-busybox") {
assets[i].Variants = append(assets[i].Variants, "busybox")
}
}
}

View File

@@ -1,33 +0,0 @@
package gitdist
import (
"strings"
"github.com/webinstall/webi-installers/internal/storage"
)
// NormalizeVersions strips the ".windows.N" suffix from Git for Windows
// version strings to match the upstream Git version scheme.
//
// Git for Windows tags are like "v2.53.0.windows.1" or "v2.53.0.windows.2".
// Node.js strips ".windows.1" entirely and replaces ".windows.N" (N>1)
// with ".N":
//
// v2.53.0.windows.1 → v2.53.0
// v2.53.0.windows.2 → v2.53.0.2
func NormalizeVersions(assets []storage.Asset) {
for i := range assets {
v := assets[i].Version
idx := strings.Index(v, ".windows.")
if idx < 0 {
continue
}
suffix := v[idx+len(".windows."):]
base := v[:idx]
if suffix == "1" {
assets[i].Version = base
} else {
assets[i].Version = base + "." + suffix
}
}
}

View File

@@ -1,120 +0,0 @@
// Package gitea fetches releases from a Gitea or Forgejo instance.
//
// Gitea's release API lives under:
//
// GET {baseurl}/api/v1/repos/{owner}/{repo}/releases
//
// The response shape is similar to GitHub's but not identical. This package
// handles pagination, authentication, and deserialization independently.
package gitea
import (
"context"
"encoding/json"
"fmt"
"iter"
"net/http"
"regexp"
"strings"
)
// Release is one release from the Gitea releases API.
type Release struct {
TagName string `json:"tag_name"`
Name string `json:"name"`
Prerelease bool `json:"prerelease"`
Draft bool `json:"draft"`
PublishedAt string `json:"published_at"` // "2023-11-05T06:38:05Z"
Assets []Asset `json:"assets"`
TarballURL string `json:"tarball_url"`
ZipballURL string `json:"zipball_url"`
}
// Asset is one downloadable file attached to a release.
type Asset struct {
Name string `json:"name"` // "pathman-v0.6.0-darwin-amd64.tar.gz"
BrowserDownloadURL string `json:"browser_download_url"` // full URL
Size int64 `json:"size"`
}
// Auth holds optional credentials for authenticated API access.
type Auth struct {
Token string // personal access token or API key
}
// Fetch retrieves releases from a Gitea instance, paginating automatically.
// Each yield is one page of releases.
//
// The baseURL should be the Gitea root (e.g. "https://git.rootprojects.org").
// The /api/v1 prefix is appended automatically.
func Fetch(ctx context.Context, client *http.Client, baseURL, owner, repo string, auth *Auth) iter.Seq2[[]Release, error] {
return func(yield func([]Release, error) bool) {
base := strings.TrimRight(baseURL, "/")
page := 1
for {
url := fmt.Sprintf("%s/api/v1/repos/%s/%s/releases?limit=50&page=%d",
base, owner, repo, page)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
yield(nil, fmt.Errorf("gitea: %w", err))
return
}
req.Header.Set("Accept", "application/json")
if auth != nil && auth.Token != "" {
req.Header.Set("Authorization", "token "+auth.Token)
}
resp, err := client.Do(req)
if err != nil {
yield(nil, fmt.Errorf("gitea: fetch %s: %w", url, err))
return
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
yield(nil, fmt.Errorf("gitea: fetch %s: %s", url, resp.Status))
return
}
var releases []Release
err = json.NewDecoder(resp.Body).Decode(&releases)
resp.Body.Close()
if err != nil {
yield(nil, fmt.Errorf("gitea: decode %s: %w", url, err))
return
}
if !yield(releases, nil) {
return
}
// Gitea uses Link headers like GitHub for pagination.
if nextURL := nextPageURL(resp.Header.Get("Link")); nextURL != "" {
url = nextURL
page++ // not strictly needed since we follow the URL, but keeps logic clear
continue
}
// No next link — also stop if we got fewer results than requested.
if len(releases) < 50 {
return
}
page++
}
}
}
var reNextLink = regexp.MustCompile(`<([^>]+)>;\s*rel="next"`)
func nextPageURL(link string) string {
if link == "" {
return ""
}
m := reNextLink.FindStringSubmatch(link)
if m == nil {
return ""
}
return m[1]
}

View File

@@ -1,107 +0,0 @@
package gitea_test
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"github.com/webinstall/webi-installers/internal/releases/gitea"
)
const testReleases = `[
{
"tag_name": "v0.6.0",
"name": "v0.6.0",
"prerelease": false,
"draft": false,
"published_at": "2023-11-05T06:38:05Z",
"tarball_url": "https://example.com/archive/v0.6.0.tar.gz",
"zipball_url": "https://example.com/archive/v0.6.0.zip",
"assets": [
{
"name": "tool-v0.6.0-linux-amd64.tar.gz",
"browser_download_url": "https://example.com/releases/download/v0.6.0/tool-v0.6.0-linux-amd64.tar.gz",
"size": 89215
}
]
}
]`
func TestFetch(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/v1/repos/root/tool/releases" {
t.Errorf("unexpected path: %s", r.URL.Path)
http.NotFound(w, r)
return
}
w.Write([]byte(testReleases))
}))
defer srv.Close()
ctx := context.Background()
var all []gitea.Release
for releases, err := range gitea.Fetch(ctx, srv.Client(), srv.URL, "root", "tool", nil) {
if err != nil {
t.Fatal(err)
}
all = append(all, releases...)
}
if len(all) != 1 {
t.Fatalf("got %d releases, want 1", len(all))
}
if all[0].TagName != "v0.6.0" {
t.Errorf("TagName = %q, want %q", all[0].TagName, "v0.6.0")
}
if len(all[0].Assets) != 1 {
t.Errorf("got %d assets, want 1", len(all[0].Assets))
}
if all[0].TarballURL == "" {
t.Error("TarballURL is empty")
}
}
func TestFetchAuth(t *testing.T) {
var gotAuth string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotAuth = r.Header.Get("Authorization")
w.Write([]byte("[]"))
}))
defer srv.Close()
ctx := context.Background()
auth := &gitea.Auth{Token: "abc123"}
for _, err := range gitea.Fetch(ctx, srv.Client(), srv.URL, "root", "tool", auth) {
if err != nil {
t.Fatal(err)
}
}
if gotAuth != "token abc123" {
t.Errorf("Authorization = %q, want %q", gotAuth, "token abc123")
}
}
func TestFetchLive(t *testing.T) {
if testing.Short() {
t.Skip("skipping network test in short mode")
}
ctx := context.Background()
client := &http.Client{}
var total int
for releases, err := range gitea.Fetch(ctx, client, "https://git.rootprojects.org", "root", "pathman", nil) {
if err != nil {
t.Fatal(err)
}
total += len(releases)
}
if total < 1 {
t.Errorf("got %d releases, expected at least 1", total)
}
t.Logf("fetched %d releases", total)
}

View File

@@ -1,25 +0,0 @@
// Package gitea provides variant tagging for Gitea releases.
//
// Gitea publishes "gogit" builds that use an alternative pure-Go Git
// backend instead of the default C Git library.
package gitea
import (
"strings"
"github.com/webinstall/webi-installers/internal/storage"
)
// TagVariants tags gitea-specific build variants.
//
// Files containing "-gogit-" in the filename are tagged with the "gogit"
// variant. These use a pure-Go Git backend rather than the default C Git
// library.
func TagVariants(assets []storage.Asset) {
for i := range assets {
lower := strings.ToLower(assets[i].Filename)
if strings.Contains(lower, "gogit") {
assets[i].Variants = append(assets[i].Variants, "gogit")
}
}
}

View File

@@ -1,25 +0,0 @@
// Package giteasrc fetches source archives from Gitea/Forgejo releases.
//
// Some packages are installed from the auto-generated source tarballs
// rather than uploaded binary assets. This package fetches releases and
// exposes the tarball/zipball URLs.
//
// Use [gitea] for packages that use uploaded binary assets.
package giteasrc
import (
"context"
"iter"
"net/http"
"github.com/webinstall/webi-installers/internal/releases/gitea"
)
// Fetch retrieves releases from a Gitea instance for the given owner/repo.
// Paginates automatically, yielding one batch per API page.
//
// Callers should use [gitea.Release.TarballURL] and
// [gitea.Release.ZipballURL] rather than the Assets list.
func Fetch(ctx context.Context, client *http.Client, baseURL, owner, repo string, auth *gitea.Auth) iter.Seq2[[]gitea.Release, error] {
return gitea.Fetch(ctx, client, baseURL, owner, repo, auth)
}

View File

@@ -1,22 +0,0 @@
// Package github fetches releases from the GitHub API.
//
// This is a thin wrapper around [githubish] that sets the base URL to
// https://api.github.com. Use [githubish] directly for Gitea, Forgejo,
// or other GitHub-compatible forges.
package github
import (
"context"
"iter"
"net/http"
"github.com/webinstall/webi-installers/internal/releases/githubish"
)
const baseURL = "https://api.github.com"
// Fetch retrieves releases from GitHub for the given owner/repo.
// Paginates automatically, yielding one batch per API page.
func Fetch(ctx context.Context, client *http.Client, owner, repo string, auth *githubish.Auth) iter.Seq2[[]githubish.Release, error] {
return githubish.Fetch(ctx, client, baseURL, owner, repo, auth)
}

View File

@@ -1,112 +0,0 @@
// Package githubish fetches releases from GitHub-compatible APIs.
//
// GitHub, Gitea, Forgejo, and other forges expose the same releases
// endpoint shape:
//
// GET /repos/{owner}/{repo}/releases
//
// This package handles pagination (Link headers), authentication, and
// deserialization. It does not transform or normalize the data.
package githubish
import (
"context"
"encoding/json"
"fmt"
"iter"
"net/http"
"regexp"
)
// Release is one release from a GitHub-compatible API.
// Fields mirror the upstream JSON — only the fields Webi cares about are
// included; the rest are silently dropped by the decoder.
type Release struct {
TagName string `json:"tag_name"`
Name string `json:"name"`
Prerelease bool `json:"prerelease"`
Draft bool `json:"draft"`
PublishedAt string `json:"published_at"` // "2025-10-22T13:00:26Z"
Assets []Asset `json:"assets"`
TarballURL string `json:"tarball_url"` // auto-generated source tarball
ZipballURL string `json:"zipball_url"` // auto-generated source zipball
}
// Asset is one downloadable file attached to a release.
type Asset struct {
Name string `json:"name"` // "ripgrep-15.1.0-x86_64-apple-darwin.tar.gz"
BrowserDownloadURL string `json:"browser_download_url"` // full URL
Size int64 `json:"size"`
ContentType string `json:"content_type"`
}
// Auth holds optional credentials for authenticated API access.
// Without auth, GitHub's public rate limit is 60 requests/hour.
type Auth struct {
Token string // personal access token or fine-grained token
}
// Fetch retrieves releases from a GitHub-compatible API, paginating
// automatically. Each yield is one page of releases.
//
// The baseURL should be the API root (e.g. "https://api.github.com").
// For Gitea: "https://gitea.example.com/api/v1".
func Fetch(ctx context.Context, client *http.Client, baseURL, owner, repo string, auth *Auth) iter.Seq2[[]Release, error] {
return func(yield func([]Release, error) bool) {
url := fmt.Sprintf("%s/repos/%s/%s/releases?per_page=100", baseURL, owner, repo)
for url != "" {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
yield(nil, fmt.Errorf("githubish: %w", err))
return
}
req.Header.Set("Accept", "application/json")
if auth != nil && auth.Token != "" {
req.Header.Set("Authorization", "Bearer "+auth.Token)
}
resp, err := client.Do(req)
if err != nil {
yield(nil, fmt.Errorf("githubish: fetch %s: %w", url, err))
return
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
yield(nil, fmt.Errorf("githubish: fetch %s: %s", url, resp.Status))
return
}
var releases []Release
err = json.NewDecoder(resp.Body).Decode(&releases)
resp.Body.Close()
if err != nil {
yield(nil, fmt.Errorf("githubish: decode %s: %w", url, err))
return
}
if !yield(releases, nil) {
return
}
url = nextPageURL(resp.Header.Get("Link"))
}
}
}
// reNextLink matches `<URL>; rel="next"` in a Link header.
var reNextLink = regexp.MustCompile(`<([^>]+)>;\s*rel="next"`)
// nextPageURL extracts the "next" URL from a GitHub Link header.
// Returns "" if there is no next page.
func nextPageURL(link string) string {
if link == "" {
return ""
}
m := reNextLink.FindStringSubmatch(link)
if m == nil {
return ""
}
return m[1]
}

View File

@@ -1,201 +0,0 @@
package githubish_test
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/webinstall/webi-installers/internal/releases/githubish"
)
const page1 = `[
{
"tag_name": "v2.0.0",
"name": "v2.0.0",
"prerelease": false,
"draft": false,
"published_at": "2025-06-01T12:00:00Z",
"assets": [
{
"name": "tool-v2.0.0-linux-amd64.tar.gz",
"browser_download_url": "https://example.com/tool-v2.0.0-linux-amd64.tar.gz",
"size": 5000000,
"content_type": "application/gzip"
}
]
}
]`
const page2 = `[
{
"tag_name": "v1.0.0",
"name": "v1.0.0",
"prerelease": false,
"draft": false,
"published_at": "2024-01-15T08:00:00Z",
"assets": [
{
"name": "tool-v1.0.0-linux-amd64.tar.gz",
"browser_download_url": "https://example.com/tool-v1.0.0-linux-amd64.tar.gz",
"size": 4000000,
"content_type": "application/gzip"
},
{
"name": "tool-v1.0.0-darwin-arm64.tar.gz",
"browser_download_url": "https://example.com/tool-v1.0.0-darwin-arm64.tar.gz",
"size": 4500000,
"content_type": "application/gzip"
}
]
}
]`
func TestFetchPagination(t *testing.T) {
var srvURL string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/repos/acme/tool/releases" {
t.Errorf("unexpected path: %s", r.URL.Path)
http.NotFound(w, r)
return
}
page := r.URL.Query().Get("page")
switch page {
case "", "1":
// Link header pointing to page 2
w.Header().Set("Link",
fmt.Sprintf(`<%s/repos/acme/tool/releases?per_page=100&page=2>; rel="next"`, srvURL))
w.Write([]byte(page1))
case "2":
// No Link header — last page
w.Write([]byte(page2))
default:
http.NotFound(w, r)
}
}))
defer srv.Close()
srvURL = srv.URL
ctx := context.Background()
var batches int
var allReleases []githubish.Release
for releases, err := range githubish.Fetch(ctx, srv.Client(), srv.URL, "acme", "tool", nil) {
if err != nil {
t.Fatalf("batch %d: %v", batches, err)
}
batches++
allReleases = append(allReleases, releases...)
}
if batches != 2 {
t.Errorf("got %d batches, want 2", batches)
}
if len(allReleases) != 2 {
t.Fatalf("got %d releases, want 2", len(allReleases))
}
// Page 1: v2.0.0
if allReleases[0].TagName != "v2.0.0" {
t.Errorf("release[0].TagName = %q, want %q", allReleases[0].TagName, "v2.0.0")
}
if len(allReleases[0].Assets) != 1 {
t.Errorf("release[0] has %d assets, want 1", len(allReleases[0].Assets))
}
// Page 2: v1.0.0
if allReleases[1].TagName != "v1.0.0" {
t.Errorf("release[1].TagName = %q, want %q", allReleases[1].TagName, "v1.0.0")
}
if len(allReleases[1].Assets) != 2 {
t.Errorf("release[1] has %d assets, want 2", len(allReleases[1].Assets))
}
}
func TestFetchPrerelease(t *testing.T) {
body := `[{"tag_name":"v1.0.0-rc1","name":"","prerelease":true,"draft":false,"published_at":"2025-01-01T00:00:00Z","assets":[]}]`
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(body))
}))
defer srv.Close()
ctx := context.Background()
for releases, err := range githubish.Fetch(ctx, srv.Client(), srv.URL, "acme", "tool", nil) {
if err != nil {
t.Fatal(err)
}
if len(releases) != 1 {
t.Fatalf("got %d releases, want 1", len(releases))
}
if !releases[0].Prerelease {
t.Error("expected Prerelease = true")
}
if releases[0].TagName != "v1.0.0-rc1" {
t.Errorf("TagName = %q, want %q", releases[0].TagName, "v1.0.0-rc1")
}
}
}
func TestFetchAuth(t *testing.T) {
var gotAuth string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotAuth = r.Header.Get("Authorization")
w.Write([]byte("[]"))
}))
defer srv.Close()
ctx := context.Background()
auth := &githubish.Auth{Token: "ghp_test123"}
for _, err := range githubish.Fetch(ctx, srv.Client(), srv.URL, "acme", "tool", auth) {
if err != nil {
t.Fatal(err)
}
}
if gotAuth != "Bearer ghp_test123" {
t.Errorf("Authorization = %q, want %q", gotAuth, "Bearer ghp_test123")
}
}
func TestFetchHTTPError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "not found", http.StatusNotFound)
}))
defer srv.Close()
ctx := context.Background()
for _, err := range githubish.Fetch(ctx, srv.Client(), srv.URL, "acme", "tool", nil) {
if err == nil {
t.Fatal("expected error for 404 response")
}
return
}
}
func TestFetchEarlyBreak(t *testing.T) {
var requests int
var srvURL string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests++
// Always advertise a next page
w.Header().Set("Link",
fmt.Sprintf(`<%s/repos/acme/tool/releases?per_page=100&page=%d>; rel="next"`, srvURL, requests+1))
w.Write([]byte(`[{"tag_name":"v1.0.0","name":"","prerelease":false,"draft":false,"published_at":"2025-01-01T00:00:00Z","assets":[]}]`))
}))
defer srv.Close()
srvURL = srv.URL
ctx := context.Background()
for _, err := range githubish.Fetch(ctx, srv.Client(), srv.URL, "acme", "tool", nil) {
if err != nil {
t.Fatal(err)
}
break // stop after first page
}
if requests != 1 {
t.Errorf("server received %d requests, want 1 (early break should stop pagination)", requests)
}
}

View File

@@ -1,27 +0,0 @@
// Package githubsrc fetches source archives from GitHub releases.
//
// Some packages (shell scripts, vim plugins) are installed from the
// auto-generated source tarballs rather than uploaded binary assets.
// This package fetches releases and exposes the tarball/zipball URLs.
//
// Use [github] for packages that use uploaded binary assets.
package githubsrc
import (
"context"
"iter"
"net/http"
"github.com/webinstall/webi-installers/internal/releases/githubish"
)
const baseURL = "https://api.github.com"
// Fetch retrieves releases from GitHub for the given owner/repo.
// Paginates automatically, yielding one batch per API page.
//
// Callers should use [githubish.Release.TarballURL] and
// [githubish.Release.ZipballURL] rather than the Assets list.
func Fetch(ctx context.Context, client *http.Client, owner, repo string, auth *githubish.Auth) iter.Seq2[[]githubish.Release, error] {
return githubish.Fetch(ctx, client, baseURL, owner, repo, auth)
}

View File

@@ -1,122 +0,0 @@
// Package gitlab fetches releases from a GitLab instance.
//
// GitLab's releases API differs from GitHub's in structure:
//
// GET /api/v4/projects/:id/releases
//
// Where :id is the URL-encoded project path (e.g. "group%2Frepo") or a
// numeric project ID. Assets are split into auto-generated source archives
// and manually attached links. Pagination uses page/per_page query params
// and X-Total-Pages response headers (not Link headers).
//
// This package handles pagination, authentication, and deserialization.
// It does not transform or normalize the data.
package gitlab
import (
"context"
"encoding/json"
"fmt"
"iter"
"net/http"
"net/url"
"strconv"
)
// Release is one release from the GitLab releases API.
type Release struct {
TagName string `json:"tag_name"`
Name string `json:"name"`
ReleasedAt string `json:"released_at"` // "2025-10-22T13:00:26Z"
Assets Assets `json:"assets"`
}
// Assets holds both auto-generated source archives and attached links.
type Assets struct {
Sources []Source `json:"sources"`
Links []Link `json:"links"`
}
// Source is an auto-generated source archive (tar.gz, zip, etc.).
type Source struct {
Format string `json:"format"` // "zip", "tar.gz", "tar.bz2", "tar"
URL string `json:"url"`
}
// Link is a file attached to a release (binary, package, etc.).
type Link struct {
ID int `json:"id"`
Name string `json:"name"`
URL string `json:"url"`
DirectAssetPath string `json:"direct_asset_path"`
LinkType string `json:"link_type"` // "other", "runbook", "image", "package"
}
// Auth holds optional credentials for authenticated API access.
type Auth struct {
Token string // personal access token or deploy token
}
// Fetch retrieves releases from a GitLab instance, paginating automatically.
// Each yield is one page of releases.
//
// The baseURL should be the GitLab root (e.g. "https://gitlab.com").
// The project is identified by its path (e.g. "group/repo") — it will be
// URL-encoded automatically.
func Fetch(ctx context.Context, client *http.Client, baseURL, project string, auth *Auth) iter.Seq2[[]Release, error] {
return func(yield func([]Release, error) bool) {
encodedProject := url.PathEscape(project)
page := 1
for {
reqURL := fmt.Sprintf("%s/api/v4/projects/%s/releases?per_page=100&page=%d",
baseURL, encodedProject, page)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil)
if err != nil {
yield(nil, fmt.Errorf("gitlab: %w", err))
return
}
req.Header.Set("Accept", "application/json")
if auth != nil && auth.Token != "" {
req.Header.Set("PRIVATE-TOKEN", auth.Token)
}
resp, err := client.Do(req)
if err != nil {
yield(nil, fmt.Errorf("gitlab: fetch %s: %w", reqURL, err))
return
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
yield(nil, fmt.Errorf("gitlab: fetch %s: %s", reqURL, resp.Status))
return
}
var releases []Release
err = json.NewDecoder(resp.Body).Decode(&releases)
resp.Body.Close()
if err != nil {
yield(nil, fmt.Errorf("gitlab: decode %s: %w", reqURL, err))
return
}
if !yield(releases, nil) {
return
}
// Check if there are more pages.
totalPages := 1
if tp := resp.Header.Get("X-Total-Pages"); tp != "" {
if n, err := strconv.Atoi(tp); err == nil {
totalPages = n
}
}
if page >= totalPages {
return
}
page++
}
}
}

View File

@@ -1,182 +0,0 @@
package gitlab_test
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/webinstall/webi-installers/internal/releases/gitlab"
)
const page1 = `[
{
"tag_name": "v2.0.0",
"name": "v2.0.0",
"released_at": "2025-06-01T12:00:00Z",
"assets": {
"sources": [
{"format": "tar.gz", "url": "https://example.com/archive/v2.0.0.tar.gz"},
{"format": "zip", "url": "https://example.com/archive/v2.0.0.zip"}
],
"links": [
{
"id": 1,
"name": "tool-v2.0.0-linux-amd64.tar.gz",
"url": "https://example.com/tool-v2.0.0-linux-amd64.tar.gz",
"direct_asset_path": "/binaries/linux-amd64",
"link_type": "package"
}
]
}
}
]`
const page2 = `[
{
"tag_name": "v1.0.0",
"name": "v1.0.0",
"released_at": "2024-01-15T08:00:00Z",
"assets": {
"sources": [
{"format": "tar.gz", "url": "https://example.com/archive/v1.0.0.tar.gz"}
],
"links": []
}
}
]`
func TestFetchPagination(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Go's http server decodes %2F back to /, so check RawPath
// for the encoded form or Path for the decoded form.
wantRaw := "/api/v4/projects/group%2Ftool/releases"
wantDecoded := "/api/v4/projects/group/tool/releases"
if r.URL.RawPath != wantRaw && r.URL.Path != wantDecoded {
t.Errorf("unexpected path: raw=%q decoded=%q", r.URL.RawPath, r.URL.Path)
http.NotFound(w, r)
return
}
page := r.URL.Query().Get("page")
w.Header().Set("X-Total-Pages", "2")
switch page {
case "", "1":
w.Write([]byte(page1))
case "2":
w.Write([]byte(page2))
default:
http.NotFound(w, r)
}
}))
defer srv.Close()
ctx := context.Background()
var batches int
var allReleases []gitlab.Release
for releases, err := range gitlab.Fetch(ctx, srv.Client(), srv.URL, "group/tool", nil) {
if err != nil {
t.Fatalf("batch %d: %v", batches, err)
}
batches++
allReleases = append(allReleases, releases...)
}
if batches != 2 {
t.Errorf("got %d batches, want 2", batches)
}
if len(allReleases) != 2 {
t.Fatalf("got %d releases, want 2", len(allReleases))
}
// Page 1: v2.0.0
r1 := allReleases[0]
if r1.TagName != "v2.0.0" {
t.Errorf("release[0].TagName = %q, want %q", r1.TagName, "v2.0.0")
}
if len(r1.Assets.Sources) != 2 {
t.Errorf("release[0] has %d sources, want 2", len(r1.Assets.Sources))
}
if len(r1.Assets.Links) != 1 {
t.Errorf("release[0] has %d links, want 1", len(r1.Assets.Links))
}
if r1.Assets.Links[0].LinkType != "package" {
t.Errorf("release[0] link type = %q, want %q", r1.Assets.Links[0].LinkType, "package")
}
// Page 2: v1.0.0
r2 := allReleases[1]
if r2.TagName != "v1.0.0" {
t.Errorf("release[1].TagName = %q, want %q", r2.TagName, "v1.0.0")
}
if len(r2.Assets.Links) != 0 {
t.Errorf("release[1] has %d links, want 0", len(r2.Assets.Links))
}
}
func TestFetchAuth(t *testing.T) {
var gotAuth string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotAuth = r.Header.Get("PRIVATE-TOKEN")
w.Write([]byte("[]"))
}))
defer srv.Close()
ctx := context.Background()
auth := &gitlab.Auth{Token: "glpat-test123"}
for _, err := range gitlab.Fetch(ctx, srv.Client(), srv.URL, "group/tool", auth) {
if err != nil {
t.Fatal(err)
}
}
if gotAuth != "glpat-test123" {
t.Errorf("PRIVATE-TOKEN = %q, want %q", gotAuth, "glpat-test123")
}
}
func TestFetchSinglePage(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// No X-Total-Pages header — defaults to 1 page.
w.Write([]byte(page1))
}))
defer srv.Close()
ctx := context.Background()
var batches int
for _, err := range gitlab.Fetch(ctx, srv.Client(), srv.URL, "group/tool", nil) {
if err != nil {
t.Fatal(err)
}
batches++
}
if batches != 1 {
t.Errorf("got %d batches, want 1 (no X-Total-Pages means single page)", batches)
}
}
func TestFetchEarlyBreak(t *testing.T) {
var requests int
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests++
w.Header().Set("X-Total-Pages", "10")
w.Write([]byte(fmt.Sprintf(`[{"tag_name":"v%d.0.0","name":"","released_at":"2025-01-01T00:00:00Z","assets":{"sources":[],"links":[]}}]`, requests)))
}))
defer srv.Close()
ctx := context.Background()
for _, err := range gitlab.Fetch(ctx, srv.Client(), srv.URL, "group/tool", nil) {
if err != nil {
t.Fatal(err)
}
break // stop after first page
}
if requests != 1 {
t.Errorf("server received %d requests, want 1", requests)
}
}

View File

@@ -1,25 +0,0 @@
// Package gitlabsrc fetches source archives from GitLab releases.
//
// Some packages are installed from the auto-generated source archives
// rather than attached binary links. This package fetches releases and
// exposes the source archive URLs.
//
// Use [gitlab] for packages that use attached release links (binaries).
package gitlabsrc
import (
"context"
"iter"
"net/http"
"github.com/webinstall/webi-installers/internal/releases/gitlab"
)
// Fetch retrieves releases from a GitLab instance.
// Paginates automatically, yielding one batch per API page.
//
// Callers should use [gitlab.Release.Assets.Sources] rather than
// [gitlab.Release.Assets.Links].
func Fetch(ctx context.Context, client *http.Client, baseURL, project string, auth *gitlab.Auth) iter.Seq2[[]gitlab.Release, error] {
return gitlab.Fetch(ctx, client, baseURL, project, auth)
}

View File

@@ -1,178 +0,0 @@
// Package gittag fetches release information from git tags in a bare repo.
//
// Some packages (vim plugins, shell scripts) are installed by cloning a git
// repo rather than downloading a binary. For these, each tag is a "release"
// and the download URL is the repo's git URL.
//
// This package clones (or fetches) a bare repo to a local cache directory,
// lists version-like tags, and returns them with their commit metadata.
// HEAD is also included as a potential release.
package gittag
import (
"context"
"fmt"
"iter"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"crypto/rand"
"encoding/hex"
)
// Entry is one tag (or HEAD) from a git repo.
type Entry struct {
Version string // tag name or date-based version for HEAD
GitTag string // the ref that can be passed to `git clone --branch`
CommitHash string // abbreviated commit hash
Date string // ISO 8601 commit date (author date)
}
// reVersionTag matches tags that look like versions: v1, v1.2, 1.0.0-rc, etc.
var reVersionTag = regexp.MustCompile(`^v?\d+(\.\d+)`)
// Fetch clones or updates a bare repo, then yields its version-like tags
// and HEAD as entries. The repoDir is the parent directory where bare repos
// are cached.
//
// Yields one batch containing all tags plus HEAD.
func Fetch(ctx context.Context, gitURL, repoDir string) iter.Seq2[[]Entry, error] {
return func(yield func([]Entry, error) bool) {
repoName := filepath.Base(gitURL)
repoName = strings.TrimSuffix(repoName, ".git")
repoPath := filepath.Join(repoDir, repoName+".git")
if err := ensureRepo(ctx, repoPath, gitURL); err != nil {
yield(nil, fmt.Errorf("gittag: %w", err))
return
}
tags, err := listVersionTags(ctx, repoPath)
if err != nil {
yield(nil, fmt.Errorf("gittag: %w", err))
return
}
var entries []Entry
for _, tag := range tags {
info, err := commitInfo(ctx, repoPath, tag)
if err != nil {
yield(nil, fmt.Errorf("gittag: commit info for %q: %w", tag, err))
return
}
info.Version = tag
info.GitTag = tag
entries = append(entries, info)
}
// HEAD as an additional entry
head, err := commitInfo(ctx, repoPath, "HEAD")
if err != nil {
yield(nil, fmt.Errorf("gittag: commit info for HEAD: %w", err))
return
}
branch, err := headBranch(ctx, repoPath)
if err != nil {
yield(nil, fmt.Errorf("gittag: HEAD branch: %w", err))
return
}
head.GitTag = branch
// Version for HEAD is set by the caller (date-based, etc.)
entries = append(entries, head)
yield(entries, nil)
}
}
// ensureRepo clones the repo if it doesn't exist, or fetches if it does.
func ensureRepo(ctx context.Context, repoPath, gitURL string) error {
if _, err := os.Stat(repoPath); err == nil {
// Exists — fetch updates.
cmd := exec.CommandContext(ctx, "git", "--git-dir="+repoPath, "fetch")
cmd.Stderr = os.Stderr
return cmd.Run()
}
// Clone bare with tree filter (metadata only).
var b [8]byte
rand.Read(b[:])
id := hex.EncodeToString(b[:])
tmpPath := repoPath + "." + id + ".tmp"
cmd := exec.CommandContext(ctx, "git", "clone", "--bare", "--filter=tree:0", gitURL, tmpPath)
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
os.RemoveAll(tmpPath)
return fmt.Errorf("clone %s: %w", gitURL, err)
}
// Atomic swap — if repoPath appeared in a race, keep it and discard ours.
if err := os.Rename(tmpPath, repoPath); err != nil {
os.RemoveAll(tmpPath)
// If rename failed because repoPath now exists, that's fine.
if _, statErr := os.Stat(repoPath); statErr == nil {
return nil
}
return err
}
return nil
}
// listVersionTags returns tags that look like version numbers, newest first.
func listVersionTags(ctx context.Context, repoPath string) ([]string, error) {
cmd := exec.CommandContext(ctx, "git", "--git-dir="+repoPath, "tag")
out, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("git tag: %w", err)
}
var tags []string
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
if line == "" {
continue
}
if reVersionTag.MatchString(line) {
tags = append(tags, line)
}
}
// Reverse so newest tags come first (git tag outputs alphabetically).
for i, j := 0, len(tags)-1; i < j; i, j = i+1, j-1 {
tags[i], tags[j] = tags[j], tags[i]
}
return tags, nil
}
// commitInfo returns the abbreviated hash and author date for a commitish.
func commitInfo(ctx context.Context, repoPath, commitish string) (Entry, error) {
cmd := exec.CommandContext(ctx, "git", "--git-dir="+repoPath,
"log", "-1", "--format=%h %ad", "--date=iso-strict", commitish)
out, err := cmd.Output()
if err != nil {
return Entry{}, fmt.Errorf("git log %s: %w", commitish, err)
}
parts := strings.Fields(strings.TrimSpace(string(out)))
if len(parts) < 2 {
return Entry{}, fmt.Errorf("unexpected git log output: %q", out)
}
return Entry{
CommitHash: parts[0],
Date: parts[1],
}, nil
}
// headBranch returns the symbolic ref for HEAD (e.g. "main", "master").
func headBranch(ctx context.Context, repoPath string) (string, error) {
cmd := exec.CommandContext(ctx, "git", "--git-dir="+repoPath,
"rev-parse", "--abbrev-ref", "HEAD")
out, err := cmd.Output()
if err != nil {
return "", fmt.Errorf("git rev-parse HEAD: %w", err)
}
return strings.TrimSpace(string(out)), nil
}

View File

@@ -1,56 +0,0 @@
package gittag_test
import (
"context"
"testing"
"github.com/webinstall/webi-installers/internal/releases/gittag"
)
func TestFetch(t *testing.T) {
if testing.Short() {
t.Skip("skipping network/git test in short mode")
}
ctx := context.Background()
repoDir := t.TempDir()
// vim-commentary has a small number of tags.
var entries []gittag.Entry
for batch, err := range gittag.Fetch(ctx, "https://github.com/tpope/vim-commentary.git", repoDir) {
if err != nil {
t.Fatalf("Fetch: %v", err)
}
entries = append(entries, batch...)
}
if len(entries) < 2 {
t.Fatalf("got %d entries, expected at least 2 (tags + HEAD)", len(entries))
}
// Last entry should be HEAD (no Version set by the fetcher).
head := entries[len(entries)-1]
if head.CommitHash == "" {
t.Error("HEAD entry has empty CommitHash")
}
if head.Date == "" {
t.Error("HEAD entry has empty Date")
}
if head.GitTag == "" {
t.Error("HEAD entry has empty GitTag (branch name)")
}
// At least one tag entry should have a version.
found := false
for _, e := range entries[:len(entries)-1] {
if e.Version != "" {
found = true
break
}
}
if !found {
t.Error("no tag entries have a Version set")
}
t.Logf("fetched %d entries (last is HEAD on %q)", len(entries), head.GitTag)
}

Some files were not shown because too many files have changed in this diff Show More