diff --git a/.gitignore b/.gitignore index 5f5aa3b..0f6658f 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,11 @@ install-*.ps1 _cache/ node_modules/ +# local test fixtures (regenerated by _webi/test-live-*.js) +testdata/ +LIVE_cache/ +distributables.csv + # temporary & backup files .*.sw* *.bak @@ -30,6 +35,7 @@ node_modules/ # agent session files agents/ +LOCAL.md # other .DS_Store diff --git a/AGENTS.md b/AGENTS.md index e701723..9832f7d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,6 +3,26 @@ Webi installs dev tools to `~/.local/` without sudo. Each installer is a small package of 3-4 files. This guide tells you how to create and modify them. +## Domains + +| Environment | Domains | +| ----------- | ------------------------------------------------ | +| Production | webi.sh, webi.ms, webinstall.dev | +| Beta | beta.webi.sh, beta.webi.ms, beta.webinstall.dev | +| Next | next.webi.sh, next.webi.ms, next.webinstall.dev | + +- **webi.sh** — POSIX shell installer: `curl https://webi.sh/node | sh` +- **webi.ms** — PowerShell installer: `curl.exe https://webi.ms/node | powershell` +- **webinstall.dev** — canonical domain, serves both shell scripts and the + browser-facing cheat sheet pages + +The domain controls the default script format. You can force the same behavior +on any domain via User-Agent: +- `curl -A "MS" ...` — triggers PowerShell output (same as webi.ms) +- `curl ...` (default UA) — triggers POSIX shell output (same as webi.sh) +- `curl -A "$(uname -srm)" ...` — once the API is activated, the full + `uname -srm` string (e.g. `Linux 6.1.0 x86_64`) guides OS/arch selection + ## Why Webi Exists Webi makes tool installation trivially repeatable for people who aren't @@ -37,9 +57,11 @@ about PATH, permissions, or platform differences. Three things matter: Key infrastructure directories (do not modify without good reason): -- `_webi/` — bootstrap templates, `normalize.js` (auto-detects OS/arch/ext from - filenames) -- `_common/` — shared JS: `github.js`, `githubish.js`, `gitea.js`, `fetcher.js` +- `_webi/` — bootstrap templates, `transform-releases.js` (API endpoint), + `builds-cacher.js` (reads cache JSON), `serve-installer.js` (installer + scripts) +- `_common/` — shared JS fetcher libraries (being phased out — Go daemon now + fetches upstream) - `_example/` — canonical template for new packages - `_examples/` — specialized templates (goreleaser, xz-compressed) @@ -57,83 +79,50 @@ Ref: | 🔗 | Alias/redirect to another package | `ripgrep` → `rg` | | 📝 | Bespoke / custom install | `rustlang` | -## releases.js +## Data Architecture -Fetches release metadata and returns a normalized object. Most packages use -GitHub releases: +There are two data paths. Both read from pre-generated cache — the Node.js +server does NOT fetch upstream APIs. -```js -'use strict'; +``` +API path (JSON/TAB output): + Request → transform-releases.js → ~/.cache/webi/legacy/{pkg}.json → filter + sort + Vocabulary: macos, amd64, arm64, armv7l (API vocabulary) + normalize.js: REMOVED — cache provides all fields directly -var github = require('../_common/github.js'); -var owner = 'OWNER'; -var repo = 'REPO'; - -let Releases = module.exports; - -Releases.latest = async function () { - let all = await github(null, owner, repo); - return all; -}; - -Releases.sample = async function () { - let normalize = require('../_webi/normalize.js'); - let all = await Releases.latest(); - all = normalize(all); - all.releases = all.releases.slice(0, 5); - return all; -}; - -if (module === require.main) { - (async function () { - let samples = await Releases.sample(); - console.info(JSON.stringify(samples, null, 2)); - })(); -} +Installer path (bash/ps1 script output): + Request → serve-installer.js → builds.js → builds-cacher.js + Vocabulary: darwin, x86_64, aarch64 (build-classifier vocabulary) ``` -### Common release transformations +Cache is generated by the Go daemon (`webicached`) and stored flat in +`~/.cache/webi/legacy/{pkg}.json` (resolved via `Os.homedir()` in the Node +readers — no `_cache` symlink, no month subdirectory). Each file contains: +- Top-level summary arrays: `oses`, `arches`, `libcs`, `formats` +- `releases` array with pre-classified fields: `os`, `arch`, `libc`, `ext`, + `version`, `channel`, `download` +- `download` template string -**Strip version prefix** (monorepo or tool-prefixed tags): +### Canonical vocabulary (cache and API) -```js -// e.g. "tools/monorel/v0.6.5" → "v0.6.5" -rel.version = rel.version.replace(/^tools\/monorel\//, ''); +**OS**: `macos` (not darwin), `linux`, `windows`, `freebsd`, etc. +**Arch**: `amd64` (not x86_64), `arm64` (not aarch64), `armv7l` (not armv7) +**Libc**: `none` (never empty), `gnu`, `musl`, `msvc` +**Ext**: `tar.gz`, `zip`, `exe` (no leading dot; `exe` for bare binaries) -// e.g. "cli-v1.2.3" → "v1.2.3" -rel.version = rel.version.replace(/^cli-/, ''); -``` +## releases.js (legacy — being phased out) -**Filter releases** (monorepo with multiple tools, or unwanted assets): +The `{pkg}/releases.js` files previously fetched upstream release metadata. +These are being replaced by the Go cache daemon. Existing files are kept as +documentation of release sources but are no longer called by the server. -```js -all.releases = all.releases.filter(function (rel) { - // Keep only releases for this tool - return rel.version.startsWith('tools/monorel/'); -}); -``` - -Apply transformations inside `Releases.latest`, before returning `all`. - -**Available sources** beyond `github.js`: - -- `_common/gitea.js` — Gitea servers -- `_common/git-tag.js` — Git tag listing -- Custom fetch from any JSON API (see `go/releases.js`, `terraform/releases.js`) - -### Testing releases.js +### Testing the API (current) ```sh -node -e " - let Releases = require('.//releases.js'); - Releases.sample().then(function (all) { - console.log(JSON.stringify(all, null, 2)); - }); -" +curl -sS 'https://beta.webi.sh/api/releases/.json?os=macos&arch=arm64&limit=5' | jq . ``` -Verify: versions are clean semver (`0.6.5` not `tools/monorel/v0.6.5`), OS/arch -detected correctly, download URLs resolve. +Verify: versions present, correct OS/arch vocabulary, download URLs resolve. ## install.sh @@ -365,9 +354,11 @@ Commit messages: `feat(): add installer`, `fix(): update install.sh`, must filter in `releases.js` and strip the tag prefix from the version. - **No `--version` flag**: Some tools lack version introspection. Comment out `pkg_get_current_version` — webi still works, it just can't skip reinstalls. -- **normalize.js auto-detection**: OS/arch/ext are guessed from download - filenames. If the tool uses non-standard naming, you may need to set `os`, - `arch`, or `ext` explicitly in `releases.js`. +- **Cache directory**: `builds-cacher.js` and `transform-releases.js` read + exclusively from `~/.cache/webi/legacy/{pkg}.json` (resolved via + `Os.homedir()`). No date bucketing, no `_cache` symlink, no month rollover + hazard. If a package returns `0.0.0`, check that `~/.cache/webi/legacy/{pkg}.json` + exists and that `webicached` is running. - **Goreleaser archives**: Typically contain a bare binary at the archive root (not nested in a directory). Use `mv ./cmd "$pkg_src_cmd"`. diff --git a/_scripts/deploy-webinstall b/_scripts/deploy-webinstall new file mode 100755 index 0000000..29238a1 --- /dev/null +++ b/_scripts/deploy-webinstall @@ -0,0 +1,100 @@ +#!/bin/sh +set -e +set -u + +# Deploy the Node.js webinstall (installer server) to a target host. +# Usage: _scripts/deploy-webinstall +# e.g. _scripts/deploy-webinstall beta.webi.sh +# _scripts/deploy-webinstall next.webi.sh + +fn_main() { + g_host="${1:-}" + if test -z "${g_host}"; then + printf 'Usage: %s \n' "$0" >&2 + printf ' e.g. %s beta.webi.sh\n' "$0" >&2 + exit 1 + fi + + # beta.webi.sh → ~/srv/beta.webinstall.dev/installers/ + # next.webi.sh → ~/srv/next.webinstall.dev/installers/ + b_subdomain="${g_host%%.*}" + g_dest="~/srv/${b_subdomain}.webinstall.dev/installers/" + + # Verify the build-classifier submodule is populated. rsync will + # happily push an empty submodule directory, after which Node + # crashes at startup with `Cannot find module './build-classifier/ + # host-targets.js'`. New worktrees from `git worktree add` don't + # init submodules by default. + if ! test -f ./_webi/build-classifier/host-targets.js; then + printf 'fatal: _webi/build-classifier submodule not initialized\n' >&2 + printf ' run: git submodule update --init _webi/build-classifier\n' >&2 + exit 1 + fi + + printf '%s\n' "Deploying to ${g_host}:${g_dest} ..." + + rsync -avz --delete \ + --exclude='.git' \ + --exclude='node_modules' \ + --exclude='_cache' \ + --exclude='LIVE_cache' \ + --exclude='webicached' \ + --exclude='classify' \ + --exclude='fetchraw' \ + --exclude='inspect' \ + --exclude='e2etest' \ + --exclude='zigtest' \ + --exclude='distributables.csv' \ + --exclude='agents/' \ + ./ "${g_host}:${g_dest}" + + printf '%s\n' "Restarting webinstall ..." + + # shellcheck disable=SC2029 + ssh "${g_host}" " + . ~/.config/envman/PATH.env 2>/dev/null + rm -rf ${g_dest}_cache 2>/dev/null + serviceman restart webinstall + " + + printf '%s\n' "Waiting for restart ..." + sleep 3 + + fn_smoke_test +} + +fn_smoke_test() { + printf '%s\n' "Smoke testing https://${g_host}/ ..." + + b_headers="$(curl -sI --max-time 5 "https://${g_host}/api/releases/go.json?limit=1")" + + if printf '%s' "${b_headers}" | grep -qi 'X-Robots-Tag: noindex'; then + printf '%s\n' " [ok] X-Robots-Tag: noindex" + else + printf '%s\n' " [FAIL] Missing X-Robots-Tag header" + fi + + if printf '%s' "${b_headers}" | grep -qi 'rel="canonical"'; then + printf '%s\n' " [ok] Link: canonical" + else + printf '%s\n' " [FAIL] Missing canonical Link header" + fi + + b_version="$(curl -sS --max-time 5 "https://${g_host}/api/releases/go.json?limit=1" | jq -r '.[0].version')" + if test "${b_version}" != "0.0.0" && test -n "${b_version}"; then + printf '%s\n' " [ok] API returns go ${b_version}" + else + printf '%s\n' " [FAIL] API returned error or empty for go" + fi + + b_shebang="$(curl -sS --max-time 5 "https://${g_host}/node" | head -1)" + if test "${b_shebang}" = "#!/bin/sh"; then + printf '%s\n' " [ok] Installer path serves shell script" + else + printf '%s\n' " [FAIL] Installer path unexpected: ${b_shebang}" + fi + + printf '%s\n' "Done." +} + +fn_main "$@"