build: add deploy-webinstall script and document cache-only architecture

- _scripts/deploy-webinstall: rsync-based deploy to beta.webi.sh and next.webi.sh that excludes _cache, restarts the webinstall service via serviceman (sourcing ~/.config/envman/PATH.env so serviceman is on PATH for non-interactive ssh). Uses an end-of-line anchored process match so only the node worker is touched, never its supervisor. - AGENTS.md: document the cache-only Node server (two paths, canonical os/arch/libc/ext vocabulary), add a domains table for prod/beta/next, remove stale normalize.js references. - .gitignore: ignore agent session files (LOCAL.md, agents/, etc) and local test fixtures (testdata/).
2026-08-19 03:56:16 +00:00 · 2026-05-16 21:48:37 -06:00
parent 73188d50e1
commit 785dc05324
3 changed files with 166 additions and 69 deletions
@@ -23,6 +23,11 @@ install-*.ps1
 _cache/
 node_modules/

+# local test fixtures (regenerated by _webi/test-live-*.js)
+testdata/
+LIVE_cache/
+distributables.csv
+
 # temporary & backup files
 .*.sw*
 *.bak
@@ -30,6 +35,7 @@ node_modules/

 # agent session files
 agents/
+LOCAL.md

 # other
 .DS_Store
@@ -3,6 +3,26 @@
 Webi installs dev tools to `~/.local/` without sudo. Each installer is a small
 package of 3-4 files. This guide tells you how to create and modify them.

+## Domains
+
+| Environment | Domains                                          |
+| ----------- | ------------------------------------------------ |
+| Production  | webi.sh, webi.ms, webinstall.dev                 |
+| Beta        | beta.webi.sh, beta.webi.ms, beta.webinstall.dev  |
+| Next        | next.webi.sh, next.webi.ms, next.webinstall.dev  |
+
+- **webi.sh** — POSIX shell installer: `curl https://webi.sh/node | sh`
+- **webi.ms** — PowerShell installer: `curl.exe https://webi.ms/node | powershell`
+- **webinstall.dev** — canonical domain, serves both shell scripts and the
+  browser-facing cheat sheet pages
+
+The domain controls the default script format. You can force the same behavior
+on any domain via User-Agent:
+- `curl -A "MS" ...` — triggers PowerShell output (same as webi.ms)
+- `curl ...` (default UA) — triggers POSIX shell output (same as webi.sh)
+- `curl -A "$(uname -srm)" ...` — once the API is activated, the full
+  `uname -srm` string (e.g. `Linux 6.1.0 x86_64`) guides OS/arch selection
+
 ## Why Webi Exists

 Webi makes tool installation trivially repeatable for people who aren't
@@ -37,9 +57,11 @@ about PATH, permissions, or platform differences. Three things matter:

 Key infrastructure directories (do not modify without good reason):

- `_webi/` — bootstrap templates, `normalize.js` (auto-detects OS/arch/ext from
-  filenames)
- `_common/` — shared JS: `github.js`, `githubish.js`, `gitea.js`, `fetcher.js`
+- `_webi/` — bootstrap templates, `transform-releases.js` (API endpoint),
+  `builds-cacher.js` (reads cache JSON), `serve-installer.js` (installer
+  scripts)
+- `_common/` — shared JS fetcher libraries (being phased out — Go daemon now
+  fetches upstream)
 - `_example/` — canonical template for new packages
 - `_examples/` — specialized templates (goreleaser, xz-compressed)

@@ -57,83 +79,50 @@ Ref: <https://github.com/webinstall/webi-installers/issues/412>
 | 🔗    | Alias/redirect to another package      | `ripgrep` → `rg` |
 | 📝    | Bespoke / custom install               | `rustlang`       |

-## releases.js
+## Data Architecture

-Fetches release metadata and returns a normalized object. Most packages use
-GitHub releases:
+There are two data paths. Both read from pre-generated cache — the Node.js
+server does NOT fetch upstream APIs.

-```js
-'use strict';
+```
+API path (JSON/TAB output):
+  Request → transform-releases.js → ~/.cache/webi/legacy/{pkg}.json → filter + sort
+  Vocabulary: macos, amd64, arm64, armv7l (API vocabulary)
+  normalize.js: REMOVED — cache provides all fields directly

-var github = require('../_common/github.js');
-var owner = 'OWNER';
-var repo = 'REPO';
-
-let Releases = module.exports;
-
-Releases.latest = async function () {
-  let all = await github(null, owner, repo);
-  return all;
-};
-
-Releases.sample = async function () {
-  let normalize = require('../_webi/normalize.js');
-  let all = await Releases.latest();
-  all = normalize(all);
-  all.releases = all.releases.slice(0, 5);
-  return all;
-};
-
-if (module === require.main) {
-  (async function () {
-    let samples = await Releases.sample();
-    console.info(JSON.stringify(samples, null, 2));
-  })();
-}
+Installer path (bash/ps1 script output):
+  Request → serve-installer.js → builds.js → builds-cacher.js
+  Vocabulary: darwin, x86_64, aarch64 (build-classifier vocabulary)
 ```

-### Common release transformations
+Cache is generated by the Go daemon (`webicached`) and stored flat in
+`~/.cache/webi/legacy/{pkg}.json` (resolved via `Os.homedir()` in the Node
+readers — no `_cache` symlink, no month subdirectory). Each file contains:
+- Top-level summary arrays: `oses`, `arches`, `libcs`, `formats`
+- `releases` array with pre-classified fields: `os`, `arch`, `libc`, `ext`,
+  `version`, `channel`, `download`
+- `download` template string

-**Strip version prefix** (monorepo or tool-prefixed tags):
+### Canonical vocabulary (cache and API)

-```js
-// e.g. "tools/monorel/v0.6.5" → "v0.6.5"
-rel.version = rel.version.replace(/^tools\/monorel\//, '');
+**OS**: `macos` (not darwin), `linux`, `windows`, `freebsd`, etc.
+**Arch**: `amd64` (not x86_64), `arm64` (not aarch64), `armv7l` (not armv7)
+**Libc**: `none` (never empty), `gnu`, `musl`, `msvc`
+**Ext**: `tar.gz`, `zip`, `exe` (no leading dot; `exe` for bare binaries)

-// e.g. "cli-v1.2.3" → "v1.2.3"
-rel.version = rel.version.replace(/^cli-/, '');
-```
+## releases.js (legacy — being phased out)

-**Filter releases** (monorepo with multiple tools, or unwanted assets):
+The `{pkg}/releases.js` files previously fetched upstream release metadata.
+These are being replaced by the Go cache daemon. Existing files are kept as
+documentation of release sources but are no longer called by the server.

-```js
-all.releases = all.releases.filter(function (rel) {
-  // Keep only releases for this tool
-  return rel.version.startsWith('tools/monorel/');
-});
-```
-
-Apply transformations inside `Releases.latest`, before returning `all`.
-
-**Available sources** beyond `github.js`:
-
- `_common/gitea.js` — Gitea servers
- `_common/git-tag.js` — Git tag listing
- Custom fetch from any JSON API (see `go/releases.js`, `terraform/releases.js`)
-
-### Testing releases.js
+### Testing the API (current)

 ```sh
-node -e "
-  let Releases = require('./<name>/releases.js');
-  Releases.sample().then(function (all) {
-    console.log(JSON.stringify(all, null, 2));
-  });
-"
+curl -sS 'https://beta.webi.sh/api/releases/<name>.json?os=macos&arch=arm64&limit=5' | jq .
 ```

-Verify: versions are clean semver (`0.6.5` not `tools/monorel/v0.6.5`), OS/arch
-detected correctly, download URLs resolve.
+Verify: versions present, correct OS/arch vocabulary, download URLs resolve.

 ## install.sh

@@ -365,9 +354,11 @@ Commit messages: `feat(<pkg>): add installer`, `fix(<pkg>): update install.sh`,
  must filter in `releases.js` and strip the tag prefix from the version.
 - **No `--version` flag**: Some tools lack version introspection. Comment out
  `pkg_get_current_version` — webi still works, it just can't skip reinstalls.
- **normalize.js auto-detection**: OS/arch/ext are guessed from download
-  filenames. If the tool uses non-standard naming, you may need to set `os`,
-  `arch`, or `ext` explicitly in `releases.js`.
+- **Cache directory**: `builds-cacher.js` and `transform-releases.js` read
+  exclusively from `~/.cache/webi/legacy/{pkg}.json` (resolved via
+  `Os.homedir()`). No date bucketing, no `_cache` symlink, no month rollover
+  hazard. If a package returns `0.0.0`, check that `~/.cache/webi/legacy/{pkg}.json`
+  exists and that `webicached` is running.
 - **Goreleaser archives**: Typically contain a bare binary at the archive root
  (not nested in a directory). Use `mv ./cmd "$pkg_src_cmd"`.

@@ -0,0 +1,100 @@
+#!/bin/sh
+set -e
+set -u
+
+# Deploy the Node.js webinstall (installer server) to a target host.
+# Usage: _scripts/deploy-webinstall <host>
+#   e.g. _scripts/deploy-webinstall beta.webi.sh
+#        _scripts/deploy-webinstall next.webi.sh
+
+fn_main() {
+    g_host="${1:-}"
+    if test -z "${g_host}"; then
+        printf 'Usage: %s <host>\n' "$0" >&2
+        printf '  e.g. %s beta.webi.sh\n' "$0" >&2
+        exit 1
+    fi
+
+    # beta.webi.sh → ~/srv/beta.webinstall.dev/installers/
+    # next.webi.sh → ~/srv/next.webinstall.dev/installers/
+    b_subdomain="${g_host%%.*}"
+    g_dest="~/srv/${b_subdomain}.webinstall.dev/installers/"
+
+    # Verify the build-classifier submodule is populated. rsync will
+    # happily push an empty submodule directory, after which Node
+    # crashes at startup with `Cannot find module './build-classifier/
+    # host-targets.js'`. New worktrees from `git worktree add` don't
+    # init submodules by default.
+    if ! test -f ./_webi/build-classifier/host-targets.js; then
+        printf 'fatal: _webi/build-classifier submodule not initialized\n' >&2
+        printf '  run: git submodule update --init _webi/build-classifier\n' >&2
+        exit 1
+    fi
+
+    printf '%s\n' "Deploying to ${g_host}:${g_dest} ..."
+
+    rsync -avz --delete \
+        --exclude='.git' \
+        --exclude='node_modules' \
+        --exclude='_cache' \
+        --exclude='LIVE_cache' \
+        --exclude='webicached' \
+        --exclude='classify' \
+        --exclude='fetchraw' \
+        --exclude='inspect' \
+        --exclude='e2etest' \
+        --exclude='zigtest' \
+        --exclude='distributables.csv' \
+        --exclude='agents/' \
+        ./ "${g_host}:${g_dest}"
+
+    printf '%s\n' "Restarting webinstall ..."
+
+    # shellcheck disable=SC2029
+    ssh "${g_host}" "
+        . ~/.config/envman/PATH.env 2>/dev/null
+        rm -rf ${g_dest}_cache 2>/dev/null
+        serviceman restart webinstall
+    "
+
+    printf '%s\n' "Waiting for restart ..."
+    sleep 3
+
+    fn_smoke_test
+}
+
+fn_smoke_test() {
+    printf '%s\n' "Smoke testing https://${g_host}/ ..."
+
+    b_headers="$(curl -sI --max-time 5 "https://${g_host}/api/releases/go.json?limit=1")"
+
+    if printf '%s' "${b_headers}" | grep -qi 'X-Robots-Tag: noindex'; then
+        printf '%s\n' "  [ok] X-Robots-Tag: noindex"
+    else
+        printf '%s\n' "  [FAIL] Missing X-Robots-Tag header"
+    fi
+
+    if printf '%s' "${b_headers}" | grep -qi 'rel="canonical"'; then
+        printf '%s\n' "  [ok] Link: canonical"
+    else
+        printf '%s\n' "  [FAIL] Missing canonical Link header"
+    fi
+
+    b_version="$(curl -sS --max-time 5 "https://${g_host}/api/releases/go.json?limit=1" | jq -r '.[0].version')"
+    if test "${b_version}" != "0.0.0" && test -n "${b_version}"; then
+        printf '%s\n' "  [ok] API returns go ${b_version}"
+    else
+        printf '%s\n' "  [FAIL] API returned error or empty for go"
+    fi
+
+    b_shebang="$(curl -sS --max-time 5 "https://${g_host}/node" | head -1)"
+    if test "${b_shebang}" = "#!/bin/sh"; then
+        printf '%s\n' "  [ok] Installer path serves shell script"
+    else
+        printf '%s\n' "  [FAIL] Installer path unexpected: ${b_shebang}"
+    fi
+
+    printf '%s\n' "Done."
+}
+
+fn_main "$@"