fix(go): use early returns in pkg_link for clarity

Co-authored-by: coolaj86 <122831+coolaj86@users.noreply.github.com>

.github/ISSUE_TEMPLATE/new_installer.md

@@ -35,15 +35,18 @@ info, and doing a find and replace on a few file system path names.
    ```
 2. Copy the example template and update with info from Official Releases:
    <https://github.com/___CHANGE/ME___/releases>
+
    ```bash
    rsync -av _example/ CHANGE-ME/
    ```
+
    - [ ] update `CHANGE-ME/release.js` to use the official repo
    - [ ] Learn how `CHANGE-ME` unpacks (i.e. as a single file? as a .tar.gz? as
          a .tar.gz with a folder named CHANGE-ME?)
    - [ ] find and replace to change the name
      - [ ] update `CHANGE-ME/install.sh` (see `bat` and `jq` as examples)
      - [ ] update `CHANGE-ME/install.ps1` (see `bat` and `jq` as examples)
+
 3. Needs an updated tagline and cheat sheet
    - [ ] update `CHANGE-ME/README.md`
      - [ ] official URL

.gitignore

@@ -1,7 +1,24 @@
-.env
-node_modules
+# generated artifacts
 install-*.sh
 install-*.bat
 install-*.ps1
+
+# local config
+.env.*
+*.env
+.env
+!example.env
+
+# caches
+_cache/
+node_modules/
+
+# temporary & backup files
 .*.sw*
 *.bak
+*.bak.*
+
+# other
+.DS_Store
+desktop.ini
+.directory

.jshintrc

@@ -1,4 +1,7 @@
 {
+  "globals": {
+    "AbortController": false
+  },
   "browser": true,
   "node": true,
   "esversion": 11,

AGENTS.md

@@ -0,0 +1,372 @@
+# Webi Installers — Agent Guide
+
+Webi installs dev tools to `~/.local/` without sudo. Each installer is a small
+package of 3-4 files. This guide tells you how to create and modify them.
+
+## Why Webi Exists
+
+Webi makes tool installation trivially repeatable for people who aren't
+sysadmins — freelance clients, junior devs, anyone who shouldn't have to care
+about PATH, permissions, or platform differences. Three things matter:
+
+1. **Install without friction.** No sudo, no manual PATH edits, no "necessary
+   but unimportant" steps leaking into the experience.
+2. **Know where things are.** The Files section tells you exactly what got
+   created or modified. Nothing should be mysterious.
+3. **Copy-paste recipes.** The cheat sheet is what you'd send someone less
+   experienced than yourself instead of a project's full README — scannable,
+   concrete, easy to reference by name.
+
+## Quick Start: Adding a New Installer
+
+1. Identify the **package type** (see [Categories](#categories) below)
+2. Find an existing installer of the same type to use as a template
+3. Create `<name>/releases.js`, `install.sh`, `install.ps1`, `README.md`
+4. Test with the command in [Testing releases.js](#testing-releasesjs)
+5. Run formatters before committing (see [Code Style](#code-style))
+
+## Directory Layout
+
+```
+<package-name>/
+  README.md          # YAML frontmatter + docs
+  releases.js        # Fetches release metadata (Node.js)
+  install.sh         # POSIX shell installer (macOS/Linux)
+  install.ps1        # PowerShell installer (Windows) — optional
+```
+
+Key infrastructure directories (do not modify without good reason):
+
+- `_webi/` — bootstrap templates, `normalize.js` (auto-detects OS/arch/ext from
+  filenames)
+- `_common/` — shared JS: `github.js`, `githubish.js`, `gitea.js`, `fetcher.js`
+- `_example/` — canonical template for new packages
+- `_examples/` — specialized templates (goreleaser, xz-compressed)
+
+## Categories
+
+Ref: <https://github.com/webinstall/webi-installers/issues/412>
+
+| Type  | Description                            | Template to copy |
+| ----- | -------------------------------------- | ---------------- |
+| `bin` | Single binary in tar/zip               | `koji`, `delta`  |
+| `bin` | Single bare binary (no archive)        | `arc`, `shfmt`   |
+| `bin` | Goreleaser-style archives              | `keypairs`       |
+| 📦    | Self-contained package (bin/man/share) | `node`, `go`     |
+| 📂    | Multiple binaries/scripts              | `pg`             |
+| 🔗    | Alias/redirect to another package      | `ripgrep` → `rg` |
+| 📝    | Bespoke / custom install               | `rustlang`       |
+
+## releases.js
+
+Fetches release metadata and returns a normalized object. Most packages use
+GitHub releases:
+
+```js
+'use strict';
+
+var github = require('../_common/github.js');
+var owner = 'OWNER';
+var repo = 'REPO';
+
+let Releases = module.exports;
+
+Releases.latest = async function () {
+  let all = await github(null, owner, repo);
+  return all;
+};
+
+Releases.sample = async function () {
+  let normalize = require('../_webi/normalize.js');
+  let all = await Releases.latest();
+  all = normalize(all);
+  all.releases = all.releases.slice(0, 5);
+  return all;
+};
+
+if (module === require.main) {
+  (async function () {
+    let samples = await Releases.sample();
+    console.info(JSON.stringify(samples, null, 2));
+  })();
+}
+```
+
+### Common release transformations
+
+**Strip version prefix** (monorepo or tool-prefixed tags):
+
+```js
+// e.g. "tools/monorel/v0.6.5" → "v0.6.5"
+rel.version = rel.version.replace(/^tools\/monorel\//, '');
+
+// e.g. "cli-v1.2.3" → "v1.2.3"
+rel.version = rel.version.replace(/^cli-/, '');
+```
+
+**Filter releases** (monorepo with multiple tools, or unwanted assets):
+
+```js
+all.releases = all.releases.filter(function (rel) {
+  // Keep only releases for this tool
+  return rel.version.startsWith('tools/monorel/');
+});
+```
+
+Apply transformations inside `Releases.latest`, before returning `all`.
+
+**Available sources** beyond `github.js`:
+
+- `_common/gitea.js` — Gitea servers
+- `_common/git-tag.js` — Git tag listing
+- Custom fetch from any JSON API (see `go/releases.js`, `terraform/releases.js`)
+
+### Testing releases.js
+
+```sh
+node -e "
+  let Releases = require('./<name>/releases.js');
+  Releases.sample().then(function (all) {
+    console.log(JSON.stringify(all, null, 2));
+  });
+"
+```
+
+Verify: versions are clean semver (`0.6.5` not `tools/monorel/v0.6.5`), OS/arch
+detected correctly, download URLs resolve.
+
+## install.sh
+
+POSIX shell (`sh`, not bash). Always wrapped in a function:
+
+```sh
+#!/bin/sh
+# shellcheck disable=SC2034
+
+set -e
+set -u
+
+__init_pkgname() {
+    # These 6 variables are required
+    pkg_cmd_name="cmd"
+
+    pkg_dst_cmd="$HOME/.local/bin/cmd"
+    pkg_dst="$pkg_dst_cmd"
+
+    pkg_src_cmd="$HOME/.local/opt/cmd-v$WEBI_VERSION/bin/cmd"
+    pkg_src_dir="$HOME/.local/opt/cmd-v$WEBI_VERSION"
+    pkg_src="$pkg_src_cmd"
+
+    pkg_install() {
+        mkdir -p "$(dirname "$pkg_src_cmd")"
+        mv ./cmd "$pkg_src_cmd"
+    }
+
+    pkg_get_current_version() {
+        cmd --version 2> /dev/null | head -n 1 | cut -d' ' -f2
+    }
+}
+
+__init_pkgname
+```
+
+### Framework variables available in install.sh
+
+Set by the webi bootstrap (`_webi/package-install.tpl.sh`):
+
+| Variable        | Example             | Description           |
+| --------------- | ------------------- | --------------------- |
+| `WEBI_VERSION`  | `1.2.3`             | Selected version      |
+| `WEBI_PKG_URL`  | `https://...`       | Download URL          |
+| `WEBI_PKG_FILE` | `foo-v1.2.3.tar.gz` | Download filename     |
+| `WEBI_OS`       | `linux`             | Detected OS           |
+| `WEBI_ARCH`     | `amd64`             | Detected architecture |
+| `WEBI_EXT`      | `tar.gz`            | Archive extension     |
+| `WEBI_CHANNEL`  | `stable`            | Release channel       |
+| `PKG_NAME`      | `foo`               | Package name          |
+
+### Override functions
+
+| Function                    | Purpose                                       |
+| --------------------------- | --------------------------------------------- |
+| `pkg_install()`             | **Required.** Move files to `$pkg_src`        |
+| `pkg_get_current_version()` | Parse installed version from command output   |
+| `pkg_post_install()`        | Post-install setup (git config, shell config) |
+| `pkg_done_message()`        | Custom completion message                     |
+| `pkg_link()`                | Override default symlink behavior             |
+| `pkg_pre_install()`         | Custom pre-install logic                      |
+
+### Framework helper functions
+
+| Function                 | Purpose                            |
+| ------------------------ | ---------------------------------- |
+| `webi_download()`        | Download package if not cached     |
+| `webi_extract()`         | Extract archive by extension       |
+| `webi_path_add <dir>`    | Add to PATH via envman             |
+| `webi_link()`            | Create versioned symlinks          |
+| `webi_check_installed()` | Check if version already installed |
+
+### pkg_install patterns
+
+**Bare binary in archive root:**
+
+```sh
+mv ./cmd "$pkg_src_cmd"
+```
+
+**Binary in a subdirectory (goreleaser-style `cmd-OS-arch/cmd`):**
+
+```sh
+mv ./cmd-*/cmd "$pkg_src_cmd"
+```
+
+**Flexible detection (handles multiple archive layouts):**
+
+```sh
+if test -f ./cmd; then
+    mv ./cmd "$pkg_src_cmd"
+elif test -e ./cmd-*/cmd; then
+    mv ./cmd-*/cmd "$pkg_src_cmd"
+elif test -e ./cmd-*; then
+    mv ./cmd-* "$pkg_src_cmd"
+fi
+```
+
+## install.ps1
+
+PowerShell for Windows. Uses `$Env:` variables. See `_example/install.ps1` for
+the full template. Key differences from install.sh:
+
+- Paths use backslashes, commands end in `.exe`
+- `$Env:USERPROFILE` instead of `$HOME`
+- `Test-Path`, `Move-Item`, `Copy-Item` instead of shell equivalents
+- Downloads go to `$Env:USERPROFILE\Downloads\webi\`
+- Temp work in `.local\tmp`, use `Push-Location`/`Pop-Location`
+- Symlinks done via `Copy-Item` (not actual symlinks)
+
+## README.md
+
+````markdown
+---
+title: toolname
+homepage: https://github.com/owner/repo
+tagline: |
+  toolname: A short one-line description.
+---
+
+To update or switch versions, run `webi toolname@stable` (or `@v2`, `@beta`,
+etc).
+
+### Files
+
+These are the files that are created and/or modified with this installer:
+
+```text
+~/.config/envman/PATH.env
+~/.local/bin/toolname
+~/.local/opt/toolname-VERSION/bin/toolname
+```
+
+## Cheat Sheet
+
+> `toolname` does X. Brief description.
+
+### How to use toolname
+
+```sh
+toolname --example
+```
+````
+
+Note: **Files goes above Cheat Sheet**, not inside it.
+
+### Cheat Sheet tone and style
+
+Webi cheat sheets are **opinionated quick-reference guides**, not comprehensive
+documentation. Think "colleague's sticky note" — not the project's official
+README.
+
+The tool is the topic, but **the problem is the reason**. Cheat sheets are
+organized around tasks the reader already wants to do — the tool is how they get
+there. Headings reference the tool (the reader came to this page on purpose),
+but the content solves the underlying problem completely:
+
+- "How to reverse proxy to Node" (caddy knowledge, not just node)
+- "How to run a Node app as a System Service" (serviceman knowledge)
+- "How to Enable Secure Remote Postgres Access" (openssl, pg_hba.conf, systemd)
+- "How to manually configure git to use delta" (gitconfig, not delta flags)
+- "How to make fish the default shell in iTerm2" (iTerm2 knowledge, not fish)
+
+The reader's question is "how do I do X?" and the cheat sheet answers it
+completely — including configs, adjacent tools, and platform-specific
+variations. A goreleaser cheat sheet teaches you goreleaser YAML. A postgres
+cheat sheet teaches you pg_hba.conf, openssl certs, and systemd units.
+
+Cheat sheets cross tool boundaries freely. Node's references caddy, serviceman,
+setcap-netbind, GitHub Actions. Postgres references serviceman, openrc, launchd.
+They link to each other's webi pages. The scope is "everything you need to
+accomplish this task," not "everything this one binary does."
+
+They show the actual files and configs that matter — not documentation _about_
+configs, but the configs themselves, copy-pasteable, with inline comments
+explaining the non-obvious parts.
+
+**Guidelines:**
+
+- **Show the 3-5 things someone will actually do**, with copy-pasteable
+  commands. Skip exhaustive flag lists and API docs.
+- **Lead with practical integration.** Show the exact `git config` lines, the
+  exact hook script, the exact shell alias — don't just explain the feature and
+  leave wiring up to the reader.
+- **Skip what they already know.** No need to re-explain what the tool is at
+  length — the tagline and one-liner blockquote handle that. Get to the
+  commands.
+- **Prefer concrete over abstract.** Instead of "you can configure X via a
+  config file", show the config file contents.
+
+## Shell Naming Conventions
+
+**Variables:**
+
+- `ALL_CAPS` — environment variables only (`PATH`, `HOME`, `WEBI_VERSION`)
+- `b_varname` — block-scoped (inside a function, loop, or conditional)
+- `g_varname` — global to the script (and sourced scripts)
+- `a_varname` — function arguments
+
+**Functions and commands:**
+
+- `fn_name` — helper functions (anything other than the script's main/entry
+  function)
+- `cmd_name` — command aliases, e.g. `cmd_curl='curl --fail-with-body -sSL'`
+
+## Code Style
+
+Requires `node`, `shfmt`, `pwsh`, and `pwsh-essentials` (install all via webi).
+Run before committing:
+
+```sh
+npm run fmt         # prettier (JS/MD) + shfmt (sh) + pwsh-fmt (ps1)
+npm run lint        # jshint + shellcheck + pwsh-lint
+```
+
+Commit messages: `feat(<pkg>): add installer`, `fix(<pkg>): update install.sh`,
+`docs(<pkg>): add cheat sheet`.
+
+## Naming Conventions
+
+- The canonical package name is the **command name** you type: `go`, `node`,
+  `rg`
+- The alternate/alias name is the project name: `golang`, `nodejs`, `ripgrep`
+- Package directories are lowercase with hyphens
+
+## Common Pitfalls
+
+- **Monorepo releases**: The GitHub API returns ALL releases for the repo. You
+  must filter in `releases.js` and strip the tag prefix from the version.
+- **No `--version` flag**: Some tools lack version introspection. Comment out
+  `pkg_get_current_version` — webi still works, it just can't skip reinstalls.
+- **normalize.js auto-detection**: OS/arch/ext are guessed from download
+  filenames. If the tool uses non-standard naming, you may need to set `os`,
+  `arch`, or `ext` explicitly in `releases.js`.
+- **Goreleaser archives**: Typically contain a bare binary at the archive root
+  (not nested in a directory). Use `mv ./cmd "$pkg_src_cmd"`.

CONTRIBUTING.md

@@ -10,7 +10,6 @@
 
 - You'll be asked to make changes if you don't run the code formatters and
   linters:
-
   - Node / JavaScript:
     - [prettier](https://webinstall.dev/prettier)
       ```sh
@@ -21,7 +20,6 @@
       npm run lint
       ```
   - Bash
-
     - [shfmt](https://webinstall.dev/shfmt)
       ```sh
       npm run shfmt

README.md

@@ -98,9 +98,10 @@ You just fill in the blanks.
 Just create an empty directory and run the tests until you get a good result.
 
 ```sh
-git clone git@github.com:webinstall/packages.git
-pushd packages
-npm install
+git clone git@github.com:webinstall/webi-installers.git
+pushd ./webi-installers/
+git submodule update --init
+npm clean-install
 ```
 
 ```sh
@@ -144,8 +145,8 @@ It looks like this:
 `releases.js`:
 
 ```js
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     // if you need to do something special, you can do it here
     // ...
     return all;

_common/brew.js

@@ -1,64 +1,62 @@
 'use strict';
 
+let Fetcher = require('../_common/fetcher.js');
+
 /**
- * Gets a releases from 'brew'.
+ * Gets releases from 'brew'.
  *
- * @param request
+ * @param {null} _
  * @param {string} formula
- * @returns {PromiseLike<any> | Promise<any>}
+ * @returns {Promise<any>}
  */
-function getAllReleases(request, formula) {
+async function getDistributables(_, formula) {
   if (!formula) {
     return Promise.reject('missing formula for brew');
   }
-  return request({
-    url: 'https://formulae.brew.sh/api/formula/' + formula + '.json',
-    fail: true, // https://git.coolaj86.com/coolaj86/request.js/issues/2
-    json: true,
-  })
-    .then(failOnBadStatus)
-    .then(function (resp) {
-      var ver = resp.body.versions.stable;
-      var dl = (
-        resp.body.bottle.stable.files.high_sierra ||
-        resp.body.bottle.stable.files.catalina
-      ).url.replace(new RegExp(ver.replace(/\./g, '\\.'), 'g'), '{{ v }}');
-      return [
-        {
-          version: ver,
-          download: dl.replace(/{{ v }}/g, ver),
-        },
-      ].concat(
-        resp.body.versioned_formulae.map(function (f) {
-          var ver = f.replace(/.*@/, '');
-          return {
-            version: ver,
-            download: dl,
-          };
-        }),
-      );
-    })
-    .catch(function (err) {
-      console.error('Error fetching MariaDB versions (brew)');
-      console.error(err);
-      return [];
+
+  let resp;
+  try {
+    let url = `https://formulae.brew.sh/api/formula/${formula}.json`;
+    resp = await Fetcher.fetch(url, {
+      headers: { Accept: 'application/json' },
     });
-}
-
-function failOnBadStatus(resp) {
-  if (resp.statusCode >= 400) {
-    var err = new Error('Non-successful status code: ' + resp.statusCode);
-    err.code = 'ESTATUS';
-    err.response = resp;
-    throw err;
+  } catch (e) {
+    /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error
+    let err = e;
+    if (err.code === 'E_FETCH_RELEASES') {
+      err.message = `failed to fetch '${formula}' release data from 'brew': ${err.response.status} ${err.response.body}`;
+    }
+    throw e;
   }
-  return resp;
+  let body = JSON.parse(resp.body);
+
+  var ver = body.versions.stable;
+  var dl = (
+    body.bottle.stable.files.high_sierra || body.bottle.stable.files.catalina
+  ).url.replace(new RegExp(ver.replace(/\./g, '\\.'), 'g'), '{{ v }}');
+  return [
+    {
+      version: ver,
+      download: dl.replace(/{{ v }}/g, ver),
+    },
+  ].concat(
+    body.versioned_formulae.map(
+      /** @param {String} f */
+      function (f) {
+        var ver = f.replace(/.*@/, '');
+        return {
+          version: ver,
+          download: dl,
+        };
+      },
+    ),
+  );
 }
 
-module.exports = getAllReleases;
+module.exports = getDistributables;
 
 if (module === require.main) {
-  getAllReleases(require('@root/request'), 'mariadb').then(function (all) {
+  getDistributables(null, 'mariadb').then(function (all) {
     console.info(JSON.stringify(all, null, 2));
   });
 }

_common/fetcher.js

@@ -0,0 +1,56 @@
+'use strict';
+
+let Fetcher = module.exports;
+
+/**
+ * @typedef ResponseSummary
+ * @prop {Boolean} ok
+ * @prop {Headers} headers
+ * @prop {Number} status
+ * @prop {String} body
+ */
+
+/**
+ * @param {String} url
+ * @param {RequestInit} opts
+ * @returns {Promise<ResponseSummary>}
+ */
+Fetcher.fetch = async function (url, opts) {
+  let resp = await fetch(url, opts);
+  let summary = Fetcher.throwIfNotOk(resp);
+
+  return summary;
+};
+
+/**
+ * @param {Response} resp
+ * @returns {Promise<ResponseSummary>}
+ */
+Fetcher.throwIfNotOk = async function (resp) {
+  let text = await resp.text();
+
+  if (!resp.ok) {
+    let headers = Array.from(resp.headers);
+    console.error('[Fetcher] error: Response Headers:', headers);
+    console.error('[Fetcher] error: Response Text:', text);
+    let err = new Error(`fetch was not ok`);
+    Object.assign({
+      status: 503,
+      code: 'E_FETCH_RELEASES',
+      response: {
+        status: resp.status,
+        headers: headers,
+        body: text,
+      },
+    });
+    throw err;
+  }
+
+  let summary = {
+    ok: resp.ok,
+    headers: resp.headers,
+    status: resp.status,
+    body: text,
+  };
+  return summary;
+};

_common/git-tag.js

@@ -115,7 +115,7 @@ Repos.getCommitInfo = async function (repoPath, commitish) {
  * @param {string} gitUrl
  * @returns {PromiseLike<any> | Promise<any>}
  */
-async function getAllReleases(gitUrl) {
+async function getDistributables(gitUrl) {
   let all = {
     releases: [],
     download: '',
@@ -190,7 +190,7 @@ async function getAllReleases(gitUrl) {
   return all;
 }
 
-module.exports = getAllReleases;
+module.exports = getDistributables;
 
 if (module === require.main) {
   (async function main() {
@@ -203,7 +203,7 @@ if (module === require.main) {
       //'https://github.com/dense-analysis/ale.git',
     ];
     for (let url of testRepos) {
-      let all = await getAllReleases(url);
+      let all = await getDistributables(url);
 
       all = require('../_webi/normalize.js')(all);
       console.info(JSON.stringify(all, null, 2));

_common/gitea.js

@@ -1,39 +1,47 @@
 'use strict';
 
-var ghRelease = require('./github.js');
+var GitHubish = require('./githubish.js');
 
 /**
- * Gets the releases for 'ripgrep'. This function could be trimmed down and made
- * for use with any github release.
+ * Lists Gitea Releases (w/ uploaded assets)
  *
- * @param request
- * @param {string} owner
- * @param {string} repo
- * @returns {PromiseLike<any> | Promise<any>}
+ * @param {null} _ - deprecated
+ * @param {String} owner
+ * @param {String} repo
+ * @param {String} baseurl
+ * @param {String} [username]
+ * @param {String} [token]
  */
-function getAllReleases(request, owner, repo, baseurl) {
-  if (!baseurl) {
-    return Promise.reject('missing baseurl');
-  }
-  return ghRelease(request, owner, repo, baseurl + '/api/v1').then(
-    function (all) {
-      return all;
-    },
-  );
+async function getDistributables(
+  _,
+  owner,
+  repo,
+  baseurl,
+  username = '',
+  token = '',
+) {
+  baseurl = `${baseurl}/api/v1`;
+  let all = await GitHubish.getDistributables({
+    owner,
+    repo,
+    baseurl,
+    username,
+    token,
+  });
+  return all;
 }
 
-module.exports = getAllReleases;
+module.exports = getDistributables;
 
 if (module === require.main) {
-  getAllReleases(
-    require('@root/request'),
-    'coolaj86',
-    'go-pathman',
-    'https://git.coolaj86.com',
-  ).then(
-    //getAllReleases(require('@root/request'), 'root', 'serviceman', 'https://git.rootprojects.org').then(
-    function (all) {
-      console.info(JSON.stringify(all, null, 2));
-    },
-  );
+  getDistributables(
+    null,
+    'root',
+    'pathman',
+    'https://git.rootprojects.org',
+    '',
+    '',
+  ).then(function (all) {
+    console.info(JSON.stringify(all, null, 2));
+  });
 }

_common/github-source.js

@@ -1,133 +1,40 @@
 'use strict';
 
-require('dotenv').config();
+require('dotenv').config({ path: '.env' });
+
+let GitHubSource = module.exports;
+
+let GitHubishSource = require('./githubish-source.js');
 
 /**
- * Gets the releases for 'ripgrep'. This function could be trimmed down and made
- * for use with any github release.
- *
- * @param request
- * @param {string} owner
- * @param {string} repo
- * @returns {PromiseLike<any> | Promise<any>}
+ * @param {Object} opts
+ * @param {String} opts.owner
+ * @param {String} opts.repo
+ * @param {String} [opts.baseurl]
+ * @param {String} [opts.username]
+ * @param {String} [opts.token]
  */
-async function getAllReleases(
-  request,
+GitHubSource.getDistributables = async function ({
   owner,
   repo,
-  oses,
-  arches,
   baseurl = 'https://api.github.com',
-) {
-  if (!owner) {
-    return Promise.reject('missing owner for repo');
-  }
-  if (!repo) {
-    return Promise.reject('missing repo name');
-  }
-
-  let req = {
-    url: `${baseurl}/repos/${owner}/${repo}/releases`,
-    json: true,
-  };
-
-  // TODO I really don't like global config, find a way to do better
-  if (process.env.GITHUB_USERNAME) {
-    req.auth = {
-      user: process.env.GITHUB_USERNAME,
-      pass: process.env.GITHUB_TOKEN,
-    };
-  }
-
-  let resp = await request(req);
-  let gHubResp = resp.body;
-  let all = {
-    releases: [],
-    // TODO make this ':baseurl' + ':releasename'
-    download: '',
-  };
-
-  for (let release of gHubResp) {
-    // TODO tags aren't always semver / sensical
-    let tag = release['tag_name'];
-    let lts = /(\b|_)(lts)(\b|_)/.test(release['tag_name']);
-    let channel = 'stable';
-    if (release['prerelease']) {
-      channel = 'beta';
-    }
-    let date = release['published_at'] || '';
-    date = date.replace(/T.*/, '');
-
-    let urls = [release.tarball_url, release.zipball_url];
-    for (let url of urls) {
-      let resp = await request({
-        method: 'HEAD',
-        followRedirect: true,
-        followAllRedirects: true,
-        followOriginalHttpMethod: true,
-        url: url,
-        stream: true,
-      });
-      // Workaround for bug where method changes to GET
-      resp.destroy();
-
-      // content-disposition: attachment; filename=BeyondCodeBootcamp-DuckDNS.sh-v1.0.1-0-ga2f4bde.zip
-      let name = resp.headers['content-disposition'].replace(
-        /.*filename=([^;]+)(;|$)/,
-        '$1',
-      );
-      all.releases.push({
-        name: name,
-        version: tag,
-        lts: lts,
-        channel: channel,
-        date: date,
-        os: '', // will be guessed by download filename
-        arch: '', // will be guessed by download filename
-        ext: '', // will be normalized
-        download: resp.request.uri.href,
-      });
-    }
-  }
-
-  if (oses) {
-    return combinate(all, oses, arches);
-  }
-
+  username = process.env.GITHUB_USERNAME || '',
+  token = process.env.GITHUB_TOKEN || '',
+}) {
+  let all = await GitHubishSource.getDistributables({
+    owner,
+    repo,
+    baseurl,
+    username,
+    token,
+  });
   return all;
-}
-
-function combinate(all, oses, arches) {
-  let releases = all.releases;
-  // ex: arches = ['amd64', 'arm64', 'armv7l', 'armv6l', 'x86'];
-  // ex: oses = ['macos', 'linux', 'bsd', 'posix'];
-
-  let combos = [];
-  for (let release of releases) {
-    for (let arch of arches) {
-      for (let os of oses) {
-        let combo = {
-          arch: arch,
-          os: os,
-        };
-        let rel = Object.assign({}, release, combo);
-        combos.push(rel);
-      }
-    }
-  }
-  all.releases = combos;
-
-  return all;
-}
-
-module.exports = getAllReleases;
+};
 
 if (module === require.main) {
-  getAllReleases(
-    require('@root/request'),
-    'BeyondCodeBootcamp',
-    'DuckDNS.sh',
-  ).then(function (all) {
-    console.info(JSON.stringify(all, null, 2));
-  });
+  GitHubSource.getDistributables(null, 'BeyondCodeBootcamp', 'DuckDNS.sh').then(
+    function (all) {
+      console.info(JSON.stringify(all, null, 2));
+    },
+  );
 }

_common/github.js

@@ -1,102 +1,42 @@
 'use strict';
 
-require('dotenv').config();
+require('dotenv').config({ path: '.env' });
+
+let GitHubish = require('./githubish.js');
 
 /**
  * Lists GitHub Releases (w/ uploaded assets)
  *
- * @param request
- * @param {string} owner
- * @param {string} repo
- * @returns {PromiseLike<any> | Promise<any>}
+ * @param {null} _ - deprecated
+ * @param {String} owner
+ * @param {String} repo
+ * @param {String} [baseurl]
+ * @param {String} [username]
+ * @param {String} [token]
  */
-async function getAllReleases(
-  request,
+module.exports = async function (
+  _,
   owner,
   repo,
   baseurl = 'https://api.github.com',
+  username = process.env.GITHUB_USERNAME || '',
+  token = process.env.GITHUB_TOKEN || '',
 ) {
-  if (!owner) {
-    throw new Error('missing owner for repo');
-  }
-  if (!repo) {
-    throw new Error('missing repo name');
-  }
-
-  var req = {
-    url: `${baseurl}/repos/${owner}/${repo}/releases`,
-    json: true,
-  };
-
-  // TODO I really don't like global config, find a way to do better
-  if (process.env.GITHUB_USERNAME) {
-    req.auth = {
-      user: process.env.GITHUB_USERNAME,
-      pass: process.env.GITHUB_TOKEN,
-    };
-  }
-
-  let resp = await request(req);
-  if (!resp.ok) {
-    console.error('Bad Resp Headers:', resp.headers);
-    console.error('Bad Resp Body:', resp.body);
-    throw new Error('the elusive releases BOOGEYMAN strikes again');
-  }
-
-  let gHubResp = resp.body;
-  let all = {
-    releases: [],
-    // todo make this ':baseurl' + ':releasename'
-    download: '',
-  };
-
-  try {
-    gHubResp.forEach(transformReleases);
-  } catch (e) {
-    console.error(e.message);
-    console.error('Error Headers:', resp.headers);
-    console.error('Error Body:', resp.body);
-    throw e;
-  }
-
-  function transformReleases(release) {
-    for (let asset of release['assets']) {
-      let name = asset['name'];
-      let date = release['published_at']?.replace(/T.*/, '');
-      let download = asset['browser_download_url'];
-
-      // TODO tags aren't always semver / sensical
-      let version = release['tag_name'];
-      let channel;
-      if (release['prerelease']) {
-        // -rcX, -preview, -beta, etc will be checked in _webi/normalize.js
-        channel = 'beta';
-      }
-      let lts = /(\b|_)(lts)(\b|_)/.test(release['tag_name']);
-
-      all.releases.push({
-        name: name,
-        version: version,
-        lts: lts,
-        channel: channel,
-        date: date,
-        os: '', // will be guessed by download filename
-        arch: '', // will be guessed by download filename
-        ext: '', // will be normalized
-        download: download,
-      });
-    }
-  }
-
+  let all = await GitHubish.getDistributables({
+    owner,
+    repo,
+    baseurl,
+    username,
+    token,
+  });
   return all;
-}
+};
 
-module.exports = getAllReleases;
+let GitHub = module.exports;
+GitHub.getDistributables = module.exports;
 
 if (module === require.main) {
-  getAllReleases(require('@root/request'), 'BurntSushi', 'ripgrep').then(
-    function (all) {
-      console.info(JSON.stringify(all, null, 2));
-    },
-  );
+  GitHub.getDistributables(null, 'BurntSushi', 'ripgrep').then(function (all) {
+    console.info(JSON.stringify(all, null, 2));
+  });
 }

_common/githubish-source.js

@@ -0,0 +1,174 @@
+'use strict';
+
+let Fetcher = require('../_common/fetcher.js');
+
+let GitHubishSource = module.exports;
+
+/**
+ * Lists GitHub-Like Releases (source tarball & zip)
+ *
+ * @param {Object} opts
+ * @param {String} opts.owner
+ * @param {String} opts.repo
+ * @param {String} opts.baseurl
+ * @param {String} [opts.username]
+ * @param {String} [opts.token]
+ */
+GitHubishSource.getDistributables = async function ({
+  owner,
+  repo,
+  baseurl,
+  username = '',
+  token = '',
+}) {
+  if (!owner) {
+    throw new Error('missing owner for repo');
+  }
+  if (!repo) {
+    throw new Error('missing repo name');
+  }
+  if (!baseurl) {
+    throw new Error('missing baseurl');
+  }
+
+  let url = `${baseurl}/repos/${owner}/${repo}/releases`;
+  let opts = {
+    headers: {
+      'Content-Type': 'appplication/json',
+    },
+  };
+
+  if (token) {
+    let userpass = `${username}:${token}`;
+    let basicAuth = btoa(userpass);
+    Object.assign(opts.headers, {
+      Authorization: `Basic ${basicAuth}`,
+    });
+  }
+
+  let resp;
+  try {
+    resp = await Fetcher.fetch(url, opts);
+  } catch (e) {
+    /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error
+    let err = e;
+    if (err.code === 'E_FETCH_RELEASES') {
+      err.message = `failed to fetch '${baseurl}' (githubish-source, user '${username}) release data: ${err.response.status} ${err.response.body}`;
+    }
+    throw e;
+  }
+  let gHubResp = JSON.parse(resp.body);
+
+  let all = {
+    /** @type {Array<BuildInfo>} */
+    releases: [],
+    download: '',
+  };
+
+  for (let release of gHubResp) {
+    let dists = GitHubishSource.releaseToDistributables(release);
+    for (let dist of dists) {
+      let updates =
+        await GitHubishSource.followDistributableDownloadAttachment(dist);
+      Object.assign(dist, updates);
+      all.releases.push(dist);
+    }
+  }
+
+  return all;
+};
+
+/**
+ * @typedef BuildInfo
+ * @prop {String} [name] - name to use instead of filename for hash urls
+ * @prop {String} version
+ * @prop {String} [_version]
+ * @prop {String} [arch]
+ * @prop {String} channel
+ * @prop {String} date
+ * @prop {String} download
+ * @prop {String} [ext]
+ * @prop {String} [_filename]
+ * @prop {String} [hash]
+ * @prop {String} [libc]
+ * @prop {Boolean} [_musl]
+ * @prop {Boolean} [lts]
+ * @prop {String} [size]
+ * @prop {String} os
+ */
+
+/**
+ * @param {any} ghRelease - TODO
+ * @returns {Array<BuildInfo>}
+ */
+GitHubishSource.releaseToDistributables = function (ghRelease) {
+  let ghTag = ghRelease['tag_name']; // TODO tags aren't always semver / sensical
+  let lts = /(\b|_)(lts)(\b|_)/.test(ghRelease['tag_name']);
+  let channel = 'stable';
+  if (ghRelease['prerelease']) {
+    channel = 'beta';
+  }
+  let date = ghRelease['published_at'] || '';
+  date = date.replace(/T.*/, '');
+
+  let urls = [ghRelease.tarball_url, ghRelease.zipball_url];
+  /** @type {Array<BuildInfo>} */
+  let dists = [];
+  for (let url of urls) {
+    dists.push({
+      name: '',
+      version: ghTag,
+      lts: lts,
+      channel: channel,
+      date: date,
+      os: '*',
+      arch: '*',
+      libc: '',
+      ext: '',
+      download: url,
+    });
+  }
+
+  return dists;
+};
+
+/**
+ * @param {BuildInfo} dist
+ */
+GitHubishSource.followDistributableDownloadAttachment = async function (dist) {
+  let abortCtrl = new AbortController();
+  let resp = await fetch(dist.download, {
+    method: 'HEAD',
+    redirect: 'follow',
+    signal: abortCtrl.signal,
+  });
+  let headers = Object.fromEntries(resp.headers);
+
+  // Workaround for bug where METHOD changes to GET
+  abortCtrl.abort();
+  await resp.text().catch(function (err) {
+    if (err.name !== 'AbortError') {
+      throw err;
+    }
+  });
+
+  // ex: content-disposition: attachment; filename=BeyondCodeBootcamp-DuckDNS.sh-v1.0.1-0-ga2f4bde.zip
+  //  => BeyondCodeBootcamp-DuckDNS.sh-v1.0.1-0-ga2f4bde.zip
+  let name = headers['content-disposition'].replace(
+    /.*filename=([^;]+)(;|$)/,
+    '$1',
+  );
+  let download = resp.url;
+
+  return { name, download };
+};
+
+if (module === require.main) {
+  GitHubishSource.getDistributables({
+    owner: 'BeyondCodeBootcamp',
+    repo: 'DuckDNS.sh',
+    baseurl: 'https://api.github.com',
+  }).then(function (all) {
+    console.info(JSON.stringify(all, null, 2));
+  });
+}

_common/githubish.js

@@ -0,0 +1,137 @@
+'use strict';
+
+let Fetcher = require('../_common/fetcher.js');
+
+/**
+ * @typedef DistributableRaw
+ * @prop {String} name
+ * @prop {String} version
+ * @prop {Boolean} lts
+ * @prop {String} [channel]
+ * @prop {String} date
+ * @prop {String} os
+ * @prop {String} arch
+ * @prop {String} ext
+ * @prop {String} download
+ */
+
+let GitHubish = module.exports;
+
+/**
+ * Lists GitHub-Like Releases (w/ uploaded assets)
+ *
+ * @param {Object} opts
+ * @param {String} opts.owner
+ * @param {String} opts.repo
+ * @param {String} opts.baseurl
+ * @param {String} [opts.username]
+ * @param {String} [opts.token]
+ */
+GitHubish.getDistributables = async function ({
+  owner,
+  repo,
+  baseurl,
+  username = '',
+  token = '',
+}) {
+  if (!owner) {
+    throw new Error('missing owner for repo');
+  }
+  if (!repo) {
+    throw new Error('missing repo name');
+  }
+  if (!baseurl) {
+    throw new Error('missing baseurl');
+  }
+
+  let url = `${baseurl}/repos/${owner}/${repo}/releases`;
+  let opts = {
+    headers: {
+      'Content-Type': 'appplication/json',
+    },
+  };
+
+  if (token) {
+    let userpass = `${username}:${token}`;
+    let basicAuth = btoa(userpass);
+    Object.assign(opts.headers, {
+      Authorization: `Basic ${basicAuth}`,
+    });
+  }
+
+  let resp;
+  try {
+    resp = await Fetcher.fetch(url, opts);
+  } catch (e) {
+    /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error
+    let err = e;
+    if (err.code === 'E_FETCH_RELEASES') {
+      err.message = `failed to fetch '${baseurl}' (githubish, user '${username}) release data: ${err.response.status} ${err.response.body}`;
+    }
+    throw e;
+  }
+  let gHubResp = JSON.parse(resp.body);
+
+  let all = {
+    /** @type {Array<DistributableRaw>} */
+    releases: [],
+    // todo make this ':baseurl' + ':releasename'
+    download: '',
+  };
+
+  try {
+    gHubResp.forEach(transformReleases);
+  } catch (e) {
+    /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error
+    let err = e;
+    console.error(err.message);
+    console.error('Error Headers:', resp.headers);
+    console.error('Error Body:', resp.body);
+    let msg = `failed to transform releases from '${baseurl}' with user '${username}'`;
+    throw new Error(msg);
+  }
+
+  /**
+   * @param {any} release - TODO
+   */
+  function transformReleases(release) {
+    for (let asset of release['assets']) {
+      let name = asset['name'];
+      let date = release['published_at']?.replace(/T.*/, '');
+      let download = asset['browser_download_url'];
+
+      // TODO tags aren't always semver / sensical
+      let version = release['tag_name'];
+      let channel;
+      if (release['prerelease']) {
+        // -rcX, -preview, -beta, etc will be checked in _webi/normalize.js
+        channel = 'beta';
+      }
+      let lts = /(\b|_)(lts)(\b|_)/.test(release['tag_name']);
+
+      all.releases.push({
+        name: name,
+        version: version,
+        lts: lts,
+        channel: channel,
+        date: date,
+        os: '', // will be guessed by download filename
+        arch: '', // will be guessed by download filename
+        ext: '', // will be normalized
+        download: download,
+      });
+    }
+  }
+
+  return all;
+};
+
+if (module === require.main) {
+  GitHubish.getDistributables({
+    owner: 'BurntSushi',
+    repo: 'ripgrep',
+    baseurl: 'https://api.github.com',
+  }).then(function (all) {
+    console.info(JSON.stringify(all, null, 2));
+  });
+}

_example/README.md

@@ -23,7 +23,7 @@ install:
 ```text
 ~/.config/envman/PATH.env
 ~/.local/bin/foo
-~/.local/opt/foo
+~/.local/opt/foo/
 ```
 
 ## Cheat Sheet

_example/install.ps1

@@ -19,13 +19,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading foobar from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-IF (!(Test-Path -Path "$pkg_src_cmd")) {
+if (!(Test-Path -Path "$pkg_src_cmd")) {
     Write-Output "Installing foobar"
 
     # TODO: create package-specific temp directory

_example/releases.js

@@ -12,17 +12,26 @@ var repo = 'ripgrep';
 /**                                                                          **/
 /******************************************************************************/
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
-    return all;
-  });
+let Releases = module.exports;
+
+Releases.latest = async function () {
+  let all = await github(null, owner, repo);
+  return all;
+};
+
+Releases.sample = async function () {
+  let normalize = require('../_webi/normalize.js');
+  let all = await Releases.latest();
+  all = normalize(all);
+  // just select the first 5 for demonstration
+  all.releases = all.releases.slice(0, 5);
+  return all;
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    // just select the first 5 for demonstration
-    all.releases = all.releases.slice(0, 5);
-    console.info(JSON.stringify(all, null, 2));
-  });
+  (async function () {
+    let samples = await Releases.sample();
+
+    console.info(JSON.stringify(samples, null, 2));
+  })();
 }

_npm/scripts/install-webi.js

@@ -4,7 +4,6 @@
 
 //var pkg = require('../package.json');
 var os = require('os');
-//var request = require('@root/request');
 //var promisify = require('util').promisify;
 //var exec = promisify(require('child_process').exec);
 var exec = require('child_process').exec;

_scripts/pwsh-fmt.ps1

@@ -15,8 +15,8 @@ foreach ($my_dir in $my_dirs) {
         $my_ps1 = [System.IO.Path]::GetRelativePath($my_cwd, $my_file.FullName)
         $my_dir = [System.IO.Path]::GetDirectoryName($my_file.FullName)
 
-        if (-Not (Test-Path -PathType Leaf -Path $my_ps1) -or
-            -Not (Test-Path -PathType Container -Path $my_dir)) {
+        if (-not (Test-Path -PathType Leaf -Path $my_ps1) -or
+            -not (Test-Path -PathType Container -Path $my_dir)) {
             Write-Host ("    SKIP {0} (non-regular file or parent directory)" -f $my_ps1)
             continue
         }
@@ -31,7 +31,7 @@ foreach ($my_dir in $my_dirs) {
         $my_new_file | Set-Content -Path $my_ps1
 
         $my_new_file = $my_new_file + "`n"
-        IF ($text -ne $my_new_file) {
+        if ($text -ne $my_new_file) {
             $my_status = 1
         }
     }

_scripts/pwsh-lint.ps1

@@ -15,8 +15,8 @@ foreach ($my_dir in $my_dirs) {
         $my_ps1 = [System.IO.Path]::GetRelativePath($my_cwd, $my_file.FullName)
         $my_dir = [System.IO.Path]::GetDirectoryName($my_file.FullName)
 
-        if (-Not (Test-Path -PathType Leaf -Path $my_ps1) -or
-            -Not (Test-Path -PathType Container -Path $my_dir)) {
+        if (-not (Test-Path -PathType Leaf -Path $my_ps1) -or
+            -not (Test-Path -PathType Container -Path $my_dir)) {
             Write-Host ("    SKIP {0} (non-regular file or parent directory)" -f $my_ps1)
             continue
         }
@@ -39,7 +39,7 @@ foreach ($my_dir in $my_dirs) {
         $my_new_file | Set-Content -Path $my_ps1
 
         $my_new_file = $my_new_file + "`n"
-        IF ($my_old_file -ne $my_new_file) {
+        if ($my_old_file -ne $my_new_file) {
             $my_status = 1
         }
     }

_vim-example/install.ps1

@@ -1,6 +1,6 @@
 #!/usr/bin/env pwsh
 
-IF (!(Test-Path -Path "$Env:USERPROFILE\.vim\pack\plugins\start")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\.vim\pack\plugins\start")) {
     New-Item -Path "$Env:USERPROFILE\.vim\pack\plugins\start" -ItemType Directory -Force | Out-Null
 }
 Remove-Item -Path "$Env:USERPROFILE\.vim\pack\plugins\start\example" -Recurse -ErrorAction Ignore

_webi/builds-cacher-test.js

@@ -0,0 +1,73 @@
+'use strict';
+
+let Path = require('node:path');
+
+let BuildsCacher = require('./builds-cacher.js');
+// let Parallel = require('./parallel.js');
+
+var INSTALLERS_DIR = Path.join(__dirname, '..');
+var CACHE_DIR = Path.join(__dirname, '../_cache');
+
+async function main() {
+  let bc = BuildsCacher.create({
+    caches: CACHE_DIR,
+    installers: INSTALLERS_DIR,
+  });
+  bc.freshenRandomPackage(600 * 1000);
+
+  // let dirs = await bc.getProjectsByType();
+  // let projNames = Object.keys(dirs.valid);
+
+  let lastUpdate;
+
+  let projName = 'k9s';
+  {
+    let packages = await bc.getPackages({
+      //Releases: Releases,
+      name: projName,
+      date: new Date(),
+    });
+    lastUpdate = packages.updated;
+    console.info(
+      `Last update for '${projName}': ${packages.updated} (${packages.releases.length} assets)`,
+    );
+  }
+
+  console.info('Waiting 5s');
+  {
+    setTimeout(async function () {
+      let packages = await bc.getPackages({
+        //Releases: Releases,
+        name: projName,
+        date: new Date(),
+      });
+      console.info(
+        `Last update for '${projName}': ${packages.updated} (${packages.releases.length} assets)`,
+      );
+      if (lastUpdate < packages.updated) {
+        console.info(`PASS`);
+      } else {
+        console.info(`MAYBE fail`);
+      }
+    }, 5 * 1000);
+  }
+
+  //let parallel = 25;
+  //await Parallel.run(parallel, projNames, getAll);
+  //async function getAll(name) {
+  //  void (await bc.getPackages({
+  //    //Releases: Releases,
+  //    name: name,
+  //    date: new Date(),
+  //  }));
+  //}
+}
+
+main()
+  .then(function () {
+    console.log('Done');
+  })
+  .catch(function (e) {
+    console.error(e.stack || e);
+    process.exit(1);
+  });

_webi/builds-cacher.js

@@ -9,8 +9,6 @@ let HostTargets = require('./build-classifier/host-targets.js');
 let Lexver = require('./build-classifier/lexver.js');
 let Triplet = require('./build-classifier/triplet.js');
 
-let request = require('@root/request');
-
 var ALIAS_RE = /^alias: (\w+)$/m;
 
 var LEGACY_ARCH_MAP = {
@@ -130,6 +128,8 @@ async function readFirstBytes(path) {
 
 let promises = {};
 async function getLatestBuilds(Releases, installersDir, cacheDir, name, date) {
+  console.info(`[INFO] getLatestBuilds: ${name}`);
+
   if (!Releases) {
     Releases = require(`${installersDir}/${name}/releases.js`);
   }
@@ -151,7 +151,7 @@ async function getLatestBuilds(Releases, installersDir, cacheDir, name, date) {
 }
 
 async function getLatestBuildsInner(Releases, cacheDir, name, date) {
-  let data = await Releases.latest(request);
+  let data = await Releases.latest();
 
   if (!date) {
     date = new Date();
@@ -187,9 +187,10 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
   }
 
   let bc = {};
-  bc.usedTerms = {};
-  bc.orphanTerms = Object.assign({}, ALL_TERMS);
+  bc.ALL_TERMS = ALL_TERMS;
+  bc.orphanTerms = Object.assign({}, bc.ALL_TERMS);
   bc.unknownTerms = {};
+  bc.usedTerms = {};
   bc.formats = [];
   bc._triplets = {};
   bc._targetsByBuildIdCache = {};
@@ -323,17 +324,21 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     let isoDate = date.toISOString();
     let yearMonth = isoDate.slice(0, 7);
     let dataFile = `${cacheDir}/${yearMonth}/${name}.json`;
+    let tsFile = `${cacheDir}/${yearMonth}/${name}.updated.txt`;
 
-    // let secondsStr = await Fs.readFile(tsFile, 'ascii').catch(function (err) {
-    //   if (err.code !== 'ENOENT') {
-    //     throw err;
-    //   }
-    //   return '0';
-    // });
-    // secondsStr = secondsStr.trim();
-    // let seconds = parseFloat(secondsStr) || 0;
-
-    // let age = now - seconds;
+    let tsDate;
+    {
+      let secondsStr = await Fs.readFile(tsFile, 'ascii').catch(function (err) {
+        if (err.code !== 'ENOENT') {
+          throw err;
+        }
+        return '0';
+      });
+      secondsStr = secondsStr.trim();
+      let seconds = parseFloat(secondsStr) || 0;
+      let ms = seconds * 1000;
+      tsDate = new Date(ms);
+    }
 
     let projInfo = bc._caches[name];
 
@@ -375,112 +380,38 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     if (!projInfo) {
       projInfo = await getLatestBuilds(Releases, installersDir, cacheDir, name);
     }
+    let latestProjInfo = await BuildsCacher.transformAndUpdate(
+      name,
+      projInfo,
+      meta,
+      tsDate,
+      bc,
+    );
+    bc._caches[name] = latestProjInfo;
 
     process.nextTick(async function () {
       let now = date.valueOf();
       let age = now - projInfo.updated;
-      if (age < bc._staleAge) {
+
+      let fresh = age < bc._staleAge;
+      if (fresh) {
         return;
       }
 
       projInfo = await getLatestBuilds(Releases, installersDir, cacheDir, name);
-      transformAndUpdate(name, projInfo, meta, date);
+      let latestProjInfo = BuildsCacher.transformAndUpdate(
+        name,
+        projInfo,
+        meta,
+        date,
+        bc,
+      );
+      bc._caches[name] = latestProjInfo;
     });
 
-    transformAndUpdate(name, projInfo, meta, date);
     return projInfo;
   };
 
-  function transformAndUpdate(name, projInfo, meta, date) {
-    meta.packages = [];
-
-    let updated = date.valueOf();
-    Object.assign(projInfo, { name, updated }, meta);
-    for (let build of projInfo.releases) {
-      let buildTarget = bc.classify(projInfo, build);
-      if (!buildTarget) {
-        // ignore known, non-package extensions
-        continue;
-      }
-
-      if (buildTarget.error) {
-        let err = buildTarget.error;
-        let code = err.code || '';
-        console.error(`[ERROR]: ${code} ${projInfo.name}: ${build.name}`);
-        console.error(`>>> ${err.message} <<<`);
-        console.error(projInfo);
-        console.error(build);
-        console.error(`^^^ ${err.message} ^^^`);
-        console.error(err.stack);
-        continue;
-      }
-
-      if (!build.name) {
-        build.name = build.download.replace(/.*\//, '');
-      }
-
-      build.target = buildTarget;
-      meta.packages.push(build);
-    }
-
-    updateReleasesByTriplet(meta);
-    updateAndSortVersions(projInfo, meta);
-
-    Object.assign(projInfo, { name, updated }, meta);
-    bc._caches[name] = projInfo;
-  }
-
-  function updateReleasesByTriplet(meta) {
-    for (let build of meta.packages) {
-      let target = build.target;
-
-      let triplet = `${target.os}-${target.arch}-${target.libc}`;
-      if (!meta.releasesByTriplet[triplet]) {
-        meta.releasesByTriplet[triplet] = {};
-      }
-
-      let buildsByRelease = meta.releasesByTriplet[triplet];
-      if (!buildsByRelease[build.version]) {
-        buildsByRelease[build.version] = [];
-      }
-
-      let packages = buildsByRelease[build.version];
-      packages.push(build);
-    }
-  }
-
-  // TODO
-  //   - tag channels
-  function updateAndSortVersions(projInfo, meta) {
-    for (let build of projInfo.packages) {
-      let hasVersion = meta.versions.includes(build.version);
-      if (!hasVersion) {
-        build.lexver = Lexver.parseVersion(build.version);
-        meta.lexversMap[build.lexver] = build.version;
-      }
-    }
-
-    meta.lexvers = Object.keys(meta.lexversMap);
-    meta.lexvers.sort();
-    meta.lexvers.reverse();
-
-    meta.versions = [];
-    for (let lexver of meta.lexvers) {
-      let version = meta.lexversMap[lexver];
-      meta.versions.push(version);
-    }
-
-    projInfo.packages.sort(function (a, b) {
-      if (a.lexver > b.lexver) {
-        return -1;
-      }
-      if (a.lexver < b.lexver) {
-        return 1;
-      }
-      return 0;
-    });
-  }
-
   // Makes sure that packages are updated once an hour, on average
   bc._staleNames = [];
   bc._freshenTimeout = null;
@@ -503,6 +434,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
       name: name,
       date: new Date(),
     }));
+    console.info(`[INFO] freshenRandomPackage: ${name}`);
 
     let hour = 60 * 60 * 1000;
     let delay = minDelay;
@@ -515,134 +447,6 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     bc._freshenTimeout.unref();
   };
 
-  bc.classify = function (projInfo, build) {
-    /* jshint maxcomplexity: 25 */
-    let maybeInstallable = Triplet.maybeInstallable(projInfo, build);
-    if (!maybeInstallable) {
-      return null;
-    }
-
-    if (LEGACY_OS_MAP[build.os]) {
-      build.os = LEGACY_OS_MAP[build.os];
-    }
-    if (LEGACY_ARCH_MAP[build.arch]) {
-      build.arch = LEGACY_ARCH_MAP[build.arch];
-    }
-
-    // because some packages are shimmed to match a single download against
-    let preTarget = Object.assign({ os: '', arch: '', libc: '' }, build);
-
-    let targetId = `${preTarget.os}:${preTarget.arch}:${preTarget.libc}`;
-    let buildId = `${projInfo.name}:${targetId}@${build.download}`;
-    let target = bc._targetsByBuildIdCache[buildId];
-    if (target) {
-      Object.assign(build, { target: target, triplet: target.triplet });
-      return target;
-    }
-
-    let pattern = Triplet.toPattern(projInfo, build);
-    if (!pattern) {
-      let err = new Error(`no pattern generated for ${projInfo.name}`);
-      err.code = 'E_BUILD_NO_PATTERN';
-      target = { error: err };
-      bc._targetsByBuildIdCache[buildId] = target;
-      return target;
-    }
-
-    let rawTerms = pattern.split(/[_\{\}\/\.\-]+/g);
-    for (let term of rawTerms) {
-      delete bc.orphanTerms[term];
-      bc.usedTerms[term] = true;
-    }
-
-    // {NAME}/{NAME}-{VER}-Windows-x86_64_v2-musl.exe =>
-    //     {NAME}.windows.x86_64v2.musl.exe
-    let terms = Triplet.patternToTerms(pattern);
-    if (!terms.length) {
-      let err = new Error(`'${terms}' was trimmed to ''`);
-      target = { error: err };
-      bc._targetsByBuildIdCache[buildId] = target;
-      return target;
-    }
-
-    for (let term of terms) {
-      if (!term) {
-        continue;
-      }
-
-      if (ALL_TERMS[term]) {
-        delete bc.orphanTerms[term];
-        bc.usedTerms[term] = true;
-        continue;
-      }
-
-      bc.unknownTerms[term] = true;
-    }
-
-    // {NAME}.windows.x86_64v2.musl.exe
-    //     windows-x86_64_v2-musl
-    target = { triplet: '' };
-    void Triplet.termsToTarget(target, projInfo, build, terms);
-
-    target.triplet = `${target.arch}-${target.vendor}-${target.os}-${target.libc}`;
-
-    {
-      // TODO I don't love this hidden behavior
-      // perhaps classify should just happen when the package is loaded
-      // (and the sanity error should be removed, or thrown after the loop is complete)
-      let hasTriplet = projInfo.triplets.includes(target.triplet);
-      if (!hasTriplet) {
-        projInfo.triplets.push(target.triplet);
-      }
-      let hasOs = projInfo.oses.includes(target.os);
-      if (!hasOs) {
-        projInfo.oses.push(target.os);
-      }
-      let hasArch = projInfo.arches.includes(target.arch);
-      if (!hasArch) {
-        projInfo.arches.push(target.arch);
-      }
-      let hasLibc = projInfo.libcs.includes(target.libc);
-      if (!hasLibc) {
-        projInfo.libcs.push(target.libc);
-      }
-
-      if (!build.ext) {
-        build.ext = Triplet.buildToPackageType(build);
-      }
-      if (build.ext) {
-        if (!build.ext.startsWith('.')) {
-          build.ext = `.${build.ext}`;
-        }
-      }
-      let hasExt = projInfo.formats.includes(build.ext);
-      if (!hasExt) {
-        projInfo.formats.push(build.ext);
-      }
-      let hasGlobalExt = bc.formats.includes(build.ext);
-      if (!hasGlobalExt) {
-        bc.formats.push(build.ext);
-      }
-    }
-
-    bc._triplets[target.triplet] = true;
-    bc._targetsByBuildIdCache[buildId] = target;
-
-    let triple = [target.arch, target.vendor, target.os, target.libc];
-    for (let term of triple) {
-      if (!ALL_TERMS[term]) {
-        throw new Error(
-          `[SANITY FAIL] '${projInfo.name}' '${target.triplet}' generated unknown term '${term}'`,
-        );
-      }
-
-      delete bc.orphanTerms[term];
-      bc.usedTerms[term] = true;
-    }
-
-    return target;
-  };
-
   /**
    * Given a list of acceptable formats, get the sorted list of of formats.
    * Actually used (as per node _webi/lint-builds.js):
@@ -736,6 +540,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     exts.push('.git');
 
     // Fallbacks
+    // (we include everything to bubble an extract error over not found)
     exts.push('.app.zip');
     exts.push('.dmg');
     exts.push('.pkg');
@@ -787,9 +592,8 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
 
   /**
    * @param {ProjectInfo} projInfo
-   * @param {HostTarget} hostTarget
    */
-  bc.enumerateLatestVersions = function (projInfo, hostTarget) {
+  bc.enumerateLatestVersions = function (projInfo) {
     let lexPrefix = '';
     let matchInfo = Lexver.matchSorted(projInfo.lexvers, lexPrefix);
     let verInfo = {
@@ -919,9 +723,9 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     if (hostTarget.os === 'windows') {
       oses = ['ANYOS', 'windows'];
     } else if (hostTarget.os === 'android') {
-      oses = ['ANYOS', 'posix_2017', 'android', 'linux'];
+      oses = ['ANYOS', 'posix_2017', 'posix_2024', 'android', 'linux'];
     } else {
-      oses = ['ANYOS', 'posix_2017', hostTarget.os];
+      oses = ['ANYOS', 'posix_2017', 'posix_2024', hostTarget.os];
     }
 
     let waterfall = HostTargets.WATERFALL[hostTarget.os] || {};
@@ -956,3 +760,233 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
 
   return bc;
 };
+
+BuildsCacher._classify = function (bc, projInfo, build) {
+  /* jshint maxcomplexity: 25 */
+  let maybeInstallable = Triplet.maybeInstallable(projInfo, build);
+  if (!maybeInstallable) {
+    return null;
+  }
+
+  if (LEGACY_OS_MAP[build.os]) {
+    build.os = LEGACY_OS_MAP[build.os];
+  }
+  if (LEGACY_ARCH_MAP[build.arch]) {
+    build.arch = LEGACY_ARCH_MAP[build.arch];
+  }
+
+  // because some packages are shimmed to match a single download against
+  let preTarget = Object.assign({ os: '', arch: '', libc: '' }, build);
+
+  let targetId = `${preTarget.os}:${preTarget.arch}:${preTarget.libc}`;
+  let buildId = `${projInfo.name}:${targetId}@${build.download}`;
+  //console.log(`dbg: buildId`, buildId);
+  let target = bc._targetsByBuildIdCache[buildId];
+  if (target) {
+    Object.assign(build, { target: target, triplet: target.triplet });
+    return target;
+  }
+
+  let pattern = Triplet.toPattern(projInfo, build);
+  //console.log(`dbg: pattern`, pattern);
+  if (!pattern) {
+    let err = new Error(`no pattern generated for ${projInfo.name}`);
+    err.code = 'E_BUILD_NO_PATTERN';
+    target = { error: err };
+    bc._targetsByBuildIdCache[buildId] = target;
+    return target;
+  }
+
+  let rawTerms = pattern.split(/[_\{\}\/\.\-]+/g);
+  //console.log(`dbg: rawTerms`, rawTerms);
+  for (let term of rawTerms) {
+    delete bc.orphanTerms[term];
+    bc.usedTerms[term] = true;
+  }
+
+  // {NAME}/{NAME}-{VER}-Windows-x86_64_v2-musl.exe =>
+  //     {NAME}.windows.x86_64v2.musl.exe
+  let terms = Triplet.patternToTerms(pattern);
+  //console.log(`dbg: terms`, terms);
+  if (!terms.length) {
+    let err = new Error(`'${terms}' was trimmed to ''`);
+    target = { error: err };
+    bc._targetsByBuildIdCache[buildId] = target;
+    return target;
+  }
+
+  for (let term of terms) {
+    if (!term) {
+      continue;
+    }
+
+    if (bc.ALL_TERMS[term]) {
+      delete bc.orphanTerms[term];
+      bc.usedTerms[term] = true;
+      continue;
+    }
+
+    bc.unknownTerms[term] = true;
+  }
+
+  // {NAME}.windows.x86_64v2.musl.exe
+  //     windows-x86_64_v2-musl
+  target = { triplet: '' };
+  try {
+    void Triplet.termsToTarget(target, projInfo, build, terms);
+  } catch (e) {
+    console.error(`PACKAGE FORMAT CHANGE for '${projInfo.name}':`);
+    console.error(e.message);
+    console.error(build);
+    return null;
+  }
+
+  target.triplet = `${target.arch}-${target.vendor}-${target.os}-${target.libc}`;
+
+  {
+    // TODO I don't love this hidden behavior
+    // perhaps classify should just happen when the package is loaded
+    // (and the sanity error should be removed, or thrown after the loop is complete)
+    let hasTriplet = projInfo.triplets.includes(target.triplet);
+    if (!hasTriplet) {
+      projInfo.triplets.push(target.triplet);
+    }
+    let hasOs = projInfo.oses.includes(target.os);
+    if (!hasOs) {
+      projInfo.oses.push(target.os);
+    }
+    let hasArch = projInfo.arches.includes(target.arch);
+    if (!hasArch) {
+      projInfo.arches.push(target.arch);
+    }
+    let hasLibc = projInfo.libcs.includes(target.libc);
+    if (!hasLibc) {
+      projInfo.libcs.push(target.libc);
+    }
+
+    if (!build.ext) {
+      build.ext = Triplet.buildToPackageType(build);
+    }
+    if (build.ext) {
+      if (!build.ext.startsWith('.')) {
+        build.ext = `.${build.ext}`;
+      }
+    }
+    let hasExt = projInfo.formats.includes(build.ext);
+    if (!hasExt) {
+      projInfo.formats.push(build.ext);
+    }
+    let hasGlobalExt = bc.formats.includes(build.ext);
+    if (!hasGlobalExt) {
+      bc.formats.push(build.ext);
+    }
+  }
+
+  bc._triplets[target.triplet] = true;
+  bc._targetsByBuildIdCache[buildId] = target;
+
+  let triple = [target.arch, target.vendor, target.os, target.libc];
+  for (let term of triple) {
+    if (!bc.ALL_TERMS[term]) {
+      throw new Error(
+        `[SANITY FAIL] '${projInfo.name}' '${target.triplet}' generated unknown term '${term}'`,
+      );
+    }
+
+    delete bc.orphanTerms[term];
+    bc.usedTerms[term] = true;
+  }
+
+  return target;
+};
+
+BuildsCacher.transformAndUpdate = function (name, projInfo, meta, date, bc) {
+  meta.packages = [];
+
+  let updated = date.valueOf();
+
+  Object.assign(projInfo, { name, updated }, meta);
+  for (let build of projInfo.releases) {
+    let buildTarget = BuildsCacher._classify(bc, projInfo, build);
+    if (!buildTarget) {
+      // ignore known, non-package extensions
+      continue;
+    }
+
+    if (buildTarget.error) {
+      let err = buildTarget.error;
+      let code = err.code || '';
+      console.error(`[ERROR]: ${code} ${projInfo.name}: ${build.name}`);
+      console.error(`>>> ${err.message} <<<`);
+      console.error(projInfo);
+      console.error(build);
+      console.error(`^^^ ${err.message} ^^^`);
+      console.error(err.stack);
+      continue;
+    }
+
+    if (!build.name) {
+      build.name = build.download.replace(/.*\//, '');
+    }
+
+    build.target = buildTarget;
+    meta.packages.push(build);
+  }
+
+  BuildsCacher.updateReleasesByTriplet(meta);
+  BuildsCacher.updateAndSortVersions(projInfo, meta);
+
+  Object.assign(projInfo, { name, updated }, meta);
+  return projInfo;
+};
+
+// TODO
+//   - tag channels
+BuildsCacher.updateAndSortVersions = function (projInfo, meta) {
+  for (let build of projInfo.packages) {
+    let hasVersion = meta.versions.includes(build.version);
+    if (!hasVersion) {
+      build.lexver = Lexver.parseVersion(build.version);
+      meta.lexversMap[build.lexver] = build.version;
+    }
+  }
+
+  meta.lexvers = Object.keys(meta.lexversMap);
+  meta.lexvers.sort();
+  meta.lexvers.reverse();
+
+  meta.versions = [];
+  for (let lexver of meta.lexvers) {
+    let version = meta.lexversMap[lexver];
+    meta.versions.push(version);
+  }
+
+  projInfo.packages.sort(function (a, b) {
+    if (a.lexver > b.lexver) {
+      return -1;
+    }
+    if (a.lexver < b.lexver) {
+      return 1;
+    }
+    return 0;
+  });
+};
+
+BuildsCacher.updateReleasesByTriplet = function (meta) {
+  for (let build of meta.packages) {
+    let target = build.target;
+
+    let triplet = `${target.os}-${target.arch}-${target.libc}`;
+    if (!meta.releasesByTriplet[triplet]) {
+      meta.releasesByTriplet[triplet] = {};
+    }
+
+    let buildsByRelease = meta.releasesByTriplet[triplet];
+    if (!buildsByRelease[build.version]) {
+      buildsByRelease[build.version] = [];
+    }
+
+    let packages = buildsByRelease[build.version];
+    packages.push(build);
+  }
+};

_webi/builds.js

@@ -20,7 +20,7 @@ bc.freshenRandomPackage(600 * 1000);
 Builds.init = async function () {
   bc.freshenRandomPackage(600 * 1000);
 
-  let dirs = await bc.getProjects();
+  let dirs = await bc.getProjectsByType();
   let projNames = Object.keys(dirs.valid);
 
   let parallel = 25;

_webi/classify-one.js

@@ -0,0 +1,90 @@
+'use strict';
+
+let Path = require('node:path');
+
+// let Builds = require('./builds.js');
+let BuildsCacher = require('./builds-cacher.js');
+let Triplet = require('./build-classifier/triplet.js');
+
+async function main() {
+  let projName = process.argv[2];
+  if (!projName) {
+    console.error(``);
+    console.error(`USAGE`);
+    console.error(``);
+    console.error(`    classify-one <project-name>`);
+    console.error(``);
+    console.error(`EXAMPLE`);
+    console.error(``);
+    console.error(`    classify-one caddy`);
+    console.error(``);
+    return;
+  }
+
+  let tsDate = new Date(0);
+  let meta = {
+    // version info
+    versions: [],
+    lexvers: [],
+    lexversMap: {},
+    // culled release assets
+    packages: [],
+    releasesByTriplet: {},
+    // target info
+    triplets: [],
+    oses: [],
+    arches: [],
+    libcs: [],
+    formats: [],
+    // TODO channels: [],
+  };
+
+  let installersDir = Path.join(__dirname, '..');
+  let Releases = require(`${installersDir}/${projName}/releases.js`);
+  if (!Releases.latest) {
+    Releases.latest = Releases;
+  }
+
+  let projInfo = await Releases.latest();
+
+  // let packages = await Builds.getPackage({ name: projName });
+  // console.log(packages);
+
+  let bc = {};
+  bc.ALL_TERMS = Triplet.TERMS_PRIMARY_MAP;
+  bc.orphanTerms = Object.assign({}, bc.ALL_TERMS);
+  bc.unknownTerms = {};
+  bc.usedTerms = {};
+  bc.formats = [];
+  bc._targetsByBuildIdCache = {};
+  bc._triplets = {};
+
+  let transformed = BuildsCacher.transformAndUpdate(
+    projName,
+    projInfo,
+    meta,
+    tsDate,
+    bc,
+  );
+
+  console.log(`[DEBUG] transformed`);
+  let sample = transformed.packages.slice(0, 20);
+  console.log('packages:', sample, ':packages');
+  console.log(
+    'releasesByTriplet:',
+    transformed.releasesByTriplet['linux-x86_64-none'][transformed.versions[0]],
+    ':releasesByTriplet',
+  );
+  console.log('versions:', transformed.versions, ':versions');
+  console.log('triplets:', transformed.triplets, ':triplets');
+  console.log('oses:', transformed.oses, ':oses');
+  console.log('arches:', transformed.arches, ':arches');
+  console.log('libcs:', transformed.libcs, ':libcs');
+  console.log('formats:', transformed.formats, ':formats');
+  console.log(Object.keys(transformed));
+}
+
+main().catch(function (err) {
+  console.error('Error:');
+  console.error(err);
+});

_webi/curl-pipe-bootstrap.tpl.ps1

@@ -6,7 +6,7 @@
 ############################################################
 New-Item -Path "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 New-Item -Path "$Env:USERPROFILE\.local\bin" -ItemType Directory -Force | Out-Null
-IF ($null -eq $Env:WEBI_HOST -or $Env:WEBI_HOST -eq "") { $Env:WEBI_HOST = "https://webinstall.dev" }
+if ($null -eq $Env:WEBI_HOST -or $Env:WEBI_HOST -eq "") { $Env:WEBI_HOST = "https://webinstall.dev" }
 curl.exe -s -A "windows" "$Env:WEBI_HOST/packages/webi/webi-pwsh.ps1" -o "$Env:USERPROFILE\.local\bin\webi-pwsh.ps1"
 Set-ExecutionPolicy -Scope Process Bypass
 & "$Env:USERPROFILE\.local\bin\webi-pwsh.ps1" "{{ exename }}"

_webi/curl-pipe-bootstrap.tpl.sh

@@ -267,12 +267,23 @@ webi_bootstrap() { (
 fn_checksum() {
     a_filepath="${1}"
 
-    cmd_shasum='sha1sum'
-    if command -v shasum > /dev/null; then
-        cmd_shasum='shasum'
+    if command -v sha1sum > /dev/null; then
+        sha1sum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
+        return 0
     fi
 
-    $cmd_shasum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
+    if command -v shasum > /dev/null; then
+        shasum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
+        return 0
+    fi
+
+    if command -v sha1 > /dev/null; then
+        sha1 "${a_filepath}" | cut -d'=' -f2 | cut -c 2-9
+        return 0
+    fi
+
+    echo >&2 "    warn: no sha1 sum program"
+    date '+%F %H:%M'
 }
 
 ##############################################

_webi/installers.js

@@ -16,7 +16,7 @@ var BAD_SH_RE = /[<>'"`$\\]/;
 Installers.renderBash = async function (
   pkgdir,
   rel,
-  { baseurl, pkg, tag, ver, os = '', arch = '', libc = '', formats, latest },
+  { baseurl, pkg, tag, ver, os = '', arch = '', libc = '', formats },
 ) {
   if (!Array.isArray(formats)) {
     formats = [];
@@ -93,17 +93,18 @@ Installers.renderBash = async function (
     ['WEBI_GIT_TAG', rel.git_tag], // TODO replace with branch
     ['WEBI_LTS', rel.lts],
     ['WEBI_CHANNEL', rel.channel],
-    ['WEBI_EXT', rel.ext.replace(/tar.*/, 'tar')],
+    ['WEBI_EXT', rel.ext],
     ['WEBI_FORMATS', formats.join(',')],
     ['WEBI_PKG_URL', rel.download],
     ['WEBI_PKG_PATHNAME', pkgFile],
     ['WEBI_PKG_FILE', pkgFile], // TODO replace with pathname
     ['PKG_NAME', pkg],
+    ['PKG_STABLE', rel.stable],
+    ['PKG_LATEST', rel.latest],
     ['PKG_OSES', (rel.oses || []).join(' ')],
     ['PKG_ARCHES', (rel.arches || []).join(' ')],
     ['PKG_LIBCS', (rel.libcs || []).join(' ')],
     ['PKG_FORMATS', (rel.formats || []).join(' ')],
-    ['PKG_LATEST', latest],
   ];
 
   for (let env of envReplacements) {

_webi/normalize.js

@@ -57,7 +57,7 @@ var archMap = {
   amd64:
     /(\b|_|amd|(dar)?win(dows)?|mac(os)?|linux|osx|x)64([_\-]?bit)?(\b|_)/i,
   //x86: /(86)(\b|_)/i,
-  x86: /(\b|_|amd|(dar)?win(dows)?|mac(os)?|linux|osx|x)(86|32)([_\-]?bit)(\b|_)/i,
+  x86: /(\b|_|amd|(dar)?win(dows)?|mac(os)?|linux|osx|x)(86|32|i?386)([_\-]?bit)?(\b|_)/i,
   ppc64le: /(\b|_)(ppc64le)/i,
   ppc64: /(\b|_)(ppc64)(\b|_)/i,
   s390x: /(\b|_)(s390x)/i,
@@ -211,7 +211,8 @@ function normalize(all) {
       // won't match:
       //   - v1.0beta
       //   - v1.0-beta1b
-      let isBetaRe = /(\b|_)(preview|rc|beta|alpha)(\d+)(\b|_)/;
+      let isBetaRe =
+        /(\b|_)(alpha|beta|dev|developer|prev|preview|rc)(\d+)(\b|_)/;
       let isBeta = isBetaRe.test(rel.name);
       if (isBeta) {
         rel.channel = 'beta';

_webi/package-install.tpl.ps1

@@ -45,15 +45,15 @@ $TDim = "${Esc}[2m"
 $TReset = "${Esc}[0m"
 
 function Invoke-DownloadUrl {
-    Param (
+    param (
         [string]$URL,
         [string]$Params,
         [string]$Path,
         [switch]$Force
     )
 
-    IF (Test-Path -Path "$Path") {
-        IF (-Not $Force.IsPresent) {
+    if (Test-Path -Path "$Path") {
+        if (-not $Force.IsPresent) {
             Write-Host "    ${TDim}Found${TReset} $Path"
             return
         }
@@ -65,7 +65,7 @@ function Invoke-DownloadUrl {
 
     Write-Host "    Downloading ${TDim}from${TReset}"
     Write-Host "      ${TDim}${URL}${TReset}"
-    IF ($Params.Length -ne 0) {
+    if ($Params.Length -ne 0) {
         Write-Host "        ?$Params"
         $URL = "${URL}?${Params}"
     }
@@ -80,18 +80,18 @@ function Get-UserAgent {
     # This is the canonical CPU arch when the process is emulated
     $my_arch = "$Env:PROCESSOR_ARCHITEW6432"
 
-    IF ($my_arch -eq $null -or $my_arch -eq "") {
+    if ($my_arch -eq $null -or $my_arch -eq "") {
         # This is the canonical CPU arch when the process is native
         $my_arch = "$Env:PROCESSOR_ARCHITECTURE"
     }
 
-    IF ($my_arch -eq "AMD64") {
+    if ($my_arch -eq "AMD64") {
         # Because PowerShell is sometimes AMD64 on Windows 10 ARM
         # See https://oofhours.com/2020/02/04/powershell-on-windows-10-arm64/
-        $my_os_arch = wmic os get osarchitecture
+        $my_os_arch = (Get-CimInstance -ClassName Win32_OperatingSystem).OSArchitecture
 
         # Using -clike because of the trailing newline
-        IF ($my_os_arch -clike "ARM 64*") {
+        if ($my_os_arch -clike "ARM 64*") {
             $my_arch = "ARM64"
         }
     }
@@ -124,7 +124,7 @@ function webi_path_add($pathname) {
             $exists_in_path = $true
         }
     }
-    if (-Not $exists_in_path) {
+    if (-not $exists_in_path) {
         $all_user_paths = "${pathname};${all_user_paths}".Trim(';')
         [Environment]::SetEnvironmentVariable("Path", $all_user_paths, "User")
         $null = Sync-EnvPath

_webi/package-install.tpl.sh

@@ -27,6 +27,7 @@ __bootstrap_webi() {
     #PKG_LIBCS=
     #PKG_FORMATS=
     #PKG_LATEST=
+    #PKG_STABLE=
     WEBI_PKG_DOWNLOAD=""
     WEBI_DOWNLOAD_DIR="${HOME}/Downloads"
     if command -v xdg-user-dir > /dev/null; then
@@ -36,7 +37,7 @@ __bootstrap_webi() {
         fi
     fi
 
-    WEBI_PKG_PATH="${WEBI_DOWNLOAD_DIR}/webi/${PKG_NAME:-error}/${WEBI_VERSION:-latest}"
+    WEBI_PKG_PATH="${WEBI_DOWNLOAD_DIR}/webi/${PKG_NAME:-error}/${WEBI_VERSION:-stable}"
 
     # get the special formatted version
     # (i.e. "go is go1.14" while node is "node v12.10.8")
@@ -127,7 +128,10 @@ __bootstrap_webi() {
             echo ""
             echo "    $(t_err "Error: no '${PKG_NAME:-"Unknown Package"}@${WEBI_TAG:-"Unknown Tag"}' release for '${WEBI_OS:-"Unknown OS"}' (${WEBI_LIBC:-"Unknown Libc"}) on '${WEBI_ARCH:-"Unknown CPU"}' as one of '${WEBI_FORMATS:-"Unknown File Type"}'")"
             echo ""
-            echo "        Latest Version: ${PKG_LATEST}"
+            echo "        Latest Stable: ${PKG_STABLE}"
+            if test "${PKG_LATEST}" != "${PKG_STABLE}"; then
+                echo "        Next Version: ${PKG_LATEST}"
+            fi
             echo "        CPUs: $PKG_ARCHES"
             echo "        OSes: $PKG_OSES"
             echo "        libcs: $PKG_LIBCS"
@@ -197,20 +201,32 @@ __bootstrap_webi() {
         my_dl_rel="$(
             fn_sub_home "${WEBI_PKG_PATH}/${WEBI_PKG_FILE}"
         )"
-        if [ "tar" = "$WEBI_EXT" ]; then
+        if test "$WEBI_EXT" = "tar.zst"; then
+            echo "    Extracting $(t_path "${my_dl_rel}")"
+            unzstd -c --keep "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" | tar xf -
+        elif test "$WEBI_EXT" = "tar.xz"; then
+            echo "    Extracting $(t_path "${my_dl_rel}")"
+            unxz -c -k "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" | tar xf -
+        elif test "$WEBI_EXT" = "tar.gz"; then
+            echo "    Extracting $(t_path "${my_dl_rel}")"
+            tar xzf "${WEBI_PKG_PATH}/$WEBI_PKG_FILE"
+        elif test "$WEBI_EXT" = "tar.bz2"; then
+            echo "    Extracting $(t_path "${my_dl_rel}")"
+            tar xjf "${WEBI_PKG_PATH}/$WEBI_PKG_FILE"
+        elif test "$WEBI_EXT" = "tar"; then
             echo "    Extracting $(t_path "${my_dl_rel}")"
             tar xf "${WEBI_PKG_PATH}/$WEBI_PKG_FILE"
-        elif [ "zip" = "$WEBI_EXT" ]; then
+        elif test "$WEBI_EXT" = "zip" || test "$WEBI_EXT" = "app.zip"; then
             echo "    Extracting $(t_path "${my_dl_rel}")"
             unzip "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" > __unzip__.log
-        elif [ "exe" = "$WEBI_EXT" ]; then
+        elif test "$WEBI_EXT" = "exe"; then
             echo "    Moving $(t_path "${my_dl_rel}")"
             echo "      to $(t_path "$(fn_sub_home "$(pwd)")")"
             mv "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" .
-        elif [ "git" = "$WEBI_EXT" ]; then
+        elif test "$WEBI_EXT" = "git"; then
             echo "    Moving $(t_path "${my_dl_rel}")"
             mv "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" .
-        elif [ "xz" = "$WEBI_EXT" ]; then
+        elif test "$WEBI_EXT" = "xz"; then
             echo "    Inflating $(t_path "${my_dl_rel}")"
             unxz -c "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" > "$(basename "$WEBI_PKG_FILE")"
         else
@@ -223,8 +239,6 @@ __bootstrap_webi() {
     webi_path_add() {
         my_path="${1}"
 
-        fn_envman_init
-
         # \v was chosen as it is extremely unlikely for a filename
         # \1 could be an even better choice, but needs more testing.
         # (currently tested working on: linux & mac)
@@ -387,7 +401,6 @@ __bootstrap_webi() {
     export _webi_tmp="${_webi_tmp:-"$HOME/.local/opt/webi-tmp.d"}"
 
     mkdir -p "${WEBI_PKG_PATH}"
-    mkdir -p "$HOME/.local/bin"
     mkdir -p "$HOME/.local/opt"
 
     if test -e ~/.local/bin; then
@@ -396,6 +409,7 @@ __bootstrap_webi() {
         echo "    Creating$(t_path ' ~/.local/bin')"
         mkdir -p "$HOME/.local/bin"
     fi
+    fn_envman_init
 
     ##
     ##
@@ -776,12 +790,23 @@ webi_upgrade() { (
 fn_checksum() {
     a_filepath="${1}"
 
-    cmd_shasum='sha1sum'
-    if command -v shasum > /dev/null; then
-        cmd_shasum='shasum'
+    if command -v sha1sum > /dev/null; then
+        sha1sum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
+        return 0
     fi
 
-    $cmd_shasum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
+    if command -v shasum > /dev/null; then
+        shasum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
+        return 0
+    fi
+
+    if command -v sha1 > /dev/null; then
+        sha1 "${a_filepath}" | cut -d'=' -f2 | cut -c 2-9
+        return 0
+    fi
+
+    echo >&2 "    warn: no sha1 sum program"
+    date '+%F %H:%M'
 }
 
 ##############################################

_webi/serve-installer.js

@@ -70,18 +70,20 @@ InstallerServer.helper = async function ({
 
   console.log(`dbg: Get Project Installer Type for '${projectName}':`);
   let proj = await Builds.getProjectType(projectName);
-  console.log(proj);
+  if (proj.type === 'alias') {
+    console.log(`dbg: alias`, proj);
+    projectName = proj.detail;
+    proj = await Builds.getProjectType(projectName); // an alias should never resolve to an alias
+  }
+  console.log(`dbg: proj`, proj);
 
-  let validTypes = ['alias', 'selfhosted', 'valid'];
+  let validTypes = ['selfhosted', 'valid'];
   if (!validTypes.includes(proj.type)) {
     let msg = `'${projectName}' doesn't have an installer: '${proj.type}': '${proj.detail}'`;
     let err = new Error(msg);
     err.code = 'ENOENT';
     throw err;
   }
-  if (proj.type === 'alias') {
-    projectName = proj.detail;
-  }
 
   let tmplParams = {
     pkg: projectName,
@@ -120,8 +122,7 @@ InstallerServer.helper = async function ({
     name: projectName,
     date: new Date(),
   });
-  let latest = projInfo.versions[0];
-  Object.assign(tmplParams, { latest });
+  let latestVersions = Builds.enumerateLatestVersions(projInfo);
   //console.log('projInfo', projInfo);
 
   let buildTargetInfo = {
@@ -130,12 +131,27 @@ InstallerServer.helper = async function ({
     arches: projInfo.arches,
     libcs: projInfo.libcs,
     formats: projInfo.formats,
+    latest: latestVersions.latest,
+    stable: latestVersions.stable,
   };
 
+  // TODO .findMatchingPackages() should probably account for this
   let hasOs = projInfo.oses.includes(hostTarget.os);
+  let maybePosix = !hasOs && hostTarget.os !== 'windows';
+  if (maybePosix) {
+    let posixes = ['posix_2017', 'posix_2024'];
+    for (let posixYear of posixes) {
+      let hasPosix = projInfo.oses.includes(posixYear);
+      if (hasPosix) {
+        hasOs = true;
+        break;
+      }
+    }
+  }
   if (!hasOs) {
     hasOs = projInfo.oses.includes('ANYOS');
   }
+
   if (!hasOs) {
     let pkg1 = Object.assign(buildTargetInfo, errPackage);
     return [pkg1, tmplParams];

_webi/transform-releases.js

@@ -3,7 +3,6 @@
 var Releases = module.exports;
 
 var path = require('path');
-var request = require('@root/request');
 var _normalize = require('./normalize.js');
 
 var cache = {};
@@ -17,14 +16,18 @@ let installerDir = path.join(__dirname, '..');
 Releases.get = async function (pkgdir) {
   let get;
   try {
-    get = require(path.join(pkgdir, 'releases.js'));
+    get = require(`${pkgdir}/releases.js`);
+    // TODO update all releases files with module.exports.xxxx = 'foo';
+    if (!get.latest) {
+      get.latest = get;
+    }
   } catch (e) {
     let err = new Error('no releases.js for', pkgdir.split(/[\/\\]+/).pop());
     err.code = 'E_NO_RELEASE';
     throw err;
   }
 
-  let all = await get(request);
+  let all = await get.latest();
 
   return _normalize(all);
 };

aliasman/README.md

@@ -72,8 +72,8 @@ gc "feat: new feature"
 
 ### Common aliases
 
-Use *alias*es to make other tools you find around webi even _more_ convenient
-⚡️ (and powerful 💪).
+Use *alias*es to make other tools you find around webi even _more_ convenient ⚡️
+(and powerful 💪).
 
 ```sh
 aliasman curl 'curlie'

aliasman/releases.js

@@ -1,29 +1,21 @@
 'use strict';
 
-var githubSource = require('../_common/github-source.js');
-var owner = 'BeyondCodeBootcamp';
-var repo = 'aliasman';
+let Releases = module.exports;
 
-module.exports = function (request) {
-  let arches = [
-    'amd64',
-    'arm64',
-    'armv6l',
-    'armv7l',
-    'ppc64le',
-    'ppc64',
-    's390x',
-    'x86',
-  ];
-  let oses = ['freebsd', 'linux', 'macos', 'posix'];
-  return githubSource(request, owner, repo, oses, arches).then(function (all) {
-    all._names = ['aliasman', 'legacy'];
-    return all;
-  });
+let GitHubSource = require('../_common/github-source.js');
+let owner = 'BeyondCodeBootcamp';
+let repo = 'aliasman';
+
+Releases.latest = async function () {
+  let all = await GitHubSource.getDistributables({ owner, repo });
+  for (let pkg of all.releases) {
+    pkg.os = 'posix_2017';
+  }
+  return all;
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  Releases.latest().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all, null, 2));
   });

arc/install.ps1

@@ -19,13 +19,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading archiver from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-IF (!(Test-Path -Path "$pkg_src_cmd")) {
+if (!(Test-Path -Path "$pkg_src_cmd")) {
     Write-Output "Installing archiver"
 
     # TODO: create package-specific temp directory

arc/releases.js

@@ -4,15 +4,15 @@ var github = require('../_common/github.js');
 var owner = 'mholt';
 var repo = 'archiver';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     all._names = ['archiver', 'arc'];
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 5 for demonstration
     all.releases = all.releases.slice(0, 5);

archiver/install.ps1

@@ -1,5 +1,5 @@
 #!/bin/pwsh
 
 Write-Output "'archiver@$Env:WEBI_TAG' is an alias for 'arc@$Env:WEBI_VERSION'"
-IF ($null -eq $Env:WEBI_HOST -or $Env:WEBI_HOST -eq "") { $Env:WEBI_HOST = "https://webinstall.dev" }
+if ($null -eq $Env:WEBI_HOST -or $Env:WEBI_HOST -eq "") { $Env:WEBI_HOST = "https://webinstall.dev" }
 curl.exe -A MS -fsSL "$Env:WEBI_HOST/arc@$Env:WEBI_VERSION" | powershell

atomicparsley/install.ps1

@@ -20,7 +20,7 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Checking for (or Installing) MSVC Runtime..."
     & "$Env:USERPROFILE\.local\bin\webi-pwsh.ps1" vcruntime
 
@@ -29,7 +29,7 @@ IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-IF (!(Test-Path -Path "$pkg_src_cmd")) {
+if (!(Test-Path -Path "$pkg_src_cmd")) {
     Write-Output "Installing AtomicParsley"
 
     # TODO: create package-specific temp directory

atomicparsley/releases.js

@@ -32,8 +32,8 @@ let targets = {
   },
 };
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     for (let rel of all.releases) {
       let windows32 = rel.name.includes('WindowsX86.');
       if (windows32) {
@@ -71,7 +71,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all));
     //console.info(JSON.stringify(all, null, 2));

awless/install.ps1

@@ -20,13 +20,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading awless from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-IF (!(Test-Path -Path "$pkg_src_cmd")) {
+if (!(Test-Path -Path "$pkg_src_cmd")) {
     Write-Output "Installing awless"
 
     # TODO: create package-specific temp directory

awless/releases.js

@@ -4,8 +4,8 @@ var github = require('../_common/github.js');
 var owner = 'wallix';
 var repo = 'awless';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     // remove checksums and .deb
     all.releases = all.releases.filter(function (rel) {
       return !/(\.txt)|(\.deb)$/i.test(rel.name);
@@ -15,7 +15,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all));
   });

bat/install.ps1

@@ -3,7 +3,7 @@
 $VERNAME = "$Env:PKG_NAME-v$Env:WEBI_VERSION.exe"
 $EXENAME = "$Env:PKG_NAME.exe"
 # Fetch archive
-IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading $Env:PKG_NAME from $Env:WEBI_PKG_URL to $Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE.part"
     & Move-Item "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE.part" "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
@@ -11,11 +11,11 @@ IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
 
 # Fetch MSVC Runtime
 Write-Output "Checking for MSVC Runtime..."
-IF (-not (Test-Path "\Windows\System32\vcruntime140.dll")) {
+if (-not (Test-Path "\Windows\System32\vcruntime140.dll")) {
     & "$Env:USERPROFILE\.local\bin\webi-pwsh.ps1" vcruntime
 }
 
-IF (!(Test-Path -Path "$Env:USERPROFILE\.local\bin\$VERNAME")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\.local\bin\$VERNAME")) {
     Write-Output "Installing $Env:PKG_NAME"
     # TODO: temp directory
 

bat/releases.js

@@ -4,14 +4,14 @@ var github = require('../_common/github.js');
 var owner = 'sharkdp';
 var repo = 'bat';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     all.releases = all.releases.slice(0, 10);
     //console.info(JSON.stringify(all));

brew/brew-update-service-install

@@ -6,15 +6,23 @@ main() { (
     sed '1,/^#~\/.local\/bin\/brew-updater/d' "${0}" > ~/.local/bin/brew-update-hourly
     chmod a+x ~/.local/bin/brew-update-hourly
 
-    env PATH="$PATH" serviceman add --user \
+    echo "Checking for serviceman..."
+    ~/.local/bin/webi serviceman
+    if ! command -v serviceman > /dev/null; then
+        export PATH="$HOME/.local/bin:$PATH"
+    fi
+    serviceman --version
+
+    serviceman add --agent \
         --workdir ~/.local/opt/brew/ \
         --name sh.brew.updater -- \
         ~/.local/bin/brew-update-hourly
 ); }
 
-if main; then
-    exit 0
+if ! main; then
+    exit 1
 fi
+exit 0
 
 #~/.local/bin/brew-updater
 #!/bin/sh

bun/README.md

@@ -132,9 +132,7 @@ file)
    ```
 3. Add your project to the system launcher, running as the current user
    ```sh
-   sudo env PATH="$PATH" \
-       serviceman add --path="$PATH" --system \
-           --username "$(whoami)" --name my-project -- \
+   serviceman add --name 'my-project' --daemon -- \
        bun run ./my-project.js
    ```
 4. Restart the logging service
@@ -155,6 +153,6 @@ For **macOS**:
    ```
 3. Add your project to the system launcher, running as the current user
    ```sh
-   serviceman add --path="$PATH" --user --name my-project -- \
+   serviceman add --agent --name 'my-project' -- \
        bun run ./my-project.js
    ```

bun/releases.js

@@ -4,23 +4,45 @@ var github = require('../_common/github.js');
 var owner = 'oven-sh';
 var repo = 'bun';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
+    // collect baseline asset names so we can prefer them over non-baseline
+    // (baseline builds avoid SIGILL on older/container CPUs)
+    let baselineNames = new Set();
+    all.releases.forEach(function (r) {
+      if (r.name.includes('-baseline')) {
+        baselineNames.add(r.name.replace('-baseline', ''));
+      }
+    });
+
     all.releases = all.releases
       .filter(function (r) {
-        let isDebug = r.name.includes('-profile');
-        if (isDebug) {
+        if (r.name.includes('-profile')) {
           return false;
         }
 
-        let isAncient = r.name.includes('-baseline');
-        if (isAncient) {
+        if (r.name.endsWith('.txt') || r.name.endsWith('.asc')) {
           return false;
         }
 
+        // drop the non-baseline asset when a baseline twin exists
+        if (!r.name.includes('-baseline') && baselineNames.has(r.name)) {
+          return false;
+        }
+
+        let isMusl = r.name.includes('-musl');
+        if (isMusl) {
+          r._musl = true;
+          r.libc = 'musl';
+        } else if (r.name.includes('-linux-')) {
+          r.libc = 'gnu';
+        }
+
         return true;
       })
       .map(function (r) {
+        // bun-linux-x64-baseline.zip => bun-linux-x64
+        r.name = r.name.replace('-baseline', '').replace(/\.zip$/, '');
         // bun-v0.5.1 => v0.5.1
         r.version = r.version.replace(/bun-/g, '');
         return r;
@@ -30,7 +52,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 5 for demonstration
     all.releases = all.releases.slice(0, 5);

caddy/README.md

@@ -819,10 +819,10 @@ To avoid the nitty-gritty details of `launchd` plist files, you can use
 2. Use Serviceman to create a _launchd_ plist file
 
    ```sh
-   my_username="$( id -u -n )"
+   my_username="$(id -u -n)"
 
-   serviceman add --user --name caddy -- \
-       caddy run --config ./Caddyfile --envfile ~/.config/caddy/env
+   serviceman add --agent --name 'caddy' --workdir ./ -- \
+       caddy run --envfile ~/.config/caddy/env --config ./Caddyfile --adapter caddyfile
    ```
 
    (this will create `~/Library/LaunchAgents/caddy.plist`)
@@ -837,8 +837,8 @@ This process creates a _User-Level_ service in `~/Library/LaunchAgents`. To
 create a _System-Level_ service in `/Library/LaunchDaemons/` instead:
 
 ```sh
-sudo serviceman add --system --name caddy -- \
-   caddy run --config ./Caddyfile --envfile ~/.config/caddy/env
+serviceman add --name 'caddy' --workdir ./ --daemon -- \
+   caddy run --envfile ~/.config/caddy/env --config ./Caddyfile --adapter caddyfile
 ```
 
 ### How to run Caddy as a Windows Service
@@ -856,7 +856,7 @@ sudo serviceman add --system --name caddy -- \
 3. Create a **Startup Registry Entry** with Serviceman.
    ```sh
    serviceman.exe add --name caddy -- \
-       caddy run --config ./Caddyfile --envfile ~/.config/caddy/env
+       caddy run --envfile ~/.config/caddy/env --config ./Caddyfile --adapter caddyfile
    ```
 4. You can manage the service directly with Serviceman. For example:
    ```sh
@@ -901,10 +901,8 @@ See the notes below to run as a **User Service** or use the JSON Config.
    ```
 4. Use Serviceman to create a _systemd_ config file.
    ```sh
-   my_username="$( id -u -n )"
-   sudo env PATH="$PATH" \
-       serviceman add --system --username "${my_username}" --name caddy -- \
-           caddy run --config ./Caddyfile --envfile ~/.config/caddy/env
+   serviceman add --name 'caddy' --daemon -- \
+       caddy run --envfile ~/.config/caddy/env --config ./Caddyfile --adapter caddyfile
    ```
    (this will create `/etc/systemd/system/caddy.service`)
 5. Manage the service with `systemctl` and `journalctl`:
@@ -915,10 +913,10 @@ See the notes below to run as a **User Service** or use the JSON Config.
 
 To create a **User Service** instead:
 
-- don't use `sudo`, but do use `--user` when running `serviceman`:
+- use `--agent` when running `serviceman`:
   ```sh
-  serviceman add --user --name caddy -- \
-     caddy run --config ./Caddyfile --envfile ~/.config/caddy/env
+  serviceman add --agent --name caddy -- \
+      caddy run --envfile ~/.config/caddy/env --config ./Caddyfile --adapter caddyfile
   ```
   (this will create `~/.config/systemd/user/`)
 - user the `--user` flag to manage services and logs:
@@ -1183,7 +1181,8 @@ To prevent search engine and browser confusion
 - _DO NOT_ prevent crawling via `robots.txt` \
   (counter-intuitive, but pages _must_ be crawled for links to _NOT_ be indexed)
 - _all_ domains using public TLS certs _will_ be indexed by default \
-  (they are all linked to and crawled from various Certificate Transparency reports)
+  (they are all linked to and crawled from various Certificate Transparency
+  reports)
 - follow these guidelines even if the dev sites use HTTP Basic Auth
 
 ```Caddyfile
@@ -1363,19 +1362,13 @@ See also: <https://caddyserver.com/docs/running>
 2. Generate the `service` file: \
    - JSON Config
      ```sh
-     my_app_user="$( id -u -n )"
-     sudo env PATH="${PATH}" \
-         serviceman add --system --cap-net-bind \
-             --username "${my_app_user}" --name caddy -- \
-             caddy run --resume --envfile ./caddy.env
+     serviceman add --name 'caddy' --daemon -- \
+         caddy run --resume --envfile ./caddy.env
      ```
    - Caddyfile
      ```sh
-     my_app_user="$( id -u -n )"
-     sudo env PATH="${PATH}" \
-         serviceman add --system --cap-net-bind \
-             --username "${my_app_user}" --name caddy -- \
-             caddy run --config ./Caddyfile --envfile ./caddy.env
+     serviceman add --name 'caddy' --daemon -- \
+         caddy run --config ./Caddyfile --envfile ./caddy.env
      ```
 3. Reload `systemd` config files, the logging service (it may not be started on
    a new VPS), and caddy

caddy/install.ps1

@@ -20,13 +20,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading caddy from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-IF (!(Test-Path -Path "$pkg_src_cmd")) {
+if (!(Test-Path -Path "$pkg_src_cmd")) {
     Write-Output "Installing caddy"
 
     # TODO: create package-specific temp directory

caddy/releases.js

@@ -4,8 +4,8 @@ var github = require('../_common/github.js');
 var owner = 'caddyserver';
 var repo = 'caddy';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     // remove checksums and .deb
     all.releases = all.releases.filter(function (rel) {
       let isOneOffAsset = rel.download.includes('buildable-artifact');
@@ -20,7 +20,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all));
   });

chromedriver/install.ps1

@@ -19,13 +19,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading chromedriver from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-IF (!(Test-Path -Path "$pkg_src_cmd")) {
+if (!(Test-Path -Path "$pkg_src_cmd")) {
     Write-Output "Installing chromedriver"
 
     # TODO: create package-specific temp directory

chromedriver/releases.js

@@ -1,7 +1,9 @@
 'use strict';
 
+let Fetcher = require('../_common/fetcher.js');
+
 // See <https://googlechromelabs.github.io/chrome-for-testing/>
-var releaseApiUrl =
+const releaseApiUrl =
   'https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json';
 
 // {
@@ -40,14 +42,24 @@ var releaseApiUrl =
 //   ]
 // }
 
-module.exports = async function (request) {
-  let resp = await request({
-    url: releaseApiUrl,
-    json: true,
-  });
+module.exports = async function () {
+  let resp;
+  try {
+    resp = await Fetcher.fetch(releaseApiUrl, {
+      headers: { Accept: 'application/json' },
+    });
+  } catch (e) {
+    /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error
+    let err = e;
+    if (err.code === 'E_FETCH_RELEASES') {
+      err.message = `failed to fetch 'chromedriver' release data: ${err.response.status} ${err.response.body}`;
+    }
+    throw e;
+  }
+  let data = JSON.parse(resp.body);
 
   let builds = [];
-  for (let release of resp.body.versions) {
+  for (let release of data.versions) {
     if (!release.downloads.chromedriver) {
       continue;
     }
@@ -58,7 +70,7 @@ module.exports = async function (request) {
         version: version,
         download: asset.url,
         // I' not sure that this is actually statically built but it
-        // seems to be and at worst we'll just get bug reports for Apline
+        // seems to be and at worst we'll just get bug reports for Alpine
         libc: 'none',
       };
 
@@ -75,10 +87,15 @@ module.exports = async function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    // just select the latest 5 for demonstration
-    all.releases = all.releases.slice(-20);
-    console.info(JSON.stringify(all, null, 2));
-  });
+  module
+    .exports()
+    .then(function (all) {
+      all = require('../_webi/normalize.js')(all);
+      // just select the latest 20 for demonstration
+      all.releases = all.releases.slice(-20);
+      console.info(JSON.stringify(all, null, 2));
+    })
+    .catch(function (err) {
+      console.error('Error:', err);
+    });
 }

cilium/README.md

@@ -0,0 +1,47 @@
+---
+title: cilium
+homepage: https://github.com/cilium/cilium-cli
+tagline: |
+  cilium: manage & troubleshoot Kubernetes clusters running Cilium
+---
+
+To update or switch versions, run `webi cilium@stable` (or `@v2`, `@beta`,etc).
+
+### Files
+
+These are the files / directories that are created and/or modified with this
+install:
+
+```text
+~/.config/envman/PATH.env
+~/.local/bin/cilium
+~/.local/opt/cilium/
+```
+
+## Cheat Sheet
+
+> Cilium is an open source, cloud native solution for providing, securing, and
+> observing network connectivity between workloads, fueled by the revolutionary
+> Kernel technology eBPF.
+
+Quick Start User Guide:
+
+<https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/#k8s-install-quick>
+
+To install the default version of the Cilium image:
+
+```sh
+cilium install
+```
+
+To upgrade to a specific version of the Cilium image:
+
+```sh
+cilium upgrade --version v1.15.3
+```
+
+To check the status of the current Cilium deployment:
+
+```sh
+cilium status
+```

cilium/install.ps1

@@ -0,0 +1,45 @@
+#!/usr/bin/env pwsh
+
+##################
+# Install cilium #
+##################
+
+$pkg_cmd_name = "cilium"
+
+$pkg_dst_cmd = "$Env:USERPROFILE\.local\bin\cilium.exe"
+$pkg_dst = "$pkg_dst_cmd"
+
+$pkg_src_cmd = "$Env:USERPROFILE\.local\opt\cilium-v$Env:WEBI_VERSION\bin\cilium.exe"
+$pkg_src_bin = "$Env:USERPROFILE\.local\opt\cilium-v$Env:WEBI_VERSION\bin"
+$pkg_src_dir = "$Env:USERPROFILE\.local\opt\cilium-v$Env:WEBI_VERSION"
+$pkg_src = "$pkg_src_cmd"
+
+New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
+$pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
+
+if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+    Write-Output "Downloading cilium from $Env:WEBI_PKG_URL to $pkg_download"
+    & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
+    & Move-Item "$pkg_download.part" "$pkg_download"
+}
+
+if (!(Test-Path -Path "$pkg_src_cmd")) {
+    Write-Output "Installing cilium"
+    Push-Location .local\tmp
+
+    Remove-Item -Path ".\cilium-v*" -Recurse -ErrorAction Ignore
+    Remove-Item -Path ".\cilium.exe" -Recurse -ErrorAction Ignore
+
+    Write-Output "Unpacking $pkg_download"
+    & tar xf "$pkg_download"
+
+    Write-Output "Install Location: $pkg_src_cmd"
+    New-Item "$pkg_src_bin" -ItemType Directory -Force
+    Move-Item -Path ".\cilium-*\cilium.exe" -Destination "$pkg_src_bin"
+
+    Pop-Location
+}
+
+Write-Output "Copying into '$pkg_dst_cmd' from '$pkg_src_cmd'"
+Remove-Item -Path "$pkg_dst_cmd" -Recurse -ErrorAction Ignore
+Copy-Item -Path "$pkg_src" -Destination "$pkg_dst" -Recurse

cilium/install.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+__init_cilium() {
+    set -e
+    set -u
+
+    ##################
+    # Install cilium #
+    ##################
+
+    pkg_cmd_name="cilium"
+
+    pkg_dst_cmd="$HOME/.local/bin/cilium"
+    pkg_dst="$pkg_dst_cmd"
+
+    pkg_src_cmd="$HOME/.local/opt/cilium-v$WEBI_VERSION/bin/cilium"
+    pkg_src_dir="$HOME/.local/opt/cilium-v$WEBI_VERSION"
+    pkg_src="$pkg_src_cmd"
+
+    WEBI_SINGLE=true
+
+    # pkg_install must be defined by every package
+    pkg_install() {
+        # ~/.local/opt/cilium-v0.16.16/bin
+        mkdir -p "$(dirname "${pkg_src_cmd}")"
+
+        # mv ./hugo ~/.local/opt/cilium-v0.16.16/bin/
+        mv ./cilium "${pkg_src_cmd}"
+    }
+
+    pkg_get_current_version() {
+        cilium version 2> /dev/null |
+            head -n 1 |
+            cut -d ' ' -f 2
+    }
+
+}
+
+__init_cilium

cilium/releases.js

@@ -0,0 +1,21 @@
+'use strict';
+
+var github = require('../_common/github.js');
+var owner = 'cilium';
+var repo = 'cilium-cli';
+
+module.exports = async function () {
+  let all = await github(null, owner, repo);
+  return all;
+};
+
+if (module === require.main) {
+  (async function () {
+    let normalize = require('../_webi/normalize.js');
+    let all = await module.exports();
+    all = normalize(all);
+    // just select the first 5 for demonstration
+    all.releases = all.releases.slice(0, 5);
+    console.info(JSON.stringify(all, null, 2));
+  })();
+}

cmake/install.ps1

@@ -20,13 +20,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-IF (!(Test-Path -Path "$pkg_download")) {
+if (!(Test-Path -Path "$pkg_download")) {
     Write-Output "Downloading cmake from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-IF (!(Test-Path -Path "$pkg_src_dir")) {
+if (!(Test-Path -Path "$pkg_src_dir")) {
     Write-Output "Installing cmake"
 
     # TODO: create package-specific temp directory

cmake/releases.js

@@ -4,8 +4,8 @@ var github = require('../_common/github.js');
 var owner = 'Kitware';
 var repo = 'CMake';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     for (let rel of all.releases) {
       if (rel.version.startsWith('v')) {
         rel._version = rel.version.slice(1);
@@ -44,7 +44,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 5 for demonstration
     all.releases = all.releases.slice(0, 5);

commandlinetools/README.md

@@ -0,0 +1,55 @@
+---
+title: XCode Command Line Tools
+homepage: https://webinstall.dev/commandlinetools
+tagline: |
+  The XCode Command Line Tools include git, swift, make, clang, and other developer tools
+---
+
+## Cheat Sheet
+
+> The developer tools provided by Apple for macOS.
+
+- git
+- swift
+- make
+- clang
+- etc
+
+This is also part of [webi-essentials](../webi-essentials/).
+
+## Table of Contents
+
+- Files
+- Manual Install
+  - macOS
+  - Linux
+  - Alpine
+  - Windows
+
+### Files
+
+These are the files / directories that are created and/or modified with this
+install:
+
+```sh
+/Library/Developer/CommandLineTools/
+```
+
+### How to Install Manually
+
+It's very easy to start the installer:
+
+```sh
+xcode-select --install
+```
+
+The trick is to also have a mechanism to know when it has finished:
+
+```sh
+while ! test -x /Library/Developer/CommandLineTools/usr/bin/git ||
+    ! test -x /Library/Developer/CommandLineTools/usr/bin/make; do
+    sleep 0.25
+done
+
+echo "Command Line Tools Installed"
+```

commandlinetools/install.sh

@@ -0,0 +1,62 @@
+#!/bin/sh
+set -e
+set -u
+
+fn_install_xcode_commandlinetools() { (
+    b_os="$(uname -s)"
+    if test "${b_os}" != 'Darwin'; then
+        echo >&2 'XCode Command Line Tools are for macOS only'
+        return 1
+    fi
+
+    # streamline the output to be pretty
+    fn_check_pkg '/Library/Developer/CommandLineTools/usr/bin/clang' 'clang'
+    fn_check_pkg '/Library/Developer/CommandLineTools/usr/bin/git' 'git'
+    fn_check_pkg '/Library/Developer/CommandLineTools/usr/bin/make' 'make'
+    echo >&2 ""
+
+    # git
+    if xcode-select -p > /dev/null 2> /dev/null; then
+        echo ""
+        return 0
+    fi
+
+    cmd_xcode_cli_tools_install="xcode-select --install"
+    echo "    Running $(t_cmd "${cmd_xcode_cli_tools_install}")"
+    $cmd_xcode_cli_tools_install 2> /dev/null
+    echo ""
+    echo ">>> $(t_attn 'ACTION REQUIRED') <<<"
+    echo ""
+    echo "        $(t_attn "Click") '$(t_bold 'Install')' $(t_attn "in the pop-up")"
+    echo "        (it may appear $(t_em 'under') this window)"
+    echo ""
+    echo "^^^ $(t_attn 'ACTION REQUIRED') ^^^"
+    echo ""
+    printf "    waiting %s to finish installing Command Line Developer Tools ..." "$(t_em 'for you')"
+    while ! test -x /Library/Developer/CommandLineTools/usr/bin/git ||
+        ! test -x /Library/Developer/CommandLineTools/usr/bin/make; do
+        sleep 0.25
+    done
+    echo " $(t_info 'OK')"
+    echo "    Installed to $(t_path '/Library/Developer/CommandLineTools/')"
+    sleep 1
+); }
+
+fn_check_pkg() { (
+    a_pkg="${1}"
+    a_pkgname="${2:-$a_pkg}"
+
+    printf >&2 '    %s %s %s' \
+        "$(t_dim "Checking for")" \
+        "$(t_pkg "${a_pkgname}")" \
+        "$(t_dim "...")"
+
+    if command -v "${a_pkg}" > /dev/null; then
+        echo >&2 " $(t_dim 'OK')"
+        return 0
+    fi
+
+    echo >&2 ' missing'
+); }
+
+fn_install_xcode_commandlinetools

comrak/install.ps1

@@ -3,13 +3,13 @@
 $VERNAME = "$Env:PKG_NAME-v$Env:WEBI_VERSION.exe"
 $EXENAME = "$Env:PKG_NAME.exe"
 # Fetch archive
-IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading $Env:PKG_NAME from $Env:WEBI_PKG_URL to $Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE.part"
     & Move-Item "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE.part" "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 }
 
-IF (!(Test-Path -Path "$Env:USERPROFILE\.local\opt\$Env:PKG_NAME-v$Env:WEBI_VERSION\bin\$VERNAME")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\.local\opt\$Env:PKG_NAME-v$Env:WEBI_VERSION\bin\$VERNAME")) {
     Write-Output "Installing $Env:PKG_NAME"
     # TODO: temp directory
 

comrak/releases.js

@@ -6,8 +6,8 @@ var repo = 'comrak';
 
 var ODDITIES = ['-musleabihf.1-'];
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     let builds = [];
 
     loopBuilds: for (let build of all.releases) {
@@ -31,7 +31,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     all.releases = all.releases.slice(0, 10);
     //console.info(JSON.stringify(all));

crabz/install.ps1

@@ -20,13 +20,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-IF (!(Test-Path -Path "$pkg_download")) {
+if (!(Test-Path -Path "$pkg_download")) {
     Write-Output "Downloading crabz from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-IF (!(Test-Path -Path "$pkg_src_cmd")) {
+if (!(Test-Path -Path "$pkg_src_cmd")) {
     Write-Output "Installing crabz"
 
     # TODO: create package-specific temp directory

crabz/releases.js

@@ -4,8 +4,8 @@ var github = require('../_common/github.js');
 var owner = 'sstadick';
 var repo = 'crabz';
 
-module.exports = async function (request) {
-  let all = await github(request, owner, repo);
+module.exports = async function () {
+  let all = await github(null, owner, repo);
 
   let releases = [];
   for (let rel of all.releases) {
@@ -22,7 +22,7 @@ module.exports = async function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 5 for demonstration
     all.releases = all.releases.slice(0, 5);

curlie/install.ps1

@@ -3,13 +3,13 @@
 $VERNAME = "$Env:PKG_NAME-v$Env:WEBI_VERSION.exe"
 $EXENAME = "$Env:PKG_NAME.exe"
 # Fetch archive
-IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading $Env:PKG_NAME from $Env:WEBI_PKG_URL to $Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE.part"
     & Move-Item "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE.part" "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 }
 
-IF (!(Test-Path -Path "$Env:USERPROFILE\.local\bin\$VERNAME")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\.local\bin\$VERNAME")) {
     Write-Output "Installing $Env:PKG_NAME"
     # TODO: temp directory
 

curlie/releases.js

@@ -4,15 +4,15 @@ var github = require('../_common/github.js');
 var owner = 'rs';
 var repo = 'curlie';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     all._names = ['curlie', 'curl-httpie'];
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     all.releases = all.releases.slice(0, 10);
     //console.info(JSON.stringify(all));

dashcore-utils/README.md

@@ -100,8 +100,7 @@ mkdir -p ~/.dashcore/wallets/
 mkdir -p /mnt/slc1_vol_100g/dashcore/_data
 mkdir -p /mnt/slc1_vol_100g/dashcore/_caches
 
-sudo env PATH="$PATH" serviceman add \
-        --system --user "$my_user" --path "$PATH" --name dashd --force -- \
+serviceman add --name 'dashd' --daemon -- \
     dashd \
         -usehd \
         -conf="$HOME/.dashcore/dash.conf" \

dashcore-utils/dash.example.conf

@@ -1,7 +1,14 @@
+# for exposing RPCs, building APIs
 txindex=1
 addressindex=1
 timestampindex=1
 spentindex=1
+# listen as server (explicit default)
+listen=1
+# because its already run as a service (systemd, openrc)
+daemon=0
+# for evonodes
+#server=1
 
 [main]
 rpcuser=RPCUSER_MAIN

dashcore-utils/dashd-hd-service-install

@@ -17,7 +17,7 @@ fn_usage() { (
 ); }
 
 fn_datadir_help() { (
-    my_vol="${1:-}"
+    my_vol="${1}"
     my_user="$(
         id -n -u
     )"
@@ -25,6 +25,7 @@ fn_datadir_help() { (
         id -n -g
     )"
 
+    my_mount="$(dirname "${my_vol}")"
     echo >&2 ""
     echo >&2 "ERROR"
     echo >&2 "        '${my_vol}' is not writable"
@@ -32,8 +33,8 @@ fn_datadir_help() { (
     echo >&2 "SOLUTION"
     echo >&2 "        1. Mount a large (50gb+) volume"
     echo >&2 ""
-    echo >&2 "              sudo mkdir -p /mnt/EXAMPLE"
-    echo >&2 "              sudo mount /dev/sdx1 /mnt/EXAMPLE"
+    echo >&2 "              sudo mkdir -p ${my_mount}"
+    echo >&2 "              sudo mount /dev/sdx1 ${my_mount}"
     echo >&2 ""
     echo >&2 "        2. Create a 'dashcore' inside of it"
     echo >&2 ""
@@ -83,20 +84,8 @@ fn_srv_install() { (
         my_name="dashd-${my_netname}"
     fi
 
-    my_system_args=""
-    my_kernel="$(
-        uname -s
-    )"
-    if test "Darwin" != "${my_kernel}"; then
-        my_user="$(
-            id -u -n
-        )"
-        my_system_args="--system --username ${my_user}"
-    fi
-
     # shellcheck disable=SC2016,SC1090
-    echo 'sudo env PATH="$PATH"' \
-        "serviceman add ${my_system_args} --path \"\$PATH\" --name \"${my_name}\" --force --" \
+    echo "serviceman add --name \"${my_name}\" --" \
         "dashd " \
         "${my_net_flag}" \
         -usehd \
@@ -106,16 +95,16 @@ fn_srv_install() { (
         "-datadir=\"${my_datadir}\"" \
         "-blocksdir=\"${my_blocksdir}\""
 
+    echo ""
+    echo "Installing latest 'serviceman'..."
+    echo ""
+    "$HOME/.local/bin/webi" serviceman > /dev/null
     if ! command -v serviceman > /dev/null; then
-        echo ""
-        echo "Installing 'serviceman'..."
-        echo ""
-        {
-            curl -fsSL "${WEBI_HOST}/serviceman" | sh
-        } > /dev/null
-
-        # shellcheck disable=SC1090
-        . ~/.config/envman/PATH.env || true
+        export PATH="$HOME/.local/bin:$PATH"
+    fi
+    serviceman --version
+    if ! command -v dashd > /dev/null; then
+        export PATH="$HOME/.local/opt/dashcore/bin:$PATH"
     fi
 
     mkdir -p "$HOME/.dashcore/wallets/"
@@ -130,8 +119,7 @@ fn_srv_install() { (
     cd "${my_vol}" || return 1
     # leave options unquoted so they're interpreted separately
     # shellcheck disable=SC2086
-    sudo env PATH="${PATH}" \
-        serviceman add ${my_system_args} --path "${PATH}" --name "${my_name}" --force -- \
+    serviceman add --name "${my_name}" -- \
         dashd \
         ${my_net_flag} \
         -usehd \

dashcore-utils/install.sh

@@ -5,32 +5,38 @@ set -u
 __install_dashcore_utils() {
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dash-qt-hd" \
-        "$HOME/.local/bin/dash-qt-hd"
+        "$HOME/.local/bin/dash-qt-hd" \
+        "dash-qt-hd"
     chmod a+x "$HOME/.local/bin/dash-qt-hd"
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dash-qt-testnet" \
-        "$HOME/.local/bin/dash-qt-testnet"
+        "$HOME/.local/bin/dash-qt-testnet" \
+        "dash-qt-testnet"
     chmod a+x "$HOME/.local/bin/dash-qt-testnet"
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dashd-hd" \
-        "$HOME/.local/bin/dashd-hd"
+        "$HOME/.local/bin/dashd-hd" \
+        "dashd-hd"
     chmod a+x "$HOME/.local/bin/dashd-hd"
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dashd-testnet" \
-        "$HOME/.local/bin/dashd-testnet"
+        "$HOME/.local/bin/dashd-testnet" \
+        "dashd-testnet"
     chmod a+x "$HOME/.local/bin/dashd-testnet"
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dashd-hd-service-install" \
-        "$HOME/.local/bin/dashd-hd-service-install"
+        "$HOME/.local/bin/dashd-hd-service-install" \
+        "dashd-hd-service-install"
     chmod a+x "$HOME/.local/bin/dashd-hd-service-install"
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dashd-testnet-service-install" \
-        "$HOME/.local/bin/dashd-testnet-service-install"
+        "$HOME/.local/bin/dashd-testnet-service-install" \
+        "dashd-testnet-service-install"
     chmod a+x "$HOME/.local/bin/dashd-testnet-service-install"
 
     if ! test -e "${HOME}/.dashcore"; then
@@ -44,7 +50,8 @@ __install_dashcore_utils() {
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dash.example.conf" \
-        "$HOME/.dashcore/dash.example.conf"
+        "$HOME/.dashcore/dash.example.conf" \
+        "dash.example.conf"
 
     if ! grep -q rpcuser ~/.dashcore/dash.conf; then
         cat ~/.dashcore/dash.example.conf >> ~/.dashcore/dash.conf

dashcore/README.md

@@ -147,8 +147,8 @@ dash-qt \
 CoinJoin aids in preventing some bad actors and malicious observers being able
 to easily reconstruct details about your transactions from the publicly
 available data by creating many excess transactions. \
-(be aware, however, that dedicated bad actors can use sophisticated software that
-will reveal much of the same information over time)
+(be aware, however, that dedicated bad actors can use sophisticated software
+that will reveal much of the same information over time)
 
 `dash-qt` does not enable CoinJoin mixing by default.
 

dashcore/install.ps1

@@ -20,13 +20,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading dashcore from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-IF (!(Test-Path -Path "$pkg_src_dir")) {
+if (!(Test-Path -Path "$pkg_src_dir")) {
     Write-Output "Installing dashcore"
 
     # TODO: create package-specific temp directory

dashcore/releases.js

@@ -4,8 +4,8 @@ var github = require('../_common/github.js');
 var owner = 'dashpay';
 var repo = 'dash';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     all.releases.forEach(function (rel) {
       if (rel.name.includes('osx64')) {
         rel.os = 'macos';
@@ -22,7 +22,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 5 for demonstration
     all.releases = all.releases.slice(0, 5);

dashd/README.md

@@ -24,7 +24,7 @@ install:
 ~/.config/envman/PATH.env
 ~/.dashcore/dash.conf
 ~/.dashcore/wallets/
-~/.local/bin/bin/dashd-hd-service-install
+~/.local/bin/dashd-hd-service-install
 ~/.local/opt/dashcore/
 /mnt/<BLK_VOL>/dashcore/
 ```
@@ -219,14 +219,7 @@ You can use [`serviceman`](../serviceman/):
 **Linux**
 
 ```sh
-sudo env PATH="$PATH" \
-    serviceman add \
-        --system \
-        --username "$(id -n -u)" \
-        --path "$PATH" \
-        --name dashd \
-        --force \
-        -- \
+serviceman add --name 'dashd' -- \
     dashd \
         -usehd \
         -conf="$HOME/.dashcore/dash.conf" \
@@ -239,11 +232,7 @@ sudo env PATH="$PATH" \
 **Mac**
 
 ```sh
-serviceman add \
-    --path "$PATH" \
-    --name dashd \
-    --force \
-    -- \
+serviceman add --name 'dashd' -- \
     dashd \
         -usehd \
         -conf="$HOME/.dashcore/dash.conf" \

dashd/install.ps1

@@ -20,13 +20,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading dashd from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-IF (!(Test-Path -Path "$pkg_src_dir")) {
+if (!(Test-Path -Path "$pkg_src_dir")) {
     Write-Output "Installing dashd"
 
     # TODO: create package-specific temp directory

dashmsg/install.ps1

@@ -19,13 +19,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading dashmsg from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-IF (!(Test-Path -Path "$pkg_src_cmd")) {
+if (!(Test-Path -Path "$pkg_src_cmd")) {
     Write-Output "Installing dashmsg"
 
     # TODO: create package-specific temp directory

dashmsg/releases.js

@@ -4,14 +4,14 @@ var github = require('../_common/github.js');
 var owner = 'dashhive';
 var repo = 'dashmsg';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all, null, 2));
   });

delta/install.ps1

@@ -19,13 +19,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading delta from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-IF (!(Test-Path -Path "$pkg_src_cmd")) {
+if (!(Test-Path -Path "$pkg_src_cmd")) {
     Write-Output "Installing delta"
 
     # TODO: create package-specific temp directory

delta/releases.js

@@ -4,14 +4,14 @@ var github = require('../_common/github.js');
 var owner = 'dandavison';
 var repo = 'delta';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 15 for demonstration
     all.releases = all.releases.slice(0, 15);

deno/README.md

@@ -18,7 +18,7 @@ etc).
 The obligatory Hello World
 
 ```sh
-deno run https://deno.land/std/examples/welcome.ts
+deno run https://docs.deno.com/examples/scripts/hello_world.ts
 ```
 
 Run a local file

deno/install.ps1

@@ -1,14 +1,14 @@
 #!/usr/bin/env pwsh
 
 # Fetch archive
-IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading $Env:PKG_NAME from $Env:WEBI_PKG_URL to $Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
     #Invoke-WebRequest https://nodejs.org/dist/v12.16.2/node-v12.16.2-win-x64.zip -OutFile node-v12.16.2-win-x64.zip
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE.part"
     & Move-Item "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE.part" "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 }
 
-IF (!(Test-Path -Path "$Env:USERPROFILE\.local\opt\$Env:PKG_NAME-v$Env:WEBI_VERSION")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\.local\opt\$Env:PKG_NAME-v$Env:WEBI_VERSION")) {
     Write-Output "Installing $Env:PKG_NAME"
     # TODO: temp directory
 

deno/releases.js

@@ -6,8 +6,8 @@ var github = require('../_common/github.js');
 var owner = 'denoland';
 var repo = 'deno';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     // remove checksums and .deb
     all.releases = all.releases
       .filter(function (rel) {
@@ -35,7 +35,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all, null, 2));
   });

dotenv-linter/install.ps1

@@ -20,18 +20,18 @@ $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch MSVC Runtime
 Write-Output "Checking for MSVC Runtime..."
-IF (-not (Test-Path "\Windows\System32\vcruntime140.dll")) {
+if (-not (Test-Path "\Windows\System32\vcruntime140.dll")) {
     & "$Env:USERPROFILE\.local\bin\webi-pwsh.ps1" vcruntime
 }
 
 # Fetch archive
-IF (!(Test-Path -Path "$pkg_download")) {
+if (!(Test-Path -Path "$pkg_download")) {
     Write-Output "Downloading dotenv-linter from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-IF (!(Test-Path -Path "$pkg_src_cmd")) {
+if (!(Test-Path -Path "$pkg_src_cmd")) {
     Write-Output "Installing dotenv-linter"
 
     # TODO: create package-specific temp directory

dotenv-linter/releases.js

@@ -4,14 +4,14 @@ var github = require('../_common/github.js');
 var owner = 'dotenv-linter';
 var repo = 'dotenv-linter';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 5 for demonstration
     all.releases = all.releases.slice(0, 5);

dotenv/install.ps1

@@ -19,13 +19,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading dotenv from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-IF (!(Test-Path -Path "$pkg_src_cmd")) {
+if (!(Test-Path -Path "$pkg_src_cmd")) {
     Write-Output "Installing dotenv"
 
     # TODO: create package-specific temp directory

dotenv/releases.js

@@ -4,14 +4,14 @@ var github = require('../_common/github.js');
 var owner = 'therootcompany';
 var repo = 'dotenv';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all));
   });

duckdns.sh/releases.js

@@ -1,29 +1,21 @@
 'use strict';
 
-var githubSource = require('../_common/github-source.js');
-var owner = 'BeyondCodeBootcamp';
-var repo = 'DuckDNS.sh';
+let Releases = module.exports;
 
-module.exports = function (request) {
-  let arches = [
-    'amd64',
-    'arm64',
-    'armv6l',
-    'armv7l',
-    'ppc64le',
-    'ppc64',
-    's390x',
-    'x86',
-  ];
-  let oses = ['freebsd', 'linux', 'macos', 'posix'];
-  return githubSource(request, owner, repo, oses, arches).then(function (all) {
-    all._names = ['DuckDNS.sh', 'duckdns.sh', 'legacy'];
-    return all;
-  });
+let GitHubSource = require('../_common/github-source.js');
+let owner = 'BeyondCodeBootcamp';
+let repo = 'DuckDNS.sh';
+
+Releases.latest = async function () {
+  let all = await GitHubSource.getDistributables({ owner, repo });
+  for (let pkg of all.releases) {
+    pkg.os = 'posix_2017';
+  }
+  return all;
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  Releases.latest().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all, null, 2));
   });

duckdns/install.sh

@@ -2,10 +2,16 @@
 set -e
 set -u
 
-echo "'duckdns@${WEBI_TAG:-stable}' is reserved for future use."
-echo
-echo "Did you mean 'duckdns.sh@${WEBI_VERSION-}'?"
 WEBI_HOST=${WEBI_HOST:-"https://webi.sh"}
+
 echo ""
-echo "    curl -fsSL '$WEBI_HOST/duckdns.sh@${WEBI_VERSION-}' | sh"
+echo "ERROR"
+echo "    installer name 'duckdns' is reserved for future use"
+echo ""
+echo "SOLUTION"
+echo "    Did you mean 'duckdns.sh'?"
+echo ""
+echo "    curl -fsSL '$WEBI_HOST/duckdns.sh' | sh"
+echo ""
+
 exit 1

fd/install.ps1

@@ -3,13 +3,13 @@
 $VERNAME = "$Env:PKG_NAME-v$Env:WEBI_VERSION.exe"
 $EXENAME = "$Env:PKG_NAME.exe"
 # Fetch archive
-IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading $Env:PKG_NAME from $Env:WEBI_PKG_URL to $Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE.part"
     & Move-Item "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE.part" "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 }
 
-IF (!(Test-Path -Path "$Env:USERPROFILE\.local\bin\$VERNAME")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\.local\bin\$VERNAME")) {
     Write-Output "Installing $Env:PKG_NAME"
     # TODO: temp directory
 

fd/releases.js

@@ -4,8 +4,8 @@ var github = require('../_common/github.js');
 var owner = 'sharkdp';
 var repo = 'fd';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     let builds = [];
 
     for (let build of all.releases) {
@@ -22,7 +22,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     all.releases = all.releases.slice(0, 10);
     //console.info(JSON.stringify(all));

ffmpeg/install.ps1

@@ -19,13 +19,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading ffmpeg from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     Move-Item -Path "$pkg_download.part" -Destination "$pkg_download" -Force
 }
 
-IF (!(Test-Path -Path "$pkg_src_cmd")) {
+if (!(Test-Path -Path "$pkg_src_cmd")) {
     Write-Output "Installing ffmpeg"
 
     # TODO: create package-specific temp directory

ffmpeg/releases.js

@@ -6,8 +6,8 @@ var github = require('../_common/github.js');
 var owner = 'eugeneware';
 var repo = 'ffmpeg-static';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     all.releases = all.releases
       .filter(function (rel) {
         let isFfmpeg = rel.name.includes('ffmpeg');
@@ -38,7 +38,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all));
   });

ffuf/install.ps1

@@ -20,13 +20,13 @@ New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
 $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 
 # Fetch archive
-IF (-Not (Test-Path -Path "$pkg_download")) {
+if (-not (Test-Path -Path "$pkg_download")) {
     Write-Output "Downloading ffuf from $Env:WEBI_PKG_URL to $pkg_download"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
     & Move-Item "$pkg_download.part" "$pkg_download"
 }
 
-IF (-Not (Test-Path -Path "$pkg_src_cmd")) {
+if (-not (Test-Path -Path "$pkg_src_cmd")) {
     Write-Output "Installing ffuf"
 
     # TODO: create package-specific temp directory

ffuf/releases.js

@@ -4,14 +4,14 @@ var github = require('../_common/github.js');
 var owner = 'ffuf';
 var repo = 'ffuf';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 5 for demonstration
     all.releases = all.releases.slice(0, 5);

fish/README.md

@@ -43,7 +43,7 @@ the file:
 ```sh
 #!/bin/bash
 
-echo "Who am I? I'm $(whoami)."
+echo "Who am I? I'm $(id -u -n)."
 ```
 
 You can also run bash explicitly:
@@ -99,7 +99,7 @@ You should use `chsh` to change your shell:
 ```sh
 #!/bin/sh
 
-sudo chsh -s "$(command -v fish)" "$(whoami)"
+sudo chsh -s "$(command -v fish)" "$(id -u -n)"
 ```
 
 If vim uses `fish` instead of `bash`, annoying errors will happen.

fish/releases.js

@@ -6,8 +6,8 @@ var repo = 'fish-shell';
 
 var ODDITIES = ['bundledpcre'];
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     all.releases = all.releases
       .map(function (rel) {
         for (let oddity of ODDITIES) {
@@ -30,7 +30,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 5 for demonstration
     all.releases = all.releases.slice(0, 5);

flutter/releases.js

@@ -1,9 +1,14 @@
 'use strict';
 
-var FLUTTER_OSES = ['macos', 'linux', 'windows'];
+let Fetcher = require('../_common/fetcher.js');
 
-// stable, beta, dev
-var channelMap = {};
+let FLUTTER_OSES = ['macos', 'linux', 'windows'];
+
+/**
+ * stable, beta, dev
+ * @type {Object.<String, Boolean>}
+ */
+let channelMap = {};
 
 // This can be spot-checked against
 // https://docs.flutter.dev/release/archive?tab=windows
@@ -53,21 +58,45 @@ var channelMap = {};
 //   ]
 // }
 
-module.exports = async function (request) {
+/**
+ * @typedef BuildInfo
+ * @prop {String} version
+ * @prop {String} [_version]
+ * @prop {Boolean} lts
+ * @prop {String} channel
+ * @prop {String} date
+ * @prop {String} download
+ * @prop {String} [_filename]
+ */
+
+module.exports = async function () {
   let all = {
     download: '',
+    /** @type {Array<BuildInfo>} */
     releases: [],
+    /** @type {Array<String>} */
     channels: [],
   };
 
   for (let osname of FLUTTER_OSES) {
-    let resp = await request({
-      url: `https://storage.googleapis.com/flutter_infra_release/releases/releases_${osname}.json`,
-      json: true,
-    });
+    let resp;
+    try {
+      let url = `https://storage.googleapis.com/flutter_infra_release/releases/releases_${osname}.json`;
+      resp = await Fetcher.fetch(url, {
+        headers: { Accept: 'application/json' },
+      });
+    } catch (e) {
+      /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error
+      let err = e;
+      if (err.code === 'E_FETCH_RELEASES') {
+        err.message = `failed to fetch 'flutter' release data for ${osname}: ${err.response.status} ${err.response.body}`;
+      }
+      throw e;
+    }
+    let data = JSON.parse(resp.body);
 
-    let osBaseUrl = resp.body.base_url;
-    let osReleases = resp.body.releases;
+    let osBaseUrl = data.base_url;
+    let osReleases = data.releases;
 
     for (let asset of osReleases) {
       if (!channelMap[asset.channel]) {
@@ -80,7 +109,6 @@ module.exports = async function (request) {
         lts: false,
         channel: asset.channel,
         date: asset.release_date.replace(/T.*/, ''),
-        //sha256: asset.sha256,
         download: `${osBaseUrl}/${asset.archive}`,
         _filename: asset.archive,
       });
@@ -97,7 +125,7 @@ module.exports = async function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all.releases = all.releases.slice(25);
     console.info(JSON.stringify(all, null, 2));
   });

fzf/install.ps1

@@ -3,13 +3,13 @@
 $VERNAME = "$Env:PKG_NAME-v$Env:WEBI_VERSION.exe"
 $EXENAME = "$Env:PKG_NAME.exe"
 # Fetch archive
-IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
     Write-Output "Downloading $Env:PKG_NAME from $Env:WEBI_PKG_URL to $Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
     & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE.part"
     & Move-Item "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE.part" "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
 }
 
-IF (!(Test-Path -Path "$Env:USERPROFILE\.local\bin\$VERNAME")) {
+if (!(Test-Path -Path "$Env:USERPROFILE\.local\bin\$VERNAME")) {
     Write-Output "Installing $Env:PKG_NAME"
     # TODO: temp directory
 

fzf/releases.js

@@ -4,14 +4,14 @@ var github = require('../_common/github.js');
 var owner = 'junegunn';
 var repo = 'fzf';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     all.releases = all.releases.slice(0, 10);
     //console.info(JSON.stringify(all));