chore: add types via jsdoc

Signed-off-by: bry-val <94031627+bry-val@users.noreply.github.com>

.github/workflows/node.js.yml

@@ -23,10 +23,15 @@ jobs:
           sh ./_scripts/install-ci-deps
           echo "${HOME}/.local/bin" >> $GITHUB_PATH
           echo "${HOME}/.local/opt/pwsh" >> $GITHUB_PATH
+      - run: |
+          git submodule init
+          git submodule update
       - run: shfmt --version
       - run: shellcheck -V
       - run: node --version
       - run: npm run fmt
       - run: npm ci
       - run: npm run lint
-      - run: npm run test
+      - env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: npm run test

.gitignore

@@ -1,7 +1,24 @@
-.env
-node_modules
+# generated artifacts
 install-*.sh
 install-*.bat
 install-*.ps1
+
+# local config
+.env.*
+*.env
+.env
+!example.env
+
+# caches
+_cache/
+node_modules/
+
+# temporary & backup files
 .*.sw*
 *.bak
+*.bak.*
+
+# other
+.DS_Store
+desktop.ini
+.directory

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "_webi/build-classifier"]
+	path = _webi/build-classifier
+	url = https://github.com/webinstall/webi-build-classifier.git

.jshintrc

@@ -1,4 +1,7 @@
 {
+  "globals": {
+    "AbortController": false
+  },
   "browser": true,
   "node": true,
   "esversion": 11,

README.md

@@ -98,9 +98,10 @@ You just fill in the blanks.
 Just create an empty directory and run the tests until you get a good result.
 
 ```sh
-git clone git@github.com:webinstall/packages.git
-pushd packages
-npm install
+git clone git@github.com:webinstall/webi-installers.git
+pushd ./webi-installers/
+git submodule update --init
+npm clean-install
 ```
 
 ```sh
@@ -144,8 +145,8 @@ It looks like this:
 `releases.js`:
 
 ```js
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     // if you need to do something special, you can do it here
     // ...
     return all;

_common/brew.js

@@ -1,64 +1,62 @@
 'use strict';
 
+let Fetcher = require('../_common/fetcher.js');
+
 /**
- * Gets a releases from 'brew'.
+ * Gets releases from 'brew'.
  *
- * @param request
+ * @param {null} _
  * @param {string} formula
- * @returns {PromiseLike<any> | Promise<any>}
+ * @returns {Promise<any>}
  */
-function getAllReleases(request, formula) {
+async function getDistributables(_, formula) {
   if (!formula) {
     return Promise.reject('missing formula for brew');
   }
-  return request({
-    url: 'https://formulae.brew.sh/api/formula/' + formula + '.json',
-    fail: true, // https://git.coolaj86.com/coolaj86/request.js/issues/2
-    json: true,
-  })
-    .then(failOnBadStatus)
-    .then(function (resp) {
-      var ver = resp.body.versions.stable;
-      var dl = (
-        resp.body.bottle.stable.files.high_sierra ||
-        resp.body.bottle.stable.files.catalina
-      ).url.replace(new RegExp(ver.replace(/\./g, '\\.'), 'g'), '{{ v }}');
-      return [
-        {
-          version: ver,
-          download: dl.replace(/{{ v }}/g, ver),
-        },
-      ].concat(
-        resp.body.versioned_formulae.map(function (f) {
-          var ver = f.replace(/.*@/, '');
-          return {
-            version: ver,
-            download: dl,
-          };
-        }),
-      );
-    })
-    .catch(function (err) {
-      console.error('Error fetching MariaDB versions (brew)');
-      console.error(err);
-      return [];
+
+  let resp;
+  try {
+    let url = `https://formulae.brew.sh/api/formula/${formula}.json`;
+    resp = await Fetcher.fetch(url, {
+      headers: { Accept: 'application/json' },
     });
-}
-
-function failOnBadStatus(resp) {
-  if (resp.statusCode >= 400) {
-    var err = new Error('Non-successful status code: ' + resp.statusCode);
-    err.code = 'ESTATUS';
-    err.response = resp;
-    throw err;
+  } catch (e) {
+    /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error
+    let err = e;
+    if (err.code === 'E_FETCH_RELEASES') {
+      err.message = `failed to fetch '${formula}' release data from 'brew': ${err.response.status} ${err.response.body}`;
+    }
+    throw e;
   }
-  return resp;
+  let body = JSON.parse(resp.body);
+
+  var ver = body.versions.stable;
+  var dl = (
+    body.bottle.stable.files.high_sierra || body.bottle.stable.files.catalina
+  ).url.replace(new RegExp(ver.replace(/\./g, '\\.'), 'g'), '{{ v }}');
+  return [
+    {
+      version: ver,
+      download: dl.replace(/{{ v }}/g, ver),
+    },
+  ].concat(
+    body.versioned_formulae.map(
+      /** @param {String} f */
+      function (f) {
+        var ver = f.replace(/.*@/, '');
+        return {
+          version: ver,
+          download: dl,
+        };
+      },
+    ),
+  );
 }
 
-module.exports = getAllReleases;
+module.exports = getDistributables;
 
 if (module === require.main) {
-  getAllReleases(require('@root/request'), 'mariadb').then(function (all) {
+  getDistributables(null, 'mariadb').then(function (all) {
     console.info(JSON.stringify(all, null, 2));
   });
 }

_common/fetcher.js

@@ -0,0 +1,56 @@
+'use strict';
+
+let Fetcher = module.exports;
+
+/**
+ * @typedef ResponseSummary
+ * @prop {Boolean} ok
+ * @prop {Headers} headers
+ * @prop {Number} status
+ * @prop {String} body
+ */
+
+/**
+ * @param {String} url
+ * @param {RequestInit} opts
+ * @returns {Promise<ResponseSummary>}
+ */
+Fetcher.fetch = async function (url, opts) {
+  let resp = await fetch(url, opts);
+  let summary = Fetcher.throwIfNotOk(resp);
+
+  return summary;
+};
+
+/**
+ * @param {Response} resp
+ * @returns {Promise<ResponseSummary>}
+ */
+Fetcher.throwIfNotOk = async function (resp) {
+  let text = await resp.text();
+
+  if (!resp.ok) {
+    let headers = Array.from(resp.headers);
+    console.error('[Fetcher] error: Response Headers:', headers);
+    console.error('[Fetcher] error: Response Text:', text);
+    let err = new Error(`fetch was not ok`);
+    Object.assign({
+      status: 503,
+      code: 'E_FETCH_RELEASES',
+      response: {
+        status: resp.status,
+        headers: headers,
+        body: text,
+      },
+    });
+    throw err;
+  }
+
+  let summary = {
+    ok: resp.ok,
+    headers: resp.headers,
+    status: resp.status,
+    body: text,
+  };
+  return summary;
+};

_common/git-tag.js

@@ -6,11 +6,12 @@ var Crypto = require('crypto');
 var util = require('util');
 var exec = util.promisify(require('child_process').exec);
 var Fs = require('node:fs/promises');
+var FsSync = require('node:fs');
 var Path = require('node:path');
 
 var repoBaseDir = process.env.REPO_BASE_DIR || '';
 if (!repoBaseDir) {
-  repoBaseDir = Path.resolve('./repos');
+  repoBaseDir = Path.resolve('./_repos');
   // for stderr
   console.error(`[Warn] REPO_BASE_DIR= not set, ${repoBaseDir}`);
 }
@@ -20,8 +21,29 @@ var Repos = {};
 Repos.clone = async function (repoPath, gitUrl) {
   let uuid = Crypto.randomUUID();
   let tmpPath = `${repoPath}.${uuid}.tmp`;
+  let bakPath = `${repoPath}.${uuid}.dup`;
   await exec(`git clone --bare --filter=tree:0 ${gitUrl} ${tmpPath}`);
-  await Fs.rename(tmpPath, repoPath);
+
+  try {
+    FsSync.accessSync(repoPath);
+    return;
+  } catch (e) {
+    if (e.code !== 'ENOENT') {
+      throw e;
+    }
+  }
+
+  // sync to avoid race conditions
+  try {
+    FsSync.renameSync(repoPath, bakPath);
+  } catch (e) {
+    if (e.code !== 'ENOENT') {
+      throw e;
+    }
+  }
+  FsSync.renameSync(tmpPath, repoPath);
+
+  await Fs.rm(bakPath, { force: true, recursive: true });
 };
 
 Repos.checkExists = async function (repoPath) {
@@ -93,7 +115,7 @@ Repos.getCommitInfo = async function (repoPath, commitish) {
  * @param {string} gitUrl
  * @returns {PromiseLike<any> | Promise<any>}
  */
-async function getAllReleases(gitUrl) {
+async function getDistributables(gitUrl) {
   let all = {
     releases: [],
     download: '',
@@ -168,7 +190,7 @@ async function getAllReleases(gitUrl) {
   return all;
 }
 
-module.exports = getAllReleases;
+module.exports = getDistributables;
 
 if (module === require.main) {
   (async function main() {
@@ -181,7 +203,7 @@ if (module === require.main) {
       //'https://github.com/dense-analysis/ale.git',
     ];
     for (let url of testRepos) {
-      let all = await getAllReleases(url);
+      let all = await getDistributables(url);
 
       all = require('../_webi/normalize.js')(all);
       console.info(JSON.stringify(all, null, 2));

_common/gitea.js

@@ -1,39 +1,47 @@
 'use strict';
 
-var ghRelease = require('./github.js');
+var GitHubish = require('./githubish.js');
 
 /**
- * Gets the releases for 'ripgrep'. This function could be trimmed down and made
- * for use with any github release.
+ * Lists Gitea Releases (w/ uploaded assets)
  *
- * @param request
- * @param {string} owner
- * @param {string} repo
- * @returns {PromiseLike<any> | Promise<any>}
+ * @param {null} _ - deprecated
+ * @param {String} owner
+ * @param {String} repo
+ * @param {String} baseurl
+ * @param {String} [username]
+ * @param {String} [token]
  */
-function getAllReleases(request, owner, repo, baseurl) {
-  if (!baseurl) {
-    return Promise.reject('missing baseurl');
-  }
-  return ghRelease(request, owner, repo, baseurl + '/api/v1').then(
-    function (all) {
-      return all;
-    },
-  );
+async function getDistributables(
+  _,
+  owner,
+  repo,
+  baseurl,
+  username = '',
+  token = '',
+) {
+  baseurl = `${baseurl}/api/v1`;
+  let all = await GitHubish.getDistributables({
+    owner,
+    repo,
+    baseurl,
+    username,
+    token,
+  });
+  return all;
 }
 
-module.exports = getAllReleases;
+module.exports = getDistributables;
 
 if (module === require.main) {
-  getAllReleases(
-    require('@root/request'),
-    'coolaj86',
-    'go-pathman',
-    'https://git.coolaj86.com',
-  ).then(
-    //getAllReleases(require('@root/request'), 'root', 'serviceman', 'https://git.rootprojects.org').then(
-    function (all) {
-      console.info(JSON.stringify(all, null, 2));
-    },
-  );
+  getDistributables(
+    null,
+    'root',
+    'pathman',
+    'https://git.rootprojects.org',
+    '',
+    '',
+  ).then(function (all) {
+    console.info(JSON.stringify(all, null, 2));
+  });
 }

_common/github-source.js

@@ -1,133 +1,40 @@
 'use strict';
 
-require('dotenv').config();
+require('dotenv').config({ path: '.env' });
+
+let GitHubSource = module.exports;
+
+let GitHubishSource = require('./githubish-source.js');
 
 /**
- * Gets the releases for 'ripgrep'. This function could be trimmed down and made
- * for use with any github release.
- *
- * @param request
- * @param {string} owner
- * @param {string} repo
- * @returns {PromiseLike<any> | Promise<any>}
+ * @param {Object} opts
+ * @param {String} opts.owner
+ * @param {String} opts.repo
+ * @param {String} [opts.baseurl]
+ * @param {String} [opts.username]
+ * @param {String} [opts.token]
  */
-async function getAllReleases(
-  request,
+GitHubSource.getDistributables = async function ({
   owner,
   repo,
-  oses,
-  arches,
   baseurl = 'https://api.github.com',
-) {
-  if (!owner) {
-    return Promise.reject('missing owner for repo');
-  }
-  if (!repo) {
-    return Promise.reject('missing repo name');
-  }
-
-  let req = {
-    url: `${baseurl}/repos/${owner}/${repo}/releases`,
-    json: true,
-  };
-
-  // TODO I really don't like global config, find a way to do better
-  if (process.env.GITHUB_USERNAME) {
-    req.auth = {
-      user: process.env.GITHUB_USERNAME,
-      pass: process.env.GITHUB_TOKEN,
-    };
-  }
-
-  let resp = await request(req);
-  let gHubResp = resp.body;
-  let all = {
-    releases: [],
-    // TODO make this ':baseurl' + ':releasename'
-    download: '',
-  };
-
-  for (let release of gHubResp) {
-    // TODO tags aren't always semver / sensical
-    let tag = release['tag_name'];
-    let lts = /(\b|_)(lts)(\b|_)/.test(release['tag_name']);
-    let channel = 'stable';
-    if (release['prerelease']) {
-      channel = 'beta';
-    }
-    let date = release['published_at'] || '';
-    date = date.replace(/T.*/, '');
-
-    let urls = [release.tarball_url, release.zipball_url];
-    for (let url of urls) {
-      let resp = await request({
-        method: 'HEAD',
-        followRedirect: true,
-        followAllRedirects: true,
-        followOriginalHttpMethod: true,
-        url: url,
-        stream: true,
-      });
-      // Workaround for bug where method changes to GET
-      resp.destroy();
-
-      // content-disposition: attachment; filename=BeyondCodeBootcamp-DuckDNS.sh-v1.0.1-0-ga2f4bde.zip
-      let name = resp.headers['content-disposition'].replace(
-        /.*filename=([^;]+)(;|$)/,
-        '$1',
-      );
-      all.releases.push({
-        name: name,
-        version: tag,
-        lts: lts,
-        channel: channel,
-        date: date,
-        os: '', // will be guessed by download filename
-        arch: '', // will be guessed by download filename
-        ext: '', // will be normalized
-        download: resp.request.uri.href,
-      });
-    }
-  }
-
-  if (oses) {
-    return combinate(all, oses, arches);
-  }
-
+  username = process.env.GITHUB_USERNAME || '',
+  token = process.env.GITHUB_TOKEN || '',
+}) {
+  let all = await GitHubishSource.getDistributables({
+    owner,
+    repo,
+    baseurl,
+    username,
+    token,
+  });
   return all;
-}
-
-function combinate(all, oses, arches) {
-  let releases = all.releases;
-  // ex: arches = ['amd64', 'arm64', 'armv7l', 'armv6l', 'x86'];
-  // ex: oses = ['macos', 'linux', 'bsd', 'posix'];
-
-  let combos = [];
-  for (let release of releases) {
-    for (let arch of arches) {
-      for (let os of oses) {
-        let combo = {
-          arch: arch,
-          os: os,
-        };
-        let rel = Object.assign({}, release, combo);
-        combos.push(rel);
-      }
-    }
-  }
-  all.releases = combos;
-
-  return all;
-}
-
-module.exports = getAllReleases;
+};
 
 if (module === require.main) {
-  getAllReleases(
-    require('@root/request'),
-    'BeyondCodeBootcamp',
-    'DuckDNS.sh',
-  ).then(function (all) {
-    console.info(JSON.stringify(all, null, 2));
-  });
+  GitHubSource.getDistributables(null, 'BeyondCodeBootcamp', 'DuckDNS.sh').then(
+    function (all) {
+      console.info(JSON.stringify(all, null, 2));
+    },
+  );
 }

_common/github.js

@@ -1,102 +1,42 @@
 'use strict';
 
-require('dotenv').config();
+require('dotenv').config({ path: '.env' });
+
+let GitHubish = require('./githubish.js');
 
 /**
  * Lists GitHub Releases (w/ uploaded assets)
  *
- * @param request
- * @param {string} owner
- * @param {string} repo
- * @returns {PromiseLike<any> | Promise<any>}
+ * @param {null} _ - deprecated
+ * @param {String} owner
+ * @param {String} repo
+ * @param {String} [baseurl]
+ * @param {String} [username]
+ * @param {String} [token]
  */
-async function getAllReleases(
-  request,
+module.exports = async function (
+  _,
   owner,
   repo,
   baseurl = 'https://api.github.com',
+  username = process.env.GITHUB_USERNAME || '',
+  token = process.env.GITHUB_TOKEN || '',
 ) {
-  if (!owner) {
-    throw new Error('missing owner for repo');
-  }
-  if (!repo) {
-    throw new Error('missing repo name');
-  }
-
-  var req = {
-    url: `${baseurl}/repos/${owner}/${repo}/releases`,
-    json: true,
-  };
-
-  // TODO I really don't like global config, find a way to do better
-  if (process.env.GITHUB_USERNAME) {
-    req.auth = {
-      user: process.env.GITHUB_USERNAME,
-      pass: process.env.GITHUB_TOKEN,
-    };
-  }
-
-  let resp = await request(req);
-  if (!resp.ok) {
-    console.error('Bad Resp Headers:', resp.headers);
-    console.error('Bad Resp Body:', resp.body);
-    throw new Error('the elusive releases BOOGEYMAN strikes again');
-  }
-
-  let gHubResp = resp.body;
-  let all = {
-    releases: [],
-    // todo make this ':baseurl' + ':releasename'
-    download: '',
-  };
-
-  try {
-    gHubResp.forEach(transformReleases);
-  } catch (e) {
-    console.error(e.message);
-    console.error('Error Headers:', resp.headers);
-    console.error('Error Body:', resp.body);
-    throw e;
-  }
-
-  function transformReleases(release) {
-    for (let asset of release['assets']) {
-      let name = asset['name'];
-      let date = release['published_at']?.replace(/T.*/, '');
-      let download = asset['browser_download_url'];
-
-      // TODO tags aren't always semver / sensical
-      let version = release['tag_name'];
-      let channel;
-      if (release['prerelease']) {
-        // -rcX, -preview, -beta, etc will be checked in _webi/normalize.js
-        channel = 'beta';
-      }
-      let lts = /(\b|_)(lts)(\b|_)/.test(release['tag_name']);
-
-      all.releases.push({
-        name: name,
-        version: version,
-        lts: lts,
-        channel: channel,
-        date: date,
-        os: '', // will be guessed by download filename
-        arch: '', // will be guessed by download filename
-        ext: '', // will be normalized
-        download: download,
-      });
-    }
-  }
-
+  let all = await GitHubish.getDistributables({
+    owner,
+    repo,
+    baseurl,
+    username,
+    token,
+  });
   return all;
-}
+};
 
-module.exports = getAllReleases;
+let GitHub = module.exports;
+GitHub.getDistributables = module.exports;
 
 if (module === require.main) {
-  getAllReleases(require('@root/request'), 'BurntSushi', 'ripgrep').then(
-    function (all) {
-      console.info(JSON.stringify(all, null, 2));
-    },
-  );
+  GitHub.getDistributables(null, 'BurntSushi', 'ripgrep').then(function (all) {
+    console.info(JSON.stringify(all, null, 2));
+  });
 }

_common/githubish-source.js

@@ -0,0 +1,174 @@
+'use strict';
+
+let Fetcher = require('../_common/fetcher.js');
+
+let GitHubishSource = module.exports;
+
+/**
+ * Lists GitHub-Like Releases (source tarball & zip)
+ *
+ * @param {Object} opts
+ * @param {String} opts.owner
+ * @param {String} opts.repo
+ * @param {String} opts.baseurl
+ * @param {String} [opts.username]
+ * @param {String} [opts.token]
+ */
+GitHubishSource.getDistributables = async function ({
+  owner,
+  repo,
+  baseurl,
+  username = '',
+  token = '',
+}) {
+  if (!owner) {
+    throw new Error('missing owner for repo');
+  }
+  if (!repo) {
+    throw new Error('missing repo name');
+  }
+  if (!baseurl) {
+    throw new Error('missing baseurl');
+  }
+
+  let url = `${baseurl}/repos/${owner}/${repo}/releases`;
+  let opts = {
+    headers: {
+      'Content-Type': 'appplication/json',
+    },
+  };
+
+  if (token) {
+    let userpass = `${username}:${token}`;
+    let basicAuth = btoa(userpass);
+    Object.assign(opts.headers, {
+      Authorization: `Basic ${basicAuth}`,
+    });
+  }
+
+  let resp;
+  try {
+    resp = await Fetcher.fetch(url, opts);
+  } catch (e) {
+    /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error
+    let err = e;
+    if (err.code === 'E_FETCH_RELEASES') {
+      err.message = `failed to fetch '${baseurl}' (githubish-source, user '${username}) release data: ${err.response.status} ${err.response.body}`;
+    }
+    throw e;
+  }
+  let gHubResp = JSON.parse(resp.body);
+
+  let all = {
+    /** @type {Array<BuildInfo>} */
+    releases: [],
+    download: '',
+  };
+
+  for (let release of gHubResp) {
+    let dists = GitHubishSource.releaseToDistributables(release);
+    for (let dist of dists) {
+      let updates =
+        await GitHubishSource.followDistributableDownloadAttachment(dist);
+      Object.assign(dist, updates);
+      all.releases.push(dist);
+    }
+  }
+
+  return all;
+};
+
+/**
+ * @typedef BuildInfo
+ * @prop {String} [name] - name to use instead of filename for hash urls
+ * @prop {String} version
+ * @prop {String} [_version]
+ * @prop {String} [arch]
+ * @prop {String} channel
+ * @prop {String} date
+ * @prop {String} download
+ * @prop {String} [ext]
+ * @prop {String} [_filename]
+ * @prop {String} [hash]
+ * @prop {String} [libc]
+ * @prop {Boolean} [_musl]
+ * @prop {Boolean} [lts]
+ * @prop {String} [size]
+ * @prop {String} os
+ */
+
+/**
+ * @param {any} ghRelease - TODO
+ * @returns {Array<BuildInfo>}
+ */
+GitHubishSource.releaseToDistributables = function (ghRelease) {
+  let ghTag = ghRelease['tag_name']; // TODO tags aren't always semver / sensical
+  let lts = /(\b|_)(lts)(\b|_)/.test(ghRelease['tag_name']);
+  let channel = 'stable';
+  if (ghRelease['prerelease']) {
+    channel = 'beta';
+  }
+  let date = ghRelease['published_at'] || '';
+  date = date.replace(/T.*/, '');
+
+  let urls = [ghRelease.tarball_url, ghRelease.zipball_url];
+  /** @type {Array<BuildInfo>} */
+  let dists = [];
+  for (let url of urls) {
+    dists.push({
+      name: '',
+      version: ghTag,
+      lts: lts,
+      channel: channel,
+      date: date,
+      os: '*',
+      arch: '*',
+      libc: '',
+      ext: '',
+      download: url,
+    });
+  }
+
+  return dists;
+};
+
+/**
+ * @param {BuildInfo} dist
+ */
+GitHubishSource.followDistributableDownloadAttachment = async function (dist) {
+  let abortCtrl = new AbortController();
+  let resp = await fetch(dist.download, {
+    method: 'HEAD',
+    redirect: 'follow',
+    signal: abortCtrl.signal,
+  });
+  let headers = Object.fromEntries(resp.headers);
+
+  // Workaround for bug where METHOD changes to GET
+  abortCtrl.abort();
+  await resp.text().catch(function (err) {
+    if (err.name !== 'AbortError') {
+      throw err;
+    }
+  });
+
+  // ex: content-disposition: attachment; filename=BeyondCodeBootcamp-DuckDNS.sh-v1.0.1-0-ga2f4bde.zip
+  //  => BeyondCodeBootcamp-DuckDNS.sh-v1.0.1-0-ga2f4bde.zip
+  let name = headers['content-disposition'].replace(
+    /.*filename=([^;]+)(;|$)/,
+    '$1',
+  );
+  let download = resp.url;
+
+  return { name, download };
+};
+
+if (module === require.main) {
+  GitHubishSource.getDistributables({
+    owner: 'BeyondCodeBootcamp',
+    repo: 'DuckDNS.sh',
+    baseurl: 'https://api.github.com',
+  }).then(function (all) {
+    console.info(JSON.stringify(all, null, 2));
+  });
+}

_common/githubish.js

@@ -0,0 +1,137 @@
+'use strict';
+
+let Fetcher = require('../_common/fetcher.js');
+
+/**
+ * @typedef DistributableRaw
+ * @prop {String} name
+ * @prop {String} version
+ * @prop {Boolean} lts
+ * @prop {String} [channel]
+ * @prop {String} date
+ * @prop {String} os
+ * @prop {String} arch
+ * @prop {String} ext
+ * @prop {String} download
+ */
+
+let GitHubish = module.exports;
+
+/**
+ * Lists GitHub-Like Releases (w/ uploaded assets)
+ *
+ * @param {Object} opts
+ * @param {String} opts.owner
+ * @param {String} opts.repo
+ * @param {String} opts.baseurl
+ * @param {String} [opts.username]
+ * @param {String} [opts.token]
+ */
+GitHubish.getDistributables = async function ({
+  owner,
+  repo,
+  baseurl,
+  username = '',
+  token = '',
+}) {
+  if (!owner) {
+    throw new Error('missing owner for repo');
+  }
+  if (!repo) {
+    throw new Error('missing repo name');
+  }
+  if (!baseurl) {
+    throw new Error('missing baseurl');
+  }
+
+  let url = `${baseurl}/repos/${owner}/${repo}/releases`;
+  let opts = {
+    headers: {
+      'Content-Type': 'appplication/json',
+    },
+  };
+
+  if (token) {
+    let userpass = `${username}:${token}`;
+    let basicAuth = btoa(userpass);
+    Object.assign(opts.headers, {
+      Authorization: `Basic ${basicAuth}`,
+    });
+  }
+
+  let resp;
+  try {
+    resp = await Fetcher.fetch(url, opts);
+  } catch (e) {
+    /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error
+    let err = e;
+    if (err.code === 'E_FETCH_RELEASES') {
+      err.message = `failed to fetch '${baseurl}' (githubish, user '${username}) release data: ${err.response.status} ${err.response.body}`;
+    }
+    throw e;
+  }
+  let gHubResp = JSON.parse(resp.body);
+
+  let all = {
+    /** @type {Array<DistributableRaw>} */
+    releases: [],
+    // todo make this ':baseurl' + ':releasename'
+    download: '',
+  };
+
+  try {
+    gHubResp.forEach(transformReleases);
+  } catch (e) {
+    /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error
+    let err = e;
+    console.error(err.message);
+    console.error('Error Headers:', resp.headers);
+    console.error('Error Body:', resp.body);
+    let msg = `failed to transform releases from '${baseurl}' with user '${username}'`;
+    throw new Error(msg);
+  }
+
+  /**
+   * @param {any} release - TODO
+   */
+  function transformReleases(release) {
+    for (let asset of release['assets']) {
+      let name = asset['name'];
+      let date = release['published_at']?.replace(/T.*/, '');
+      let download = asset['browser_download_url'];
+
+      // TODO tags aren't always semver / sensical
+      let version = release['tag_name'];
+      let channel;
+      if (release['prerelease']) {
+        // -rcX, -preview, -beta, etc will be checked in _webi/normalize.js
+        channel = 'beta';
+      }
+      let lts = /(\b|_)(lts)(\b|_)/.test(release['tag_name']);
+
+      all.releases.push({
+        name: name,
+        version: version,
+        lts: lts,
+        channel: channel,
+        date: date,
+        os: '', // will be guessed by download filename
+        arch: '', // will be guessed by download filename
+        ext: '', // will be normalized
+        download: download,
+      });
+    }
+  }
+
+  return all;
+};
+
+if (module === require.main) {
+  GitHubish.getDistributables({
+    owner: 'BurntSushi',
+    repo: 'ripgrep',
+    baseurl: 'https://api.github.com',
+  }).then(function (all) {
+    console.info(JSON.stringify(all, null, 2));
+  });
+}

_example/README.md

@@ -23,7 +23,7 @@ install:
 ```text
 ~/.config/envman/PATH.env
 ~/.local/bin/foo
-~/.local/opt/foo
+~/.local/opt/foo/
 ```
 
 ## Cheat Sheet

_example/releases.js

@@ -12,17 +12,26 @@ var repo = 'ripgrep';
 /**                                                                          **/
 /******************************************************************************/
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
-    return all;
-  });
+let Releases = module.exports;
+
+Releases.latest = async function () {
+  let all = await github(null, owner, repo);
+  return all;
+};
+
+Releases.sample = async function () {
+  let normalize = require('../_webi/normalize.js');
+  let all = await Releases.latest();
+  all = normalize(all);
+  // just select the first 5 for demonstration
+  all.releases = all.releases.slice(0, 5);
+  return all;
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    // just select the first 5 for demonstration
-    all.releases = all.releases.slice(0, 5);
-    console.info(JSON.stringify(all, null, 2));
-  });
+  (async function () {
+    let samples = await Releases.sample();
+
+    console.info(JSON.stringify(samples, null, 2));
+  })();
 }

_npm/scripts/install-webi.js

@@ -4,7 +4,6 @@
 
 //var pkg = require('../package.json');
 var os = require('os');
-//var request = require('@root/request');
 //var promisify = require('util').promisify;
 //var exec = promisify(require('child_process').exec);
 var exec = require('child_process').exec;

_webi/builds-cacher-test.js

@@ -0,0 +1,73 @@
+'use strict';
+
+let Path = require('node:path');
+
+let BuildsCacher = require('./builds-cacher.js');
+// let Parallel = require('./parallel.js');
+
+var INSTALLERS_DIR = Path.join(__dirname, '..');
+var CACHE_DIR = Path.join(__dirname, '../_cache');
+
+async function main() {
+  let bc = BuildsCacher.create({
+    caches: CACHE_DIR,
+    installers: INSTALLERS_DIR,
+  });
+  bc.freshenRandomPackage(600 * 1000);
+
+  // let dirs = await bc.getProjectsByType();
+  // let projNames = Object.keys(dirs.valid);
+
+  let lastUpdate;
+
+  let projName = 'k9s';
+  {
+    let packages = await bc.getPackages({
+      //Releases: Releases,
+      name: projName,
+      date: new Date(),
+    });
+    lastUpdate = packages.updated;
+    console.info(
+      `Last update for '${projName}': ${packages.updated} (${packages.releases.length} assets)`,
+    );
+  }
+
+  console.info('Waiting 5s');
+  {
+    setTimeout(async function () {
+      let packages = await bc.getPackages({
+        //Releases: Releases,
+        name: projName,
+        date: new Date(),
+      });
+      console.info(
+        `Last update for '${projName}': ${packages.updated} (${packages.releases.length} assets)`,
+      );
+      if (lastUpdate < packages.updated) {
+        console.info(`PASS`);
+      } else {
+        console.info(`MAYBE fail`);
+      }
+    }, 5 * 1000);
+  }
+
+  //let parallel = 25;
+  //await Parallel.run(parallel, projNames, getAll);
+  //async function getAll(name) {
+  //  void (await bc.getPackages({
+  //    //Releases: Releases,
+  //    name: name,
+  //    date: new Date(),
+  //  }));
+  //}
+}
+
+main()
+  .then(function () {
+    console.log('Done');
+  })
+  .catch(function (e) {
+    console.error(e.stack || e);
+    process.exit(1);
+  });

_webi/builds.js

@@ -0,0 +1,41 @@
+'use strict';
+
+let Builds = module.exports;
+
+let Path = require('node:path');
+
+let BuildsCacher = require('./builds-cacher.js');
+// let HostTargets = require('./build-classifier/host-targets.js');
+let Parallel = require('./parallel.js');
+
+var INSTALLERS_DIR = Path.join(__dirname, '..');
+var CACHE_DIR = Path.join(__dirname, '../_cache');
+
+let bc = BuildsCacher.create({
+  caches: CACHE_DIR,
+  installers: INSTALLERS_DIR,
+});
+bc.freshenRandomPackage(600 * 1000);
+
+Builds.init = async function () {
+  bc.freshenRandomPackage(600 * 1000);
+
+  let dirs = await bc.getProjectsByType();
+  let projNames = Object.keys(dirs.valid);
+
+  let parallel = 25;
+  await Parallel.run(parallel, projNames, getAll);
+  async function getAll(name) {
+    void (await bc.getPackages({
+      //Releases: Releases,
+      name: name,
+      date: new Date(),
+    }));
+  }
+};
+
+Builds.enumerateLatestVersions = bc.enumerateLatestVersions;
+Builds.findMatchingPackages = bc.findMatchingPackages;
+Builds.getPackage = bc.getPackages;
+Builds.getProjectType = bc.getProjectType;
+Builds.selectPackage = bc.selectPackage;

_webi/classify-one.js

@@ -0,0 +1,90 @@
+'use strict';
+
+let Path = require('node:path');
+
+// let Builds = require('./builds.js');
+let BuildsCacher = require('./builds-cacher.js');
+let Triplet = require('./build-classifier/triplet.js');
+
+async function main() {
+  let projName = process.argv[2];
+  if (!projName) {
+    console.error(``);
+    console.error(`USAGE`);
+    console.error(``);
+    console.error(`    classify-one <project-name>`);
+    console.error(``);
+    console.error(`EXAMPLE`);
+    console.error(``);
+    console.error(`    classify-one caddy`);
+    console.error(``);
+    return;
+  }
+
+  let tsDate = new Date(0);
+  let meta = {
+    // version info
+    versions: [],
+    lexvers: [],
+    lexversMap: {},
+    // culled release assets
+    packages: [],
+    releasesByTriplet: {},
+    // target info
+    triplets: [],
+    oses: [],
+    arches: [],
+    libcs: [],
+    formats: [],
+    // TODO channels: [],
+  };
+
+  let installersDir = Path.join(__dirname, '..');
+  let Releases = require(`${installersDir}/${projName}/releases.js`);
+  if (!Releases.latest) {
+    Releases.latest = Releases;
+  }
+
+  let projInfo = await Releases.latest();
+
+  // let packages = await Builds.getPackage({ name: projName });
+  // console.log(packages);
+
+  let bc = {};
+  bc.ALL_TERMS = Triplet.TERMS_PRIMARY_MAP;
+  bc.orphanTerms = Object.assign({}, bc.ALL_TERMS);
+  bc.unknownTerms = {};
+  bc.usedTerms = {};
+  bc.formats = [];
+  bc._targetsByBuildIdCache = {};
+  bc._triplets = {};
+
+  let transformed = BuildsCacher.transformAndUpdate(
+    projName,
+    projInfo,
+    meta,
+    tsDate,
+    bc,
+  );
+
+  console.log(`[DEBUG] transformed`);
+  let sample = transformed.packages.slice(0, 20);
+  console.log('packages:', sample, ':packages');
+  console.log(
+    'releasesByTriplet:',
+    transformed.releasesByTriplet['linux-x86_64-none'][transformed.versions[0]],
+    ':releasesByTriplet',
+  );
+  console.log('versions:', transformed.versions, ':versions');
+  console.log('triplets:', transformed.triplets, ':triplets');
+  console.log('oses:', transformed.oses, ':oses');
+  console.log('arches:', transformed.arches, ':arches');
+  console.log('libcs:', transformed.libcs, ':libcs');
+  console.log('formats:', transformed.formats, ':formats');
+  console.log(Object.keys(transformed));
+}
+
+main().catch(function (err) {
+  console.error('Error:');
+  console.error(err);
+});

_webi/curl-pipe-bootstrap.tpl.sh

@@ -27,18 +27,42 @@ fn_show_welcome() { (
     echo ""
     echo "    $(t_attn 'Success')? Star it!   $(t_url 'https://github.com/webinstall/webi-installers')"
     echo "    $(t_attn 'Problem')? Report it: $(t_url 'https://github.com/webinstall/webi-installers/issues')"
-    echo "                        $(t_dim "(your system is") $(t_host "$(uname -s)")/$(t_host "$(uname -m)") $(t_dim "with") $(t_host "$(fn_get_libc)") $(t_dim "&") $(t_host "$(fn_get_http_client_name)")$(t_dim ")")"
+    echo "                        $(t_dim "(your system is") $(t_host "$(fn_get_os)")/$(t_host "$(uname -m)") $(t_dim "with") $(t_host "$(fn_get_libc)") $(t_dim "&") $(t_host "$(fn_get_http_client_name)")$(t_dim ")")"
 
     sleep 0.2
 ); }
 
+fn_get_os() { (
+    # Ex:
+    #     GNU/Linux
+    #     Android
+    #     Linux (often Alpine, musl)
+    #     Darwin
+    b_os="$(uname -o 2> /dev/null || echo '')"
+    b_sys="$(uname -s)"
+    if test -z "${b_os}" || test "${b_os}" = "${b_sys}"; then
+        # ex: 'Darwin' (and plain, non-GNU 'Linux')
+        echo "${b_sys}"
+        return 0
+    fi
+
+    if echo "${b_os}" | grep -q "${b_sys}"; then
+        # ex: 'GNU/Linux'
+        echo "${b_os}"
+        return 0
+    fi
+
+    # ex: 'Android/Linux'
+    echo "${b_os}/${b_sys}"
+); }
+
 fn_get_libc() { (
     # Ex:
     #     musl
     #     libc
     if ldd /bin/ls 2> /dev/null | grep -q 'musl' 2> /dev/null; then
         echo 'musl'
-    elif uname -o | grep -q 'GNU' || uname -s | grep -q 'Linux'; then
+    elif fn_get_os | grep -q 'GNU|Linux'; then
         echo 'gnu'
     else
         echo 'libc'
@@ -177,9 +201,9 @@ fn_curl() { (
 
 fn_get_target_triple_user_agent() { (
     # Ex:
-    #     x86_64/unknown Linux/5.15.107-2-pve gnu
+    #     x86_64/unknown GNU/Linux/5.15.107-2-pve gnu
     #     arm64/unknown Darwin/22.6.0 libc
-    echo "$(uname -m)/unknown $(uname -s)/$(uname -r) $(fn_get_libc)"
+    echo "$(uname -m)/unknown $(fn_get_os)/$(uname -r) $(fn_get_libc)"
 ); }
 
 fn_download_to_path() { (
@@ -243,12 +267,23 @@ webi_bootstrap() { (
 fn_checksum() {
     a_filepath="${1}"
 
-    cmd_shasum='sha1sum'
-    if command -v shasum > /dev/null; then
-        cmd_shasum='shasum'
+    if command -v sha1sum > /dev/null; then
+        sha1sum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
+        return 0
     fi
 
-    $cmd_shasum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
+    if command -v shasum > /dev/null; then
+        shasum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
+        return 0
+    fi
+
+    if command -v sha1 > /dev/null; then
+        sha1 "${a_filepath}" | cut -d'=' -f2 | cut -c 2-9
+        return 0
+    fi
+
+    echo >&2 "    warn: no sha1 sum program"
+    date '+%F %H:%M'
 }
 
 ##############################################

_webi/installers.js

@@ -1,36 +1,19 @@
 'use strict';
 
+var Installers = module.exports;
+
 var Crypto = require('crypto');
 var Fs = require('node:fs/promises');
 var path = require('node:path');
-var request = require('@root/request');
-
-var _normalize = require('../_webi/normalize.js');
 
 var reInstallTpl = /\s*#?\s*{{ installer }}/;
 
-var Releases = module.exports;
-Releases.get = async function (pkgdir) {
-  let get;
-  try {
-    get = require(path.join(pkgdir, 'releases.js'));
-  } catch (e) {
-    let err = new Error('no releases.js for', pkgdir.split(/[\/\\]+/).pop());
-    err.code = 'E_NO_RELEASE';
-    throw err;
-  }
-
-  let all = await get(request);
-
-  return _normalize(all);
-};
-
 function padScript(txt) {
   return txt.replace(/^/g, '        ');
 }
 
 var BAD_SH_RE = /[<>'"`$\\]/;
-Releases.renderBash = async function (
+Installers.renderBash = async function (
   pkgdir,
   rel,
   { baseurl, pkg, tag, ver, os = '', arch = '', libc = '', formats },
@@ -90,7 +73,7 @@ Releases.renderBash = async function (
     .join(',')
     .replace(/'/g, '');
 
-  let webiChecksum = await Releases.getWebiShChecksum();
+  let webiChecksum = await Installers.getWebiShChecksum();
   let envReplacements = [
     ['WEBI_CHECKSUM', webiChecksum],
     ['WEBI_PKG', webiPkg],
@@ -99,7 +82,7 @@ Releases.renderBash = async function (
     ['WEBI_ARCH', arch],
     ['WEBI_LIBC', libc],
     ['WEBI_TAG', tag],
-    ['WEBI_RELEASES', `${baseurl}/${releaseUrl}`],
+    ['WEBI_RELEASES', `${baseurl}${releaseUrl}`],
     ['WEBI_CSV', releaseCsv],
     ['WEBI_VERSION', rel.version],
     ['WEBI_MAJOR', v.major],
@@ -110,16 +93,18 @@ Releases.renderBash = async function (
     ['WEBI_GIT_TAG', rel.git_tag], // TODO replace with branch
     ['WEBI_LTS', rel.lts],
     ['WEBI_CHANNEL', rel.channel],
-    ['WEBI_EXT', rel.ext.replace(/tar.*/, 'tar')],
+    ['WEBI_EXT', rel.ext],
     ['WEBI_FORMATS', formats.join(',')],
     ['WEBI_PKG_URL', rel.download],
     ['WEBI_PKG_PATHNAME', pkgFile],
     ['WEBI_PKG_FILE', pkgFile], // TODO replace with pathname
     ['PKG_NAME', pkg],
-    ['PKG_OSES', rel.oses],
-    ['PKG_ARCHES', rel.arches],
-    ['PKG_LIBCS', rel.libcs],
-    ['PKG_FORMATS', (rel.formats || []).join(',')],
+    ['PKG_STABLE', rel.stable],
+    ['PKG_LATEST', rel.latest],
+    ['PKG_OSES', (rel.oses || []).join(' ')],
+    ['PKG_ARCHES', (rel.arches || []).join(' ')],
+    ['PKG_LIBCS', (rel.libcs || []).join(' ')],
+    ['PKG_FORMATS', (rel.formats || []).join(' ')],
   ];
 
   for (let env of envReplacements) {
@@ -147,7 +132,7 @@ Releases.renderBash = async function (
   return tplTxt;
 };
 
-Releases.renderPowerShell = async function (
+Installers.renderPowerShell = async function (
   pkgdir,
   rel,
   { baseurl, pkg, tag, ver, os, arch, libc = '', formats },
@@ -224,7 +209,7 @@ var _webiShMeta = {
   checksum: '',
   mtime: 0,
 };
-Releases.getWebiShChecksum = async function () {
+Installers.getWebiShChecksum = async function () {
   let now = Date.now();
   let ago = now - _webiShMeta.updated_at;
   if (ago <= _webiShMeta.stale) {

_webi/lint-builds.js

@@ -0,0 +1,338 @@
+#!/usr/bin/env node
+'use strict';
+
+let Fs = require('node:fs/promises');
+let Path = require('node:path');
+
+let BuildsCacher = require('./builds-cacher.js');
+let HostTargets = require('./build-classifier/host-targets.js');
+let Parallel = require('./parallel.js');
+
+var INSTALLERS_DIR = Path.join(__dirname, '..');
+var CACHE_DIR = Path.join(__dirname, '../_cache');
+
+let UserAgentsMap = require('./build-classifier/uas.json');
+let uas = Object.keys(UserAgentsMap);
+let uaTargetsMap = {};
+for (let ua of uas) {
+  let terms = ua.split(/[\s\/]+/g);
+  let target = {};
+  void HostTargets.termsToTarget(target, terms);
+  if (!target) {
+    continue;
+  }
+  if (target.errors.length) {
+    throw target.errors[0];
+  }
+  if (!target.os) {
+    // TODO make target null, or create error for this
+    console.warn(`no os for terms: ${terms}`);
+    //throw new Error(`terms: ${terms}`);
+    continue;
+  }
+  if (!target.arch) {
+    // TODO make target null, or create error for this
+    console.warn(`no arch for terms: ${terms}`);
+    //throw new Error(`terms: ${terms}`);
+    continue;
+  }
+  if (!target.libc) {
+    // TODO make target null, or create error for this
+    console.warn(`no libc for terms: ${terms}`);
+    //throw new Error(`terms: ${terms}`);
+    continue;
+  }
+  let triplet = `${target.os}-${target.arch}-${target.libc}`;
+  uaTargetsMap[triplet] = target;
+}
+let uaTargets = [];
+let triplets = Object.keys(uaTargetsMap);
+for (let triplet of triplets) {
+  let target = uaTargetsMap[triplet];
+  uaTargets.push(target);
+}
+
+function showDirs(dirs) {
+  {
+    let errors = Object.keys(dirs.errors);
+    console.error('');
+    console.error(`Errors: ${errors.length}`);
+    for (let name of errors) {
+      let err = dirs.errors[name];
+      console.error(`${name}/: ${err.message}`);
+    }
+  }
+
+  {
+    let hidden = Object.keys(dirs.hidden);
+    console.debug('');
+    console.debug(`Hidden: ${hidden.length}`);
+    for (let name of hidden) {
+      let kind = dirs.hidden[name];
+      if (kind === '!directory') {
+        console.debug(`    ${name}`);
+      } else {
+        console.debug(`    ${name}/`);
+      }
+    }
+  }
+
+  {
+    let alias = Object.keys(dirs.alias);
+    console.debug('');
+    console.debug(`Alias: ${alias.length}`);
+    for (let name of alias) {
+      let kind = dirs.alias[name];
+      if (kind === 'symlink') {
+        console.debug(`    ${name} => ...`);
+      } else {
+        console.debug(`    ${name}/`);
+      }
+    }
+  }
+
+  {
+    let invalids = Object.keys(dirs.invalid);
+    console.warn('');
+    console.warn(`Invalid: ${invalids.length}`);
+    for (let name of invalids) {
+      console.warn(`    ${name}/`);
+    }
+  }
+
+  {
+    let selfhosted = Object.keys(dirs.selfhosted);
+    console.info('');
+    console.info(`Self-Hosted: ${selfhosted.length}`);
+    for (let name of selfhosted) {
+      console.info(`    ${name}/`);
+    }
+  }
+
+  {
+    let valids = Object.keys(dirs.valid);
+    console.info('');
+    console.info(`Found: ${valids.length}`);
+    for (let name of valids) {
+      console.info(`    ${name}/`);
+    }
+  }
+}
+
+let bc = BuildsCacher.create({
+  caches: CACHE_DIR,
+  installers: INSTALLERS_DIR,
+});
+
+async function main() {
+  /* jshint maxcomplexity: 25 */
+  // TODO
+  //     node ./_webi/lint-builds.js caddy@beta 'x86_64/unknown Darwin libc'
+  //
+  //let [projName, userAgent] = process.argv.slice(2);
+  let projName = process.argv[2];
+  // create test case for zoxide, goreleaser, go, yq, caddy, rg
+
+  let dirs = await bc.getProjectsByType();
+  if (!projName) {
+    showDirs(dirs);
+    console.info('');
+  }
+
+  bc.freshenRandomPackage(600 * 1000);
+
+  let rows = [];
+  let triples = [];
+  let valids = Object.keys(dirs.valid);
+
+  if (projName) {
+    if (!valids.includes(projName)) {
+      throw new Error(`'${projName}' is not a valid installable project`);
+    }
+    valids = [projName];
+  }
+  //valids = ['atomicparsley', 'caddy', 'macos'];
+  //valids = ['atomicparsley'];
+
+  console.info('');
+  console.info(`Fetching project release assets`);
+  let parallel = 25;
+  let projects = [];
+  await Parallel.run(parallel, valids, getAll);
+  async function getAll(name, i) {
+    console.info(`    ${name}`);
+    let projInfo = await bc.getPackages({
+      //Releases: Releases,
+      name: name,
+      date: new Date(),
+    });
+    projects[i] = projInfo;
+  }
+
+  console.info(`Classifying build assets for...`);
+  for (let projInfo of projects) {
+    console.info(`    ${projInfo.name}`);
+
+    let nStr = projInfo.releases.length.toString();
+    let n = nStr.padStart(5, ' ');
+    let row = `##### ${n}\t${projInfo.name}\tv`;
+    rows.push(row);
+
+    // ignore known, non-package extensions
+    for (let build of projInfo.releases) {
+      let target = bc.classify(projInfo, build);
+      if (!target) {
+        // non-build file
+        continue;
+      }
+
+      if (target.error) {
+        let e = target.error;
+        if (e.code === 'E_BUILD_NO_PATTERN') {
+          console.warn(`>>> ${e.message} <<<`);
+          console.warn(projInfo);
+          console.warn(build);
+          console.warn(`^^^ ${e.message} ^^^`);
+        }
+        throw e;
+      }
+
+      if (target.unknownTerms?.length) {
+        let msg = `${projInfo.name}: unrecognized term(s) '${target.unknownTerms}' in '${build.download}'`;
+        let err = new Error(msg);
+        throw err;
+      }
+
+      triples.push(target.triplet);
+      // if (!build.version) {
+      //   throw new Error(`no version for ${pkg.name} ${build.name}`);
+      // }
+      // // For debug printing versions
+      // console.error(build.version);
+      rows.push(`${target.triplet}\t${projInfo.name}\t${build.version}`);
+    }
+  }
+
+  console.info(`Fetching builds for`);
+  for (let projInfo of projects) {
+    console.info('');
+    console.info('');
+    console.info(`    ${projInfo.name}`);
+
+    for (let target of uaTargets) {
+      let libc = target.libc || 'libc';
+      let hostTriplet = `${target.os}-${target.arch}-${libc}`;
+      console.info('');
+      console.info(`    target: ${hostTriplet}`);
+      let match = bc.findMatchingPackages(projInfo, target, {
+        ver: '',
+      });
+      if (!match) {
+        console.info(
+          `    project: ${projInfo.name}: missing build for os '${target.os}'`,
+        );
+        continue;
+      }
+
+      if (!match.releases) {
+        console.info(
+          `    project: ${projInfo.name}: missing build for os '${target.os}-${target.arch}-${libc}'`,
+        );
+      } else if (match.triplet === hostTriplet) {
+        let releaseNames = Object.keys(match.releases);
+        console.info(`    selected ${releaseNames.length}`);
+      } else {
+        let releaseNames = Object.keys(match.releases);
+        console.info(
+          `    selected ${releaseNames.length} (${match.triplet} fallback)`,
+        );
+      }
+    }
+  }
+
+  let tsv = rows.join('\n');
+  console.info('');
+  console.info('#rows', rows.length);
+  await Fs.writeFile('builds.tsv', tsv, 'utf8');
+
+  console.info('');
+  console.info('Triplets Detected:');
+  let triplets = Object.keys(bc._triplets);
+  if (triplets.length) {
+    triplets.sort();
+    console.info('   ', triplets.join('\n    '));
+  } else {
+    console.info('    (none)');
+  }
+
+  console.info('');
+  console.info('New / Unknown Terms:');
+  let unknowns = Object.keys(bc.unknownTerms);
+  if (unknowns.length) {
+    unknowns.sort();
+    console.warn('   ', unknowns.join('\n    '));
+  } else {
+    console.info('    (none)');
+  }
+
+  console.info('');
+  console.info('Unused Terms:');
+  let unuseds = Object.keys(bc.orphanTerms);
+  if (unuseds.length) {
+    unuseds.sort();
+    console.warn('   ', unuseds.join('\n    '));
+  } else {
+    console.info('    (none)');
+  }
+
+  console.info('');
+  console.info('Formats:');
+  if (bc.formats.length) {
+    let formats = bc.formats.slice();
+    formats.sort();
+    if (!formats[0]) {
+      formats[0] = '(bin)';
+    }
+    console.warn('   ', formats.join('\n    '));
+  } else {
+    console.info('    (none)');
+  }
+
+  // sort -u -k1 builds.tsv | rg -v '^#|^https?:' | rg -i arm
+  // cut -f1 builds.tsv | sort -u -k1 | rg -v '^#|^https?:' | rg -i arm
+}
+
+if (module === require.main) {
+  let times = [];
+  let now = Date.now();
+  main()
+    .then(async function () {
+      let then = Date.now();
+      let delta = then - now;
+      times.push(delta);
+      now = then;
+      await main();
+      then = Date.now();
+      delta = then - now;
+      times.push(delta);
+    })
+    .then(function () {
+      console.info('');
+      console.info('Run times');
+      for (let delta of times) {
+        let s = delta / 1000;
+        console.info(`    ${s}`);
+      }
+
+      function forceExit() {
+        console.warn(`warn: dangling event loop reference`);
+        process.exit(0);
+      }
+      let exitTimeout = setTimeout(forceExit, 250);
+      exitTimeout.unref();
+    })
+    .catch(function (err) {
+      console.error(err.stack || err);
+      process.exit(1);
+    });
+}

_webi/normalize.js

@@ -57,7 +57,7 @@ var archMap = {
   amd64:
     /(\b|_|amd|(dar)?win(dows)?|mac(os)?|linux|osx|x)64([_\-]?bit)?(\b|_)/i,
   //x86: /(86)(\b|_)/i,
-  x86: /(\b|_|amd|(dar)?win(dows)?|mac(os)?|linux|osx|x)(86|32)([_\-]?bit)(\b|_)/i,
+  x86: /(\b|_|amd|(dar)?win(dows)?|mac(os)?|linux|osx|x)(86|32|i?386)([_\-]?bit)?(\b|_)/i,
   ppc64le: /(\b|_)(ppc64le)/i,
   ppc64: /(\b|_)(ppc64)(\b|_)/i,
   s390x: /(\b|_)(s390x)/i,
@@ -211,7 +211,8 @@ function normalize(all) {
       // won't match:
       //   - v1.0beta
       //   - v1.0-beta1b
-      let isBetaRe = /(\b|_)(preview|rc|beta|alpha)(\d+)(\b|_)/;
+      let isBetaRe =
+        /(\b|_)(alpha|beta|dev|developer|prev|preview|rc)(\d+)(\b|_)/;
       let isBeta = isBetaRe.test(rel.name);
       if (isBeta) {
         rel.channel = 'beta';

_webi/package-install.tpl.ps1

@@ -88,7 +88,7 @@ function Get-UserAgent {
     IF ($my_arch -eq "AMD64") {
         # Because PowerShell is sometimes AMD64 on Windows 10 ARM
         # See https://oofhours.com/2020/02/04/powershell-on-windows-10-arm64/
-        $my_os_arch = wmic os get osarchitecture
+        $my_os_arch = (Get-CimInstance -ClassName Win32_OperatingSystem).OSArchitecture
 
         # Using -clike because of the trailing newline
         IF ($my_os_arch -clike "ARM 64*") {

_webi/package-install.tpl.sh

@@ -26,6 +26,8 @@ __bootstrap_webi() {
     #PKG_ARCHES=
     #PKG_LIBCS=
     #PKG_FORMATS=
+    #PKG_LATEST=
+    #PKG_STABLE=
     WEBI_PKG_DOWNLOAD=""
     WEBI_DOWNLOAD_DIR="${HOME}/Downloads"
     if command -v xdg-user-dir > /dev/null; then
@@ -35,7 +37,7 @@ __bootstrap_webi() {
         fi
     fi
 
-    WEBI_PKG_PATH="${WEBI_DOWNLOAD_DIR}/webi/${PKG_NAME:-error}/${WEBI_VERSION:-latest}"
+    WEBI_PKG_PATH="${WEBI_DOWNLOAD_DIR}/webi/${PKG_NAME:-error}/${WEBI_VERSION:-stable}"
 
     # get the special formatted version
     # (i.e. "go is go1.14" while node is "node v12.10.8")
@@ -125,8 +127,16 @@ __bootstrap_webi() {
         {
             echo ""
             echo "    $(t_err "Error: no '${PKG_NAME:-"Unknown Package"}@${WEBI_TAG:-"Unknown Tag"}' release for '${WEBI_OS:-"Unknown OS"}' (${WEBI_LIBC:-"Unknown Libc"}) on '${WEBI_ARCH:-"Unknown CPU"}' as one of '${WEBI_FORMATS:-"Unknown File Type"}'")"
-            echo "      '$PKG_NAME' is available for '$PKG_OSES' ($PKG_LIBCS) on '$PKG_ARCHES' as one of '$PKG_FORMATS'"
-            echo "      (check that the package name and version are correct)"
+            echo ""
+            echo "        Latest Stable: ${PKG_STABLE}"
+            if test "${PKG_LATEST}" != "${PKG_STABLE}"; then
+                echo "        Next Version: ${PKG_LATEST}"
+            fi
+            echo "        CPUs: $PKG_ARCHES"
+            echo "        OSes: $PKG_OSES"
+            echo "        libcs: $PKG_LIBCS"
+            echo "        Package Formats: $PKG_FORMATS"
+            echo "        (check that the package name and version are correct)"
 
             echo ""
             my_release_url="$(echo "$WEBI_RELEASES" | sed 's:?.*::')"
@@ -191,20 +201,32 @@ __bootstrap_webi() {
         my_dl_rel="$(
             fn_sub_home "${WEBI_PKG_PATH}/${WEBI_PKG_FILE}"
         )"
-        if [ "tar" = "$WEBI_EXT" ]; then
+        if test "$WEBI_EXT" = "tar.zst"; then
+            echo "    Extracting $(t_path "${my_dl_rel}")"
+            unzstd -c --keep "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" | tar xf -
+        elif test "$WEBI_EXT" = "tar.xz"; then
+            echo "    Extracting $(t_path "${my_dl_rel}")"
+            unxz -c -k "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" | tar xf -
+        elif test "$WEBI_EXT" = "tar.gz"; then
+            echo "    Extracting $(t_path "${my_dl_rel}")"
+            tar xzf "${WEBI_PKG_PATH}/$WEBI_PKG_FILE"
+        elif test "$WEBI_EXT" = "tar.bz2"; then
+            echo "    Extracting $(t_path "${my_dl_rel}")"
+            tar xjf "${WEBI_PKG_PATH}/$WEBI_PKG_FILE"
+        elif test "$WEBI_EXT" = "tar"; then
             echo "    Extracting $(t_path "${my_dl_rel}")"
             tar xf "${WEBI_PKG_PATH}/$WEBI_PKG_FILE"
-        elif [ "zip" = "$WEBI_EXT" ]; then
+        elif test "$WEBI_EXT" = "zip" || test "$WEBI_EXT" = "app.zip"; then
             echo "    Extracting $(t_path "${my_dl_rel}")"
             unzip "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" > __unzip__.log
-        elif [ "exe" = "$WEBI_EXT" ]; then
+        elif test "$WEBI_EXT" = "exe"; then
             echo "    Moving $(t_path "${my_dl_rel}")"
             echo "      to $(t_path "$(fn_sub_home "$(pwd)")")"
             mv "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" .
-        elif [ "git" = "$WEBI_EXT" ]; then
+        elif test "$WEBI_EXT" = "git"; then
             echo "    Moving $(t_path "${my_dl_rel}")"
             mv "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" .
-        elif [ "xz" = "$WEBI_EXT" ]; then
+        elif test "$WEBI_EXT" = "xz"; then
             echo "    Inflating $(t_path "${my_dl_rel}")"
             unxz -c "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" > "$(basename "$WEBI_PKG_FILE")"
         else
@@ -217,8 +239,6 @@ __bootstrap_webi() {
     webi_path_add() {
         my_path="${1}"
 
-        fn_envman_init
-
         # \v was chosen as it is extremely unlikely for a filename
         # \1 could be an even better choice, but needs more testing.
         # (currently tested working on: linux & mac)
@@ -381,7 +401,6 @@ __bootstrap_webi() {
     export _webi_tmp="${_webi_tmp:-"$HOME/.local/opt/webi-tmp.d"}"
 
     mkdir -p "${WEBI_PKG_PATH}"
-    mkdir -p "$HOME/.local/bin"
     mkdir -p "$HOME/.local/opt"
 
     if test -e ~/.local/bin; then
@@ -390,6 +409,7 @@ __bootstrap_webi() {
         echo "    Creating$(t_path ' ~/.local/bin')"
         mkdir -p "$HOME/.local/bin"
     fi
+    fn_envman_init
 
     ##
     ##
@@ -533,18 +553,42 @@ fn_show_welcome_back() { (
     echo ""
     echo "    $(t_attn 'Success')? Star it!   $(t_url 'https://github.com/webinstall/webi-installers')"
     echo "    $(t_attn 'Problem')? Report it: $(t_url 'https://github.com/webinstall/webi-installers/issues')"
-    echo "                        $(t_dim "(your system is") $(t_host "$(uname -s)")/$(t_host "$(uname -m)") $(t_dim "with") $(t_host "$(fn_get_libc)") $(t_dim "&") $(t_host "$(fn_get_http_client_name)")$(t_dim ")")"
+    echo "                        $(t_dim "(your system is") $(t_host "$(fn_get_os)")/$(t_host "$(uname -m)") $(t_dim "with") $(t_host "$(fn_get_libc)") $(t_dim "&") $(t_host "$(fn_get_http_client_name)")$(t_dim ")")"
 
     sleep 0.2
 ); }
 
+fn_get_os() { (
+    # Ex:
+    #     GNU/Linux
+    #     Android
+    #     Linux (often Alpine, musl)
+    #     Darwin
+    b_os="$(uname -o 2> /dev/null || echo '')"
+    b_sys="$(uname -s)"
+    if test -z "${b_os}" || test "${b_os}" = "${b_sys}"; then
+        # ex: 'Darwin' (and plain, non-GNU 'Linux')
+        echo "${b_sys}"
+        return 0
+    fi
+
+    if echo "${b_os}" | grep -q "${b_sys}"; then
+        # ex: 'GNU/Linux'
+        echo "${b_os}"
+        return 0
+    fi
+
+    # ex: 'Android/Linux'
+    echo "${b_os}/${b_sys}"
+); }
+
 fn_get_libc() { (
     # Ex:
     #     musl
     #     libc
     if ldd /bin/ls 2> /dev/null | grep -q 'musl' 2> /dev/null; then
         echo 'musl'
-    elif uname -o | grep -q 'GNU' || uname -s | grep -q 'Linux'; then
+    elif fn_get_os | grep -q 'GNU|Linux'; then
         echo 'gnu'
     else
         echo 'libc'
@@ -683,9 +727,9 @@ fn_curl() { (
 
 fn_get_target_triple_user_agent() { (
     # Ex:
-    #     x86_64/unknown Linux/5.15.107-2-pve gnu
+    #     x86_64/unknown GNU/Linux/5.15.107-2-pve gnu
     #     arm64/unknown Darwin/22.6.0 libc
-    echo "$(uname -m)/unknown $(uname -s)/$(uname -r) $(fn_get_libc)"
+    echo "$(uname -m)/unknown $(fn_get_os)/$(uname -r) $(fn_get_libc)"
 ); }
 
 fn_download_to_path() { (
@@ -746,12 +790,23 @@ webi_upgrade() { (
 fn_checksum() {
     a_filepath="${1}"
 
-    cmd_shasum='sha1sum'
-    if command -v shasum > /dev/null; then
-        cmd_shasum='shasum'
+    if command -v sha1sum > /dev/null; then
+        sha1sum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
+        return 0
     fi
 
-    $cmd_shasum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
+    if command -v shasum > /dev/null; then
+        shasum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
+        return 0
+    fi
+
+    if command -v sha1 > /dev/null; then
+        sha1 "${a_filepath}" | cut -d'=' -f2 | cut -c 2-9
+        return 0
+    fi
+
+    echo >&2 "    warn: no sha1 sum program"
+    date '+%F %H:%M'
 }
 
 ##############################################
@@ -853,7 +908,7 @@ fn_envman_init_load_sh() { (
 touch -a ~/.config/envman/PATH.env
 touch -a ~/.config/envman/ENV.env
 touch -a ~/.config/envman/alias.env
-touch -a ~/.config/envman/function.env
+touch -a ~/.config/envman/function.sh
 
 # ENV first because we may use it in PATH
 test -z "\${ENVMAN_LOAD:-}" && . ~/.config/envman/ENV.env
@@ -862,7 +917,7 @@ test -z "\${ENVMAN_LOAD:-}" && . ~/.config/envman/PATH.env
 export ENVMAN_LOAD='loaded'
 
 # function first because we may use it in alias
-test -z "\${g_envman_load_sh:-}" && . ~/.config/envman/function.env
+test -z "\${g_envman_load_sh:-}" && . ~/.config/envman/function.sh
 test -z "\${g_envman_load_sh:-}" && . ~/.config/envman/alias.env
 
 g_envman_load_sh='loaded'
@@ -918,14 +973,14 @@ fn_envman_init_load_fish() { (
 touch -a ~/.config/envman/PATH.env
 touch -a ~/.config/envman/ENV.env
 touch -a ~/.config/envman/alias.env
-touch -a ~/.config/envman/function.env
+touch -a ~/.config/envman/function.fish
 
 not set -q ENVMAN_LOAD; and source ~/.config/envman/ENV.env
 not set -q ENVMAN_LOAD; and source ~/.config/envman/PATH.env
 
 set -x ENVMAN_LOAD 'loaded'
 
-not set -q g_envman_load_fish; and source ~/.config/envman/function.env
+not set -q g_envman_load_fish; and source ~/.config/envman/function.fish
 not set -q g_envman_load_fish; and source ~/.config/envman/alias.env
 
 set -g g_envman_load_fish 'loaded'

_webi/packages.js

@@ -1,96 +0,0 @@
-'use strict';
-
-var frontmarker = require('./frontmarker.js');
-var fs = require('fs');
-var path = require('path');
-
-var pkgs = module.exports;
-pkgs.create = function (Pkgs, basepath) {
-  if (!Pkgs) {
-    Pkgs = {};
-  }
-  if (!basepath) {
-    basepath = path.join(__dirname, '../');
-  }
-
-  Pkgs.all = function () {
-    return fs.promises.readdir(basepath).then(function (nodes) {
-      var items = [];
-      return nodes
-        .reduce(function (p, node) {
-          return p.then(function () {
-            return pkgs.get(node).then(function (meta) {
-              if (meta && '_' !== node[0]) {
-                meta.name = node;
-                items.push(meta);
-              }
-            });
-          });
-        }, Promise.resolve())
-        .then(function () {
-          return items;
-        });
-    });
-  };
-
-  Pkgs.get = function (node) {
-    return fs.promises.access(path.join(basepath, node)).then(function () {
-      return Pkgs._get(node);
-    });
-  };
-  Pkgs._get = function (node) {
-    var curlbash = path.join(basepath, node, 'install.sh');
-    var readme = path.join(basepath, node, 'README.md');
-    var winstall = path.join(basepath, node, 'install.ps1');
-    return Promise.all([
-      fs.promises
-        .readFile(readme, 'utf-8')
-        .then(function (txt) {
-          // TODO
-          return frontmarker.parse(txt);
-        })
-        .catch(function (e) {
-          if ('ENOENT' !== e.code && 'ENOTDIR' !== e.code) {
-            console.error("failed to read '" + node + "/README.md'");
-            console.error(e);
-          }
-        }),
-      fs.promises.access(curlbash).catch(function (e) {
-        // no *nix installer
-        curlbash = '';
-        if ('ENOENT' !== e.code && 'ENOTDIR' !== e.code) {
-          console.error("failed to parse '" + node + "/install.sh'");
-          console.error(e);
-        }
-      }),
-      fs.promises.access(winstall).catch(function (e) {
-        // no winstaller
-        winstall = '';
-        if ('ENOENT' !== e.code && 'ENOTDIR' !== e.code) {
-          console.error("failed to read '" + node + "/install.ps1'");
-          console.error(e);
-        }
-      }),
-    ]).then(function (items) {
-      var meta = items[0] || items[1];
-      if (!meta) {
-        // doesn't exist
-        return;
-      }
-      meta.windows = !!winstall;
-      meta.bash = !!curlbash;
-
-      return meta;
-    });
-  };
-
-  return Pkgs;
-};
-pkgs.create(pkgs);
-
-if (module === require.main) {
-  pkgs.all().then(function (data) {
-    console.info('package info:');
-    console.info(data);
-  });
-}

_webi/packages.js

@@ -0,0 +1 @@
+projects.js

_webi/parallel.js

@@ -0,0 +1,44 @@
+'use strict';
+
+var Parallel = module.exports;
+Parallel.run = async function (limit, arr, fn) {
+  let index = 0;
+  let actives = [];
+  let results = [];
+  limit = Math.min(limit, arr.length);
+
+  function launch() {
+    let _index = index;
+    let p = fn(arr[_index], _index, arr);
+
+    // some tasks may be synchronous
+    // so we must push before removing
+    actives.push(p);
+
+    p.then(function _resolve(result) {
+      let i = actives.indexOf(p);
+      actives.splice(i, 1);
+      results[_index] = result;
+    });
+
+    index += 1;
+  }
+
+  // start tasks in parallel, up to limit
+  for (; actives.length < limit; ) {
+    launch();
+  }
+
+  // keep the task queue full
+  for (; index < arr.length; ) {
+    // wait for one task to complete
+    await Promise.race(actives);
+    // add one task again
+    launch();
+  }
+
+  // wait for all remaining tasks
+  await Promise.all(actives);
+
+  return results;
+};

_webi/projects.js

@@ -0,0 +1,96 @@
+'use strict';
+
+var frontmarker = require('./frontmarker.js');
+var fs = require('fs');
+var path = require('path');
+
+var pkgs = module.exports;
+pkgs.create = function (Pkgs, basepath) {
+  if (!Pkgs) {
+    Pkgs = {};
+  }
+  if (!basepath) {
+    basepath = path.join(__dirname, '../');
+  }
+
+  Pkgs.all = function () {
+    return fs.promises.readdir(basepath).then(function (nodes) {
+      var items = [];
+      return nodes
+        .reduce(function (p, node) {
+          return p.then(function () {
+            return pkgs.get(node).then(function (meta) {
+              if (meta && '_' !== node[0]) {
+                meta.name = node;
+                items.push(meta);
+              }
+            });
+          });
+        }, Promise.resolve())
+        .then(function () {
+          return items;
+        });
+    });
+  };
+
+  Pkgs.get = function (node) {
+    return fs.promises.access(path.join(basepath, node)).then(function () {
+      return Pkgs._get(node);
+    });
+  };
+  Pkgs._get = function (node) {
+    var curlbash = path.join(basepath, node, 'install.sh');
+    var readme = path.join(basepath, node, 'README.md');
+    var winstall = path.join(basepath, node, 'install.ps1');
+    return Promise.all([
+      fs.promises
+        .readFile(readme, 'utf-8')
+        .then(function (txt) {
+          // TODO
+          return frontmarker.parse(txt);
+        })
+        .catch(function (e) {
+          if ('ENOENT' !== e.code && 'ENOTDIR' !== e.code) {
+            console.error("failed to read '" + node + "/README.md'");
+            console.error(e);
+          }
+        }),
+      fs.promises.access(curlbash).catch(function (e) {
+        // no *nix installer
+        curlbash = '';
+        if ('ENOENT' !== e.code && 'ENOTDIR' !== e.code) {
+          console.error("failed to parse '" + node + "/install.sh'");
+          console.error(e);
+        }
+      }),
+      fs.promises.access(winstall).catch(function (e) {
+        // no winstaller
+        winstall = '';
+        if ('ENOENT' !== e.code && 'ENOTDIR' !== e.code) {
+          console.error("failed to read '" + node + "/install.ps1'");
+          console.error(e);
+        }
+      }),
+    ]).then(function (items) {
+      var meta = items[0] || items[1];
+      if (!meta) {
+        // doesn't exist
+        return;
+      }
+      meta.windows = !!winstall;
+      meta.bash = !!curlbash;
+
+      return meta;
+    });
+  };
+
+  return Pkgs;
+};
+pkgs.create(pkgs);
+
+if (module === require.main) {
+  pkgs.all().then(function (data) {
+    console.info('package info:');
+    console.info(data);
+  });
+}

_webi/serve-installer.js

@@ -1,19 +1,16 @@
 'use strict';
 
-var Installers = module.exports;
+var InstallerServer = module.exports;
 
-var Fs = require('fs/promises');
-var path = require('path');
+let Fs = require('fs/promises');
+let Path = require('path');
 
-var uaDetect = require('./ua-detect.js');
-var packages = require('./packages.js');
-var Releases = require('./releases.js');
+let HostTargets = require('./build-classifier/host-targets.js');
+let Builds = require('./builds.js');
+let Installers = require('./installers.js');
 
-// handlers caching and transformation, probably should be broken down
-var getReleases = require('./transform-releases.js');
-
-Installers.INSTALLERS_DIR = path.join(__dirname, '..');
-Installers.serveInstaller = async function (
+InstallerServer.INSTALLERS_DIR = Path.join(__dirname, '..');
+InstallerServer.serveInstaller = async function (
   baseurl,
   ua,
   pkg,
@@ -22,120 +19,221 @@ Installers.serveInstaller = async function (
   formats,
   libc,
 ) {
-  let [rel, opts] = await Installers.helper({
-    ua,
-    pkg,
+  let unameAgent = ua;
+  let projectName = pkg;
+  let [rel, tmplParams] = await InstallerServer.helper({
+    unameAgent,
+    projectName,
     tag,
     formats,
     libc,
   });
-  Object.assign(opts, {
+  Object.assign(tmplParams, {
     baseurl,
   });
 
-  var pkgdir = path.join(Installers.INSTALLERS_DIR, pkg);
+  var pkgdir = Path.join(InstallerServer.INSTALLERS_DIR, projectName);
   if ('ps1' === ext) {
-    return Releases.renderPowerShell(pkgdir, rel, opts);
+    return Installers.renderPowerShell(pkgdir, rel, tmplParams);
   }
-  return Releases.renderBash(pkgdir, rel, opts);
+  return Installers.renderBash(pkgdir, rel, tmplParams);
 };
-Installers.helper = async function ({ ua, pkg, tag, formats, libc }) {
-  // TODO put some of this in a middleware? or common function?
 
-  // TODO maybe move package/version/lts/channel detection into getReleases
-  var ver = tag.replace(/^v/, '');
-  var lts;
-  var channel;
+// TODO put some of this in a middleware? or common function?
+// TODO maybe move package/version/lts/channel detection into getReleases
+InstallerServer.helper = async function ({
+  unameAgent,
+  projectName,
+  tag,
+  formats,
+  libc,
+}) {
+  console.log(`dbg: Installer User-Agent: ${unameAgent}`);
 
-  switch (ver) {
-    case 'latest':
-      ver = '';
-      channel = 'stable';
-      break;
-    case 'lts':
-      lts = true;
-      channel = 'stable';
-      ver = '';
-      break;
-    case 'stable':
-      channel = 'stable';
-      ver = '';
-      break;
-    case 'beta':
-      channel = 'beta';
-      ver = '';
-      break;
-    case 'dev':
-      channel = 'dev';
-      ver = '';
-      break;
+  let releaseTarget = toReleaseTarget(tag);
+  let hostFormats = formats;
+  let terms = unameAgent.split(/[\s\/]+/g);
+  let hostTarget = {};
+  try {
+    void HostTargets.termsToTarget(hostTarget, terms);
+  } catch (e) {
+    // if we can't guarantee the results...
+    // "in the face of ambiguity, refuse the temptation to guess"
+    throw e;
+  }
+  console.log(`dbg: Installer Host Target:`);
+  console.log(hostTarget);
+
+  if (!hostTarget.os) {
+    throw new Error(`OS could not be identified by User-Agent '${unameAgent}'`);
   }
 
-  var myOs = uaDetect.os(ua);
-  var myArch = uaDetect.arch(ua);
-  var myLibc;
-  if (libc) {
-    myLibc = uaDetect.libc(libc);
+  console.log(`dbg: Get Project Installer Type for '${projectName}':`);
+  let proj = await Builds.getProjectType(projectName);
+  if (proj.type === 'alias') {
+    console.log(`dbg: alias`, proj);
+    projectName = proj.detail;
+    proj = await Builds.getProjectType(projectName); // an alias should never resolve to an alias
   }
-  if (!myLibc) {
-    myLibc = uaDetect.libc(ua);
-  }
-  if (!myLibc) {
-    myLibc = 'libc';
+  console.log(`dbg: proj`, proj);
+
+  let validTypes = ['selfhosted', 'valid'];
+  if (!validTypes.includes(proj.type)) {
+    let msg = `'${projectName}' doesn't have an installer: '${proj.type}': '${proj.detail}'`;
+    let err = new Error(msg);
+    err.code = 'ENOENT';
+    throw err;
   }
 
-  let cfg = await packages.get(pkg);
-  let releaseQuery = {
-    pkg: cfg.alias || pkg,
-    ver,
-    os: myOs,
-    arch: myArch,
-    libc: myLibc,
-    lts,
-    channel,
-    // TODO use formats for sorting, not exclusion
-    // (it's better to install xz or report an error to install zip)
-    formats,
+  let tmplParams = {
+    pkg: projectName,
+    tag: tag,
+    os: hostTarget.os,
+    arch: hostTarget.arch,
+    libc: hostTarget.libc,
+    formats: hostFormats,
     limit: 1,
   };
+  Object.assign(tmplParams, releaseTarget);
+  console.log('tmplParams', tmplParams);
 
-  let rels = await getReleases(releaseQuery);
-
-  var rel = rels.releases[0];
-  var opts = {
-    pkg: cfg.alias || pkg,
-    ver,
-    tag,
-    os: myOs,
-    arch: myArch,
-    libc: myLibc,
-    lts,
-    channel,
-    formats,
-    limit: 1,
+  let errPackage = {
+    name: 'doesntexist.ext',
+    version: '0.0.0',
+    lts: '-',
+    channel: 'error',
+    date: '1970-01-01',
+    os: hostTarget.os || '-',
+    arch: hostTarget.arch || '-',
+    libc: hostTarget.libc || '-',
+    ext: 'err',
+    download: 'https://example.com/doesntexist.ext',
+    comment:
+      'No matches found. Could be bad or missing version info' +
+      ',' +
+      "Check query parameters. Should be something like '/api/releases/{package}@{version}.tab?os={macos|linux|windows|-}&arch={amd64|x86|aarch64|arm64|armv7l|-}&libc={musl|gnu|msvc|libc|static}&limit=10'",
   };
-  rel = Object.assign(
-    {
-      oses: rels.oses,
-      arches: rels.arches,
-      libcs: rels.libcs,
-      formats: rels.formats,
-    },
-    rel,
+
+  if (proj.type === 'selfhosted') {
+    return [errPackage, tmplParams];
+  }
+
+  let projInfo = await Builds.getPackage({
+    name: projectName,
+    date: new Date(),
+  });
+  let latestVersions = Builds.enumerateLatestVersions(projInfo);
+  //console.log('projInfo', projInfo);
+
+  let buildTargetInfo = {
+    triplets: projInfo.triplets,
+    oses: projInfo.oses,
+    arches: projInfo.arches,
+    libcs: projInfo.libcs,
+    formats: projInfo.formats,
+    latest: latestVersions.latest,
+    stable: latestVersions.stable,
+  };
+
+  // TODO .findMatchingPackages() should probably account for this
+  let hasOs = projInfo.oses.includes(hostTarget.os);
+  let maybePosix = !hasOs && hostTarget.os !== 'windows';
+  if (maybePosix) {
+    let posixes = ['posix_2017', 'posix_2024'];
+    for (let posixYear of posixes) {
+      let hasPosix = projInfo.oses.includes(posixYear);
+      if (hasPosix) {
+        hasOs = true;
+        break;
+      }
+    }
+  }
+  if (!hasOs) {
+    hasOs = projInfo.oses.includes('ANYOS');
+  }
+
+  if (!hasOs) {
+    let pkg1 = Object.assign(buildTargetInfo, errPackage);
+    return [pkg1, tmplParams];
+  }
+
+  let targetRelease = Builds.findMatchingPackages(
+    projInfo,
+    hostTarget,
+    releaseTarget,
   );
+  // { triplet: `${os}-${arch}-${libc}`, packages: targetPackages
+  // , latest: projInfo.versions[0], versions: matchInfo
+  // }
 
-  return [rel, opts];
+  if (!targetRelease?.packages) {
+    let pkg1 = Object.assign(buildTargetInfo, errPackage);
+    return [pkg1, tmplParams];
+  }
+
+  let buildPkg = Builds.selectPackage(targetRelease.packages, hostFormats);
+  let ext = buildPkg.ext || '.exe';
+  if (ext.startsWith('.')) {
+    ext = ext.slice(1);
+  }
+
+  let version = targetRelease.version;
+  if (version.startsWith('v')) {
+    version = version.slice(1);
+  }
+
+  buildPkg = Object.assign(buildTargetInfo, buildPkg, { ext, version });
+  console.log('dbg: buildPkg', buildPkg);
+  console.log('dbg: tmplParams', tmplParams);
+  return [buildPkg, tmplParams];
 };
 
-var CURL_PIPE_PS1_BOOT = path.join(__dirname, 'curl-pipe-bootstrap.tpl.ps1');
-var CURL_PIPE_SH_BOOT = path.join(__dirname, 'curl-pipe-bootstrap.tpl.sh');
+let channelNames = [
+  'stable',
+  // 'hotfix',
+  'latest',
+  'rc',
+  'preview',
+  'pre',
+  'dev',
+  'beta',
+  'alpha',
+];
+
+function toReleaseTarget(tag) {
+  tag = tag.replace(/^v/, '');
+
+  let releaseTarget = {
+    channel: '',
+    lts: false,
+    version: '',
+  };
+
+  if (tag === 'lts') {
+    releaseTarget.lts = true;
+    releaseTarget.channel = 'stable';
+  } else if (channelNames.includes(tag)) {
+    releaseTarget.channel = tag;
+  } else {
+    releaseTarget.version = tag;
+  }
+
+  return releaseTarget;
+}
+
+var CURL_PIPE_PS1_BOOT = Path.join(__dirname, 'curl-pipe-bootstrap.tpl.ps1');
+var CURL_PIPE_SH_BOOT = Path.join(__dirname, 'curl-pipe-bootstrap.tpl.sh');
 var BAD_SH_RE = /[<>'"`$\\]/;
 
-Installers.getPosixCurlPipeBootstrap = async function ({ baseurl, pkg, ver }) {
+InstallerServer.getPosixCurlPipeBootstrap = async function ({
+  baseurl,
+  pkg,
+  ver,
+}) {
   let bootTxt = await Fs.readFile(CURL_PIPE_SH_BOOT, 'utf8');
 
   var webiPkg = [pkg, ver].filter(Boolean).join('@');
-  var webiChecksum = await Releases.getWebiShChecksum();
+  var webiChecksum = await Installers.getWebiShChecksum();
   var envReplacements = [
     ['WEBI_PKG', webiPkg],
     ['WEBI_HOST', baseurl],
@@ -146,7 +244,6 @@ Installers.getPosixCurlPipeBootstrap = async function ({ baseurl, pkg, ver }) {
     let name = env[0];
     let value = env[1];
 
-    // TODO create REs once, in higher scope
     let envRe = new RegExp(
       `^[ \\t]*#?[ \\t]*(export[ \\t])?[ \\t]*(${name})=.*`,
       'm',
@@ -164,7 +261,7 @@ Installers.getPosixCurlPipeBootstrap = async function ({ baseurl, pkg, ver }) {
   return bootTxt;
 };
 
-Installers.getPwshCurlPipeBootstrap = async function ({
+InstallerServer.getPwshCurlPipeBootstrap = async function ({
   baseurl,
   pkg,
   ver,
@@ -173,7 +270,7 @@ Installers.getPwshCurlPipeBootstrap = async function ({
   let bootTxt = await Fs.readFile(CURL_PIPE_PS1_BOOT, 'utf8');
 
   var webiPkg = [pkg, ver].filter(Boolean).join('@');
-  //var webiChecksum = await Releases.getWebiPs1Checksum();
+  //var webiChecksum = await InstallerServer.getWebiPs1Checksum();
   var envReplacements = [
     ['Env:WEBI_PKG', webiPkg],
     ['Env:WEBI_HOST', baseurl],
@@ -194,9 +291,6 @@ Installers.getPwshCurlPipeBootstrap = async function ({
     let tplRe = new RegExp(`{{ (${name}) }}`, 'g');
     bootTxt = bootTxt.replace(tplRe, `${value}`);
 
-    // let envRe = new RegExp(`^[ \\t]*#?[ \\t]*($$${name})[ \\t]*=.*`, 'im');
-    // bootTxt = bootTxt.replace(envRe, `$$${name} = '${value}'`);
-
     let setRe = new RegExp(
       `(#[ \\t]*)?(\\$${name})[ \\t]*=[ \\t]['"].*['"][ \\t]`,
       'im',

_webi/test.js

@@ -32,7 +32,8 @@ if (/\b-?-h(elp)?\b/.test(process.argv.join(' '))) {
 var os = require('os');
 var fs = require('fs');
 var path = require('path');
-var Releases = require('./releases.js');
+var Builds = require('./builds.js');
+var Installers = require('./installers.js');
 var ServeInstaller = require('./serve-installer.js');
 
 var pkg = process.argv[2].split('@');
@@ -64,7 +65,8 @@ console.info('Has the necessary files?');
   });
 
 console.info('');
-Releases.get(path.join(process.cwd(), pkgdir)).then(async function (all) {
+let projName = pkgdir.split('/').filter(Boolean).pop();
+Builds.getPackage({ name: projName }).then(async function (/*projInfo*/) {
   var pkgname = path.basename(pkgdir.replace(/\/$/, ''));
   var nodeOs = os.platform();
   var nodeOsRelease = os.release();
@@ -81,8 +83,8 @@ Releases.get(path.join(process.cwd(), pkgdir)).then(async function (all) {
   var formats = ['exe', 'xz', 'tar', 'zip', 'git'];
 
   let [rel, opts] = await ServeInstaller.helper({
-    ua: `${nodeOs}/${nodeOsRelease} ${nodeArch}/unknown ${nodeLibc}`,
-    pkg: pkgname,
+    unameAgent: `${nodeOs}/${nodeOsRelease} ${nodeArch}/unknown ${nodeLibc}`,
+    projectName: pkgname,
     tag: pkgtag || '',
     formats: formats,
     libc: nodeLibc,
@@ -116,8 +118,8 @@ Releases.get(path.join(process.cwd(), pkgdir)).then(async function (all) {
   console.info('');
 
   return Promise.all([
-    Releases.renderBash(pkgdir, rel, opts).catch(function () {}),
-    Releases.renderPowerShell(pkgdir, rel, opts).catch(function () {}),
+    Installers.renderBash(pkgdir, rel, opts).catch(function () {}),
+    Installers.renderPowerShell(pkgdir, rel, opts).catch(function () {}),
   ]).then(function (scripts) {
     var bashTxt = scripts[0];
     var ps1Txt = scripts[1];

_webi/transform-releases.js

@@ -1,7 +1,9 @@
 'use strict';
 
+var Releases = module.exports;
+
 var path = require('path');
-var Releases = require('./releases.js');
+var _normalize = require('./normalize.js');
 
 var cache = {};
 //var staleAge = 5 * 1000;
@@ -11,6 +13,25 @@ var expiredAge = 15 * 60 * 1000;
 
 let installerDir = path.join(__dirname, '..');
 
+Releases.get = async function (pkgdir) {
+  let get;
+  try {
+    get = require(`${pkgdir}/releases.js`);
+    // TODO update all releases files with module.exports.xxxx = 'foo';
+    if (!get.latest) {
+      get.latest = get;
+    }
+  } catch (e) {
+    let err = new Error('no releases.js for', pkgdir.split(/[\/\\]+/).pop());
+    err.code = 'E_NO_RELEASE';
+    throw err;
+  }
+
+  let all = await get.latest();
+
+  return _normalize(all);
+};
+
 // TODO needs a proper test, and more accurate (though perhaps far less simple) code
 function createFormatsSorter(formats) {
   return function sortByVerExt(a, b) {
@@ -122,7 +143,7 @@ async function getCachedReleases(pkg) {
           cache[pkg].all = all;
           complete = true;
         }),
-      sleep(5000).then(function () {
+      sleep(15000).then(function () {
         if (complete) {
           return;
         }
@@ -242,7 +263,7 @@ async function filterReleases(
   return sortedRels.slice(0, limit || 1000);
 }
 
-module.exports = function getReleases({
+Releases.getReleases = function ({
   _count,
   pkg,
   ver,
@@ -290,7 +311,7 @@ module.exports = function getReleases({
         if (_count < 1) {
           // Apple Silicon M1 hacky-do workaround fix
           if ('macos' === os && 'arm64' === arch) {
-            return getReleases({
+            return Releases.getReleases({
               pkg,
               ver,
               os,
@@ -304,7 +325,7 @@ module.exports = function getReleases({
           }
           // Windows ARM hacky-do workaround fix
           if ('windows' === os && 'arm64' === arch) {
-            return getReleases({
+            return Releases.getReleases({
               pkg,
               ver,
               os,
@@ -318,7 +339,7 @@ module.exports = function getReleases({
           }
           // Raspberry Pi 3+ on Ubuntu arm64 (via Bionic?)
           if ('linux' === os && 'arm64' === arch) {
-            return getReleases({
+            return Releases.getReleases({
               _count: _count + 1,
               pkg,
               ver,
@@ -333,7 +354,7 @@ module.exports = function getReleases({
           }
           // armv7 can run armv6
           if ('linux' === os && 'armv7l' === arch) {
-            return getReleases({
+            return Releases.getReleases({
               _count: _count + 1,
               pkg,
               ver,
@@ -351,7 +372,7 @@ module.exports = function getReleases({
           // Raspberry Pi 3+ on Raspbian arm7 (not Ubuntu arm64)
           // hail mary
           if ('linux' === os && 'armv7l' === arch) {
-            return getReleases({
+            return Releases.getReleases({
               _count: _count + 1,
               pkg,
               ver,

aliasman/releases.js

@@ -1,29 +1,21 @@
 'use strict';
 
-var githubSource = require('../_common/github-source.js');
-var owner = 'BeyondCodeBootcamp';
-var repo = 'aliasman';
+let Releases = module.exports;
 
-module.exports = function (request) {
-  let arches = [
-    'amd64',
-    'arm64',
-    'armv6l',
-    'armv7l',
-    'ppc64le',
-    'ppc64',
-    's390x',
-    'x86',
-  ];
-  let oses = ['freebsd', 'linux', 'macos', 'posix'];
-  return githubSource(request, owner, repo, oses, arches).then(function (all) {
-    all._names = ['aliasman', 'legacy'];
-    return all;
-  });
+let GitHubSource = require('../_common/github-source.js');
+let owner = 'BeyondCodeBootcamp';
+let repo = 'aliasman';
+
+Releases.latest = async function () {
+  let all = await GitHubSource.getDistributables({ owner, repo });
+  for (let pkg of all.releases) {
+    pkg.os = 'posix_2017';
+  }
+  return all;
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  Releases.latest().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all, null, 2));
   });

arc/releases.js

@@ -4,15 +4,15 @@ var github = require('../_common/github.js');
 var owner = 'mholt';
 var repo = 'archiver';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     all._names = ['archiver', 'arc'];
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 5 for demonstration
     all.releases = all.releases.slice(0, 5);

atomicparsley/releases.js

@@ -32,8 +32,8 @@ let targets = {
   },
 };
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     for (let rel of all.releases) {
       let windows32 = rel.name.includes('WindowsX86.');
       if (windows32) {
@@ -71,7 +71,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all));
     //console.info(JSON.stringify(all, null, 2));

awless/releases.js

@@ -4,8 +4,8 @@ var github = require('../_common/github.js');
 var owner = 'wallix';
 var repo = 'awless';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     // remove checksums and .deb
     all.releases = all.releases.filter(function (rel) {
       return !/(\.txt)|(\.deb)$/i.test(rel.name);
@@ -15,7 +15,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all));
   });

bat/releases.js

@@ -4,14 +4,14 @@ var github = require('../_common/github.js');
 var owner = 'sharkdp';
 var repo = 'bat';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     all.releases = all.releases.slice(0, 10);
     //console.info(JSON.stringify(all));

brew/brew-update-service-install

@@ -6,15 +6,23 @@ main() { (
     sed '1,/^#~\/.local\/bin\/brew-updater/d' "${0}" > ~/.local/bin/brew-update-hourly
     chmod a+x ~/.local/bin/brew-update-hourly
 
-    env PATH="$PATH" serviceman add --user \
+    echo "Checking for serviceman..."
+    ~/.local/bin/webi serviceman
+    if ! command -v serviceman > /dev/null; then
+        export PATH="$HOME/.local/bin:$PATH"
+    fi
+    serviceman --version
+
+    serviceman add --agent \
         --workdir ~/.local/opt/brew/ \
         --name sh.brew.updater -- \
         ~/.local/bin/brew-update-hourly
 ); }
 
-if main; then
-    exit 0
+if ! main; then
+    exit 1
 fi
+exit 0
 
 #~/.local/bin/brew-updater
 #!/bin/sh

bun/README.md

@@ -132,9 +132,7 @@ file)
    ```
 3. Add your project to the system launcher, running as the current user
    ```sh
-   sudo env PATH="$PATH" \
-       serviceman add --path="$PATH" --system \
-           --username "$(whoami)" --name my-project -- \
+   serviceman add --name 'my-project' --daemon -- \
        bun run ./my-project.js
    ```
 4. Restart the logging service
@@ -155,6 +153,6 @@ For **macOS**:
    ```
 3. Add your project to the system launcher, running as the current user
    ```sh
-   serviceman add --path="$PATH" --user --name my-project -- \
+   serviceman add --agent --name 'my-project' -- \
        bun run ./my-project.js
    ```

bun/releases.js

@@ -4,8 +4,8 @@ var github = require('../_common/github.js');
 var owner = 'oven-sh';
 var repo = 'bun';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     all.releases = all.releases
       .filter(function (r) {
         let isDebug = r.name.includes('-profile');
@@ -18,6 +18,14 @@ module.exports = function (request) {
           return false;
         }
 
+        let isMusl = r.name.includes('-musl');
+        if (isMusl) {
+          r._musl = true;
+          r.libc = 'musl';
+        } else if (r.os === 'linux') {
+          r.libc = 'gnu';
+        }
+
         return true;
       })
       .map(function (r) {
@@ -30,7 +38,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 5 for demonstration
     all.releases = all.releases.slice(0, 5);

caddy/README.md

@@ -819,10 +819,10 @@ To avoid the nitty-gritty details of `launchd` plist files, you can use
 2. Use Serviceman to create a _launchd_ plist file
 
    ```sh
-   my_username="$( id -u -n )"
+   my_username="$(id -u -n)"
 
-   serviceman add --user --name caddy -- \
-       caddy run --config ./Caddyfile --envfile ~/.config/caddy/env
+   serviceman add --agent --name 'caddy' --workdir ./ -- \
+       caddy run --envfile ~/.config/caddy/env --config ./Caddyfile --adapter caddyfile
    ```
 
    (this will create `~/Library/LaunchAgents/caddy.plist`)
@@ -837,8 +837,8 @@ This process creates a _User-Level_ service in `~/Library/LaunchAgents`. To
 create a _System-Level_ service in `/Library/LaunchDaemons/` instead:
 
 ```sh
-sudo serviceman add --system --name caddy -- \
-   caddy run --config ./Caddyfile --envfile ~/.config/caddy/env
+serviceman add --name 'caddy' --workdir ./ --daemon -- \
+   caddy run --envfile ~/.config/caddy/env --config ./Caddyfile --adapter caddyfile
 ```
 
 ### How to run Caddy as a Windows Service
@@ -856,7 +856,7 @@ sudo serviceman add --system --name caddy -- \
 3. Create a **Startup Registry Entry** with Serviceman.
    ```sh
    serviceman.exe add --name caddy -- \
-       caddy run --config ./Caddyfile --envfile ~/.config/caddy/env
+       caddy run --envfile ~/.config/caddy/env --config ./Caddyfile --adapter caddyfile
    ```
 4. You can manage the service directly with Serviceman. For example:
    ```sh
@@ -901,10 +901,8 @@ See the notes below to run as a **User Service** or use the JSON Config.
    ```
 4. Use Serviceman to create a _systemd_ config file.
    ```sh
-   my_username="$( id -u -n )"
-   sudo env PATH="$PATH" \
-       serviceman add --system --username "${my_username}" --name caddy -- \
-           caddy run --config ./Caddyfile --envfile ~/.config/caddy/env
+   serviceman add --name 'caddy' --daemon -- \
+       caddy run --envfile ~/.config/caddy/env --config ./Caddyfile --adapter caddyfile
    ```
    (this will create `/etc/systemd/system/caddy.service`)
 5. Manage the service with `systemctl` and `journalctl`:
@@ -915,10 +913,10 @@ See the notes below to run as a **User Service** or use the JSON Config.
 
 To create a **User Service** instead:
 
-- don't use `sudo`, but do use `--user` when running `serviceman`:
+- use `--agent` when running `serviceman`:
   ```sh
-  serviceman add --user --name caddy -- \
-     caddy run --config ./Caddyfile --envfile ~/.config/caddy/env
+  serviceman add --agent --name caddy -- \
+      caddy run --envfile ~/.config/caddy/env --config ./Caddyfile --adapter caddyfile
   ```
   (this will create `~/.config/systemd/user/`)
 - user the `--user` flag to manage services and logs:
@@ -1363,19 +1361,13 @@ See also: <https://caddyserver.com/docs/running>
 2. Generate the `service` file: \
    - JSON Config
      ```sh
-     my_app_user="$( id -u -n )"
-     sudo env PATH="${PATH}" \
-         serviceman add --system --cap-net-bind \
-             --username "${my_app_user}" --name caddy -- \
-             caddy run --resume --envfile ./caddy.env
+     serviceman add --name 'caddy' --daemon -- \
+         caddy run --resume --envfile ./caddy.env
      ```
    - Caddyfile
      ```sh
-     my_app_user="$( id -u -n )"
-     sudo env PATH="${PATH}" \
-         serviceman add --system --cap-net-bind \
-             --username "${my_app_user}" --name caddy -- \
-             caddy run --config ./Caddyfile --envfile ./caddy.env
+     serviceman add --name 'caddy' --daemon -- \
+         caddy run --config ./Caddyfile --envfile ./caddy.env
      ```
 3. Reload `systemd` config files, the logging service (it may not be started on
    a new VPS), and caddy

caddy/releases.js

@@ -4,8 +4,8 @@ var github = require('../_common/github.js');
 var owner = 'caddyserver';
 var repo = 'caddy';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     // remove checksums and .deb
     all.releases = all.releases.filter(function (rel) {
       let isOneOffAsset = rel.download.includes('buildable-artifact');
@@ -20,7 +20,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all));
   });

chromedriver/releases.js

@@ -1,7 +1,9 @@
 'use strict';
 
+let Fetcher = require('../_common/fetcher.js');
+
 // See <https://googlechromelabs.github.io/chrome-for-testing/>
-var releaseApiUrl =
+const releaseApiUrl =
   'https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json';
 
 // {
@@ -40,14 +42,24 @@ var releaseApiUrl =
 //   ]
 // }
 
-module.exports = async function (request) {
-  let resp = await request({
-    url: releaseApiUrl,
-    json: true,
-  });
+module.exports = async function () {
+  let resp;
+  try {
+    resp = await Fetcher.fetch(releaseApiUrl, {
+      headers: { Accept: 'application/json' },
+    });
+  } catch (e) {
+    /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error
+    let err = e;
+    if (err.code === 'E_FETCH_RELEASES') {
+      err.message = `failed to fetch 'chromedriver' release data: ${err.response.status} ${err.response.body}`;
+    }
+    throw e;
+  }
+  let data = JSON.parse(resp.body);
 
   let builds = [];
-  for (let release of resp.body.versions) {
+  for (let release of data.versions) {
     if (!release.downloads.chromedriver) {
       continue;
     }
@@ -58,7 +70,7 @@ module.exports = async function (request) {
         version: version,
         download: asset.url,
         // I' not sure that this is actually statically built but it
-        // seems to be and at worst we'll just get bug reports for Apline
+        // seems to be and at worst we'll just get bug reports for Alpine
         libc: 'none',
       };
 
@@ -75,10 +87,15 @@ module.exports = async function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
-    all = require('../_webi/normalize.js')(all);
-    // just select the latest 5 for demonstration
-    all.releases = all.releases.slice(-20);
-    console.info(JSON.stringify(all, null, 2));
-  });
+  module
+    .exports()
+    .then(function (all) {
+      all = require('../_webi/normalize.js')(all);
+      // just select the latest 20 for demonstration
+      all.releases = all.releases.slice(-20);
+      console.info(JSON.stringify(all, null, 2));
+    })
+    .catch(function (err) {
+      console.error('Error:', err);
+    });
 }

cilium/README.md

@@ -0,0 +1,47 @@
+---
+title: cilium
+homepage: https://github.com/cilium/cilium-cli
+tagline: |
+  cilium: manage & troubleshoot Kubernetes clusters running Cilium
+---
+
+To update or switch versions, run `webi cilium@stable` (or `@v2`, `@beta`,etc).
+
+### Files
+
+These are the files / directories that are created and/or modified with this
+install:
+
+```text
+~/.config/envman/PATH.env
+~/.local/bin/cilium
+~/.local/opt/cilium/
+```
+
+## Cheat Sheet
+
+> Cilium is an open source, cloud native solution for providing, securing, and
+> observing network connectivity between workloads, fueled by the revolutionary
+> Kernel technology eBPF.
+
+Quick Start User Guide:
+
+<https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/#k8s-install-quick>
+
+To install the default version of the Cilium image:
+
+```sh
+cilium install
+```
+
+To upgrade to a specific version of the Cilium image:
+
+```sh
+cilium upgrade --version v1.15.3
+```
+
+To check the status of the current Cilium deployment:
+
+```sh
+cilium status
+```

cilium/install.ps1

@@ -0,0 +1,45 @@
+#!/usr/bin/env pwsh
+
+##################
+# Install cilium #
+##################
+
+$pkg_cmd_name = "cilium"
+
+$pkg_dst_cmd = "$Env:USERPROFILE\.local\bin\cilium.exe"
+$pkg_dst = "$pkg_dst_cmd"
+
+$pkg_src_cmd = "$Env:USERPROFILE\.local\opt\cilium-v$Env:WEBI_VERSION\bin\cilium.exe"
+$pkg_src_bin = "$Env:USERPROFILE\.local\opt\cilium-v$Env:WEBI_VERSION\bin"
+$pkg_src_dir = "$Env:USERPROFILE\.local\opt\cilium-v$Env:WEBI_VERSION"
+$pkg_src = "$pkg_src_cmd"
+
+New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
+$pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
+
+IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+    Write-Output "Downloading cilium from $Env:WEBI_PKG_URL to $pkg_download"
+    & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
+    & Move-Item "$pkg_download.part" "$pkg_download"
+}
+
+IF (!(Test-Path -Path "$pkg_src_cmd")) {
+    Write-Output "Installing cilium"
+    Push-Location .local\tmp
+
+    Remove-Item -Path ".\cilium-v*" -Recurse -ErrorAction Ignore
+    Remove-Item -Path ".\cilium.exe" -Recurse -ErrorAction Ignore
+
+    Write-Output "Unpacking $pkg_download"
+    & tar xf "$pkg_download"
+
+    Write-Output "Install Location: $pkg_src_cmd"
+    New-Item "$pkg_src_bin" -ItemType Directory -Force
+    Move-Item -Path ".\cilium-*\cilium.exe" -Destination "$pkg_src_bin"
+
+    Pop-Location
+}
+
+Write-Output "Copying into '$pkg_dst_cmd' from '$pkg_src_cmd'"
+Remove-Item -Path "$pkg_dst_cmd" -Recurse -ErrorAction Ignore
+Copy-Item -Path "$pkg_src" -Destination "$pkg_dst" -Recurse

cilium/install.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+__init_cilium() {
+    set -e
+    set -u
+
+    ##################
+    # Install cilium #
+    ##################
+
+    pkg_cmd_name="cilium"
+
+    pkg_dst_cmd="$HOME/.local/bin/cilium"
+    pkg_dst="$pkg_dst_cmd"
+
+    pkg_src_cmd="$HOME/.local/opt/cilium-v$WEBI_VERSION/bin/cilium"
+    pkg_src_dir="$HOME/.local/opt/cilium-v$WEBI_VERSION"
+    pkg_src="$pkg_src_cmd"
+
+    WEBI_SINGLE=true
+
+    # pkg_install must be defined by every package
+    pkg_install() {
+        # ~/.local/opt/cilium-v0.16.16/bin
+        mkdir -p "$(dirname "${pkg_src_cmd}")"
+
+        # mv ./hugo ~/.local/opt/cilium-v0.16.16/bin/
+        mv ./cilium "${pkg_src_cmd}"
+    }
+
+    pkg_get_current_version() {
+        cilium version 2> /dev/null |
+            head -n 1 |
+            cut -d ' ' -f 2
+    }
+
+}
+
+__init_cilium

cilium/releases.js

@@ -0,0 +1,21 @@
+'use strict';
+
+var github = require('../_common/github.js');
+var owner = 'cilium';
+var repo = 'cilium-cli';
+
+module.exports = async function () {
+  let all = await github(null, owner, repo);
+  return all;
+};
+
+if (module === require.main) {
+  (async function () {
+    let normalize = require('../_webi/normalize.js');
+    let all = await module.exports();
+    all = normalize(all);
+    // just select the first 5 for demonstration
+    all.releases = all.releases.slice(0, 5);
+    console.info(JSON.stringify(all, null, 2));
+  })();
+}

cmake/releases.js

@@ -4,8 +4,8 @@ var github = require('../_common/github.js');
 var owner = 'Kitware';
 var repo = 'CMake';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     for (let rel of all.releases) {
       if (rel.version.startsWith('v')) {
         rel._version = rel.version.slice(1);
@@ -44,7 +44,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 5 for demonstration
     all.releases = all.releases.slice(0, 5);

commandlinetools/README.md

@@ -0,0 +1,55 @@
+---
+title: XCode Command Line Tools
+homepage: https://webinstall.dev/commandlinetools
+tagline: |
+  The XCode Command Line Tools include git, swift, make, clang, and other developer tools
+---
+
+## Cheat Sheet
+
+> The developer tools provided by Apple for macOS.
+
+- git
+- swift
+- make
+- clang
+- etc
+
+This is also part of [webi-essentials](../webi-essentials/).
+
+## Table of Contents
+
+- Files
+- Manual Install
+  - macOS
+  - Linux
+  - Alpine
+  - Windows
+
+### Files
+
+These are the files / directories that are created and/or modified with this
+install:
+
+```sh
+/Library/Developer/CommandLineTools/
+```
+
+### How to Install Manually
+
+It's very easy to start the installer:
+
+```sh
+xcode-select --install
+```
+
+The trick is to also have a mechanism to know when it has finished:
+
+```sh
+while ! test -x /Library/Developer/CommandLineTools/usr/bin/git ||
+    ! test -x /Library/Developer/CommandLineTools/usr/bin/make; do
+    sleep 0.25
+done
+
+echo "Command Line Tools Installed"
+```

commandlinetools/install.sh

@@ -0,0 +1,62 @@
+#!/bin/sh
+set -e
+set -u
+
+fn_install_xcode_commandlinetools() { (
+    b_os="$(uname -s)"
+    if test "${b_os}" != 'Darwin'; then
+        echo >&2 'XCode Command Line Tools are for macOS only'
+        return 1
+    fi
+
+    # streamline the output to be pretty
+    fn_check_pkg '/Library/Developer/CommandLineTools/usr/bin/clang' 'clang'
+    fn_check_pkg '/Library/Developer/CommandLineTools/usr/bin/git' 'git'
+    fn_check_pkg '/Library/Developer/CommandLineTools/usr/bin/make' 'make'
+    echo >&2 ""
+
+    # git
+    if xcode-select -p > /dev/null 2> /dev/null; then
+        echo ""
+        return 0
+    fi
+
+    cmd_xcode_cli_tools_install="xcode-select --install"
+    echo "    Running $(t_cmd "${cmd_xcode_cli_tools_install}")"
+    $cmd_xcode_cli_tools_install 2> /dev/null
+    echo ""
+    echo ">>> $(t_attn 'ACTION REQUIRED') <<<"
+    echo ""
+    echo "        $(t_attn "Click") '$(t_bold 'Install')' $(t_attn "in the pop-up")"
+    echo "        (it may appear $(t_em 'under') this window)"
+    echo ""
+    echo "^^^ $(t_attn 'ACTION REQUIRED') ^^^"
+    echo ""
+    printf "    waiting %s to finish installing Command Line Developer Tools ..." "$(t_em 'for you')"
+    while ! test -x /Library/Developer/CommandLineTools/usr/bin/git ||
+        ! test -x /Library/Developer/CommandLineTools/usr/bin/make; do
+        sleep 0.25
+    done
+    echo " $(t_info 'OK')"
+    echo "    Installed to $(t_path '/Library/Developer/CommandLineTools/')"
+    sleep 1
+); }
+
+fn_check_pkg() { (
+    a_pkg="${1}"
+    a_pkgname="${2:-$a_pkg}"
+
+    printf >&2 '    %s %s %s' \
+        "$(t_dim "Checking for")" \
+        "$(t_pkg "${a_pkgname}")" \
+        "$(t_dim "...")"
+
+    if command -v "${a_pkg}" > /dev/null; then
+        echo >&2 " $(t_dim 'OK')"
+        return 0
+    fi
+
+    echo >&2 ' missing'
+); }
+
+fn_install_xcode_commandlinetools

comrak/releases.js

@@ -6,8 +6,8 @@ var repo = 'comrak';
 
 var ODDITIES = ['-musleabihf.1-'];
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     let builds = [];
 
     loopBuilds: for (let build of all.releases) {
@@ -31,7 +31,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     all.releases = all.releases.slice(0, 10);
     //console.info(JSON.stringify(all));

crabz/releases.js

@@ -4,8 +4,8 @@ var github = require('../_common/github.js');
 var owner = 'sstadick';
 var repo = 'crabz';
 
-module.exports = async function (request) {
-  let all = await github(request, owner, repo);
+module.exports = async function () {
+  let all = await github(null, owner, repo);
 
   let releases = [];
   for (let rel of all.releases) {
@@ -22,7 +22,7 @@ module.exports = async function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 5 for demonstration
     all.releases = all.releases.slice(0, 5);

curlie/releases.js

@@ -4,15 +4,15 @@ var github = require('../_common/github.js');
 var owner = 'rs';
 var repo = 'curlie';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     all._names = ['curlie', 'curl-httpie'];
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     all.releases = all.releases.slice(0, 10);
     //console.info(JSON.stringify(all));

dashcore-utils/README.md

@@ -100,8 +100,7 @@ mkdir -p ~/.dashcore/wallets/
 mkdir -p /mnt/slc1_vol_100g/dashcore/_data
 mkdir -p /mnt/slc1_vol_100g/dashcore/_caches
 
-sudo env PATH="$PATH" serviceman add \
-        --system --user "$my_user" --path "$PATH" --name dashd --force -- \
+serviceman add --name 'dashd' --daemon -- \
     dashd \
         -usehd \
         -conf="$HOME/.dashcore/dash.conf" \

dashcore-utils/dash.example.conf

@@ -1,7 +1,14 @@
+# for exposing RPCs, building APIs
 txindex=1
 addressindex=1
 timestampindex=1
 spentindex=1
+# listen as server (explicit default)
+listen=1
+# because its already run as a service (systemd, openrc)
+daemon=0
+# for evonodes
+#server=1
 
 [main]
 rpcuser=RPCUSER_MAIN

dashcore-utils/dashd-hd-service-install

@@ -17,7 +17,7 @@ fn_usage() { (
 ); }
 
 fn_datadir_help() { (
-    my_vol="${1:-}"
+    my_vol="${1}"
     my_user="$(
         id -n -u
     )"
@@ -25,6 +25,7 @@ fn_datadir_help() { (
         id -n -g
     )"
 
+    my_mount="$(dirname "${my_vol}")"
     echo >&2 ""
     echo >&2 "ERROR"
     echo >&2 "        '${my_vol}' is not writable"
@@ -32,8 +33,8 @@ fn_datadir_help() { (
     echo >&2 "SOLUTION"
     echo >&2 "        1. Mount a large (50gb+) volume"
     echo >&2 ""
-    echo >&2 "              sudo mkdir -p /mnt/EXAMPLE"
-    echo >&2 "              sudo mount /dev/sdx1 /mnt/EXAMPLE"
+    echo >&2 "              sudo mkdir -p ${my_mount}"
+    echo >&2 "              sudo mount /dev/sdx1 ${my_mount}"
     echo >&2 ""
     echo >&2 "        2. Create a 'dashcore' inside of it"
     echo >&2 ""
@@ -83,20 +84,8 @@ fn_srv_install() { (
         my_name="dashd-${my_netname}"
     fi
 
-    my_system_args=""
-    my_kernel="$(
-        uname -s
-    )"
-    if test "Darwin" != "${my_kernel}"; then
-        my_user="$(
-            id -u -n
-        )"
-        my_system_args="--system --username ${my_user}"
-    fi
-
     # shellcheck disable=SC2016,SC1090
-    echo 'sudo env PATH="$PATH"' \
-        "serviceman add ${my_system_args} --path \"\$PATH\" --name \"${my_name}\" --force --" \
+    echo "serviceman add --name \"${my_name}\" --" \
         "dashd " \
         "${my_net_flag}" \
         -usehd \
@@ -106,16 +95,16 @@ fn_srv_install() { (
         "-datadir=\"${my_datadir}\"" \
         "-blocksdir=\"${my_blocksdir}\""
 
+    echo ""
+    echo "Installing latest 'serviceman'..."
+    echo ""
+    "$HOME/.local/bin/webi" serviceman > /dev/null
     if ! command -v serviceman > /dev/null; then
-        echo ""
-        echo "Installing 'serviceman'..."
-        echo ""
-        {
-            curl -fsSL "${WEBI_HOST}/serviceman" | sh
-        } > /dev/null
-
-        # shellcheck disable=SC1090
-        . ~/.config/envman/PATH.env || true
+        export PATH="$HOME/.local/bin:$PATH"
+    fi
+    serviceman --version
+    if ! command -v dashd > /dev/null; then
+        export PATH="$HOME/.local/opt/dashcore/bin:$PATH"
     fi
 
     mkdir -p "$HOME/.dashcore/wallets/"
@@ -130,8 +119,7 @@ fn_srv_install() { (
     cd "${my_vol}" || return 1
     # leave options unquoted so they're interpreted separately
     # shellcheck disable=SC2086
-    sudo env PATH="${PATH}" \
-        serviceman add ${my_system_args} --path "${PATH}" --name "${my_name}" --force -- \
+    serviceman add --name "${my_name}" -- \
         dashd \
         ${my_net_flag} \
         -usehd \

dashcore-utils/install.sh

@@ -5,32 +5,38 @@ set -u
 __install_dashcore_utils() {
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dash-qt-hd" \
-        "$HOME/.local/bin/dash-qt-hd"
+        "$HOME/.local/bin/dash-qt-hd" \
+        "dash-qt-hd"
     chmod a+x "$HOME/.local/bin/dash-qt-hd"
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dash-qt-testnet" \
-        "$HOME/.local/bin/dash-qt-testnet"
+        "$HOME/.local/bin/dash-qt-testnet" \
+        "dash-qt-testnet"
     chmod a+x "$HOME/.local/bin/dash-qt-testnet"
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dashd-hd" \
-        "$HOME/.local/bin/dashd-hd"
+        "$HOME/.local/bin/dashd-hd" \
+        "dashd-hd"
     chmod a+x "$HOME/.local/bin/dashd-hd"
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dashd-testnet" \
-        "$HOME/.local/bin/dashd-testnet"
+        "$HOME/.local/bin/dashd-testnet" \
+        "dashd-testnet"
     chmod a+x "$HOME/.local/bin/dashd-testnet"
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dashd-hd-service-install" \
-        "$HOME/.local/bin/dashd-hd-service-install"
+        "$HOME/.local/bin/dashd-hd-service-install" \
+        "dashd-hd-service-install"
     chmod a+x "$HOME/.local/bin/dashd-hd-service-install"
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dashd-testnet-service-install" \
-        "$HOME/.local/bin/dashd-testnet-service-install"
+        "$HOME/.local/bin/dashd-testnet-service-install" \
+        "dashd-testnet-service-install"
     chmod a+x "$HOME/.local/bin/dashd-testnet-service-install"
 
     if ! test -e "${HOME}/.dashcore"; then
@@ -44,7 +50,8 @@ __install_dashcore_utils() {
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dash.example.conf" \
-        "$HOME/.dashcore/dash.example.conf"
+        "$HOME/.dashcore/dash.example.conf" \
+        "dash.example.conf"
 
     if ! grep -q rpcuser ~/.dashcore/dash.conf; then
         cat ~/.dashcore/dash.example.conf >> ~/.dashcore/dash.conf

dashcore/releases.js

@@ -4,8 +4,8 @@ var github = require('../_common/github.js');
 var owner = 'dashpay';
 var repo = 'dash';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     all.releases.forEach(function (rel) {
       if (rel.name.includes('osx64')) {
         rel.os = 'macos';
@@ -22,7 +22,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 5 for demonstration
     all.releases = all.releases.slice(0, 5);

dashd/README.md

@@ -24,7 +24,7 @@ install:
 ~/.config/envman/PATH.env
 ~/.dashcore/dash.conf
 ~/.dashcore/wallets/
-~/.local/bin/bin/dashd-hd-service-install
+~/.local/bin/dashd-hd-service-install
 ~/.local/opt/dashcore/
 /mnt/<BLK_VOL>/dashcore/
 ```
@@ -219,14 +219,7 @@ You can use [`serviceman`](../serviceman/):
 **Linux**
 
 ```sh
-sudo env PATH="$PATH" \
-    serviceman add \
-        --system \
-        --username "$(id -n -u)" \
-        --path "$PATH" \
-        --name dashd \
-        --force \
-        -- \
+serviceman add --name 'dashd' -- \
     dashd \
         -usehd \
         -conf="$HOME/.dashcore/dash.conf" \
@@ -239,11 +232,7 @@ sudo env PATH="$PATH" \
 **Mac**
 
 ```sh
-serviceman add \
-    --path "$PATH" \
-    --name dashd \
-    --force \
-    -- \
+serviceman add --name 'dashd' -- \
     dashd \
         -usehd \
         -conf="$HOME/.dashcore/dash.conf" \

dashmsg/releases.js

@@ -4,14 +4,14 @@ var github = require('../_common/github.js');
 var owner = 'dashhive';
 var repo = 'dashmsg';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all, null, 2));
   });

delta/releases.js

@@ -4,14 +4,14 @@ var github = require('../_common/github.js');
 var owner = 'dandavison';
 var repo = 'delta';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 15 for demonstration
     all.releases = all.releases.slice(0, 15);

deno/README.md

@@ -18,7 +18,7 @@ etc).
 The obligatory Hello World
 
 ```sh
-deno run https://deno.land/std/examples/welcome.ts
+deno run https://docs.deno.com/examples/scripts/hello_world.ts
 ```
 
 Run a local file

deno/releases.js

@@ -6,8 +6,8 @@ var github = require('../_common/github.js');
 var owner = 'denoland';
 var repo = 'deno';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     // remove checksums and .deb
     all.releases = all.releases
       .filter(function (rel) {
@@ -35,7 +35,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all, null, 2));
   });

dotenv-linter/releases.js

@@ -4,14 +4,14 @@ var github = require('../_common/github.js');
 var owner = 'dotenv-linter';
 var repo = 'dotenv-linter';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 5 for demonstration
     all.releases = all.releases.slice(0, 5);

dotenv/releases.js

@@ -4,14 +4,14 @@ var github = require('../_common/github.js');
 var owner = 'therootcompany';
 var repo = 'dotenv';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all));
   });

duckdns.sh/releases.js

@@ -1,29 +1,21 @@
 'use strict';
 
-var githubSource = require('../_common/github-source.js');
-var owner = 'BeyondCodeBootcamp';
-var repo = 'DuckDNS.sh';
+let Releases = module.exports;
 
-module.exports = function (request) {
-  let arches = [
-    'amd64',
-    'arm64',
-    'armv6l',
-    'armv7l',
-    'ppc64le',
-    'ppc64',
-    's390x',
-    'x86',
-  ];
-  let oses = ['freebsd', 'linux', 'macos', 'posix'];
-  return githubSource(request, owner, repo, oses, arches).then(function (all) {
-    all._names = ['DuckDNS.sh', 'duckdns.sh', 'legacy'];
-    return all;
-  });
+let GitHubSource = require('../_common/github-source.js');
+let owner = 'BeyondCodeBootcamp';
+let repo = 'DuckDNS.sh';
+
+Releases.latest = async function () {
+  let all = await GitHubSource.getDistributables({ owner, repo });
+  for (let pkg of all.releases) {
+    pkg.os = 'posix_2017';
+  }
+  return all;
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  Releases.latest().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all, null, 2));
   });

duckdns/install.sh

@@ -2,10 +2,16 @@
 set -e
 set -u
 
-echo "'duckdns@${WEBI_TAG:-stable}' is reserved for future use."
-echo
-echo "Did you mean 'duckdns.sh@${WEBI_VERSION-}'?"
 WEBI_HOST=${WEBI_HOST:-"https://webi.sh"}
+
 echo ""
-echo "    curl -fsSL '$WEBI_HOST/duckdns.sh@${WEBI_VERSION-}' | sh"
+echo "ERROR"
+echo "    installer name 'duckdns' is reserved for future use"
+echo ""
+echo "SOLUTION"
+echo "    Did you mean 'duckdns.sh'?"
+echo ""
+echo "    curl -fsSL '$WEBI_HOST/duckdns.sh' | sh"
+echo ""
+
 exit 1

fd/releases.js

@@ -4,8 +4,8 @@ var github = require('../_common/github.js');
 var owner = 'sharkdp';
 var repo = 'fd';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     let builds = [];
 
     for (let build of all.releases) {
@@ -22,7 +22,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     all.releases = all.releases.slice(0, 10);
     //console.info(JSON.stringify(all));

ffmpeg/releases.js

@@ -6,8 +6,8 @@ var github = require('../_common/github.js');
 var owner = 'eugeneware';
 var repo = 'ffmpeg-static';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     all.releases = all.releases
       .filter(function (rel) {
         let isFfmpeg = rel.name.includes('ffmpeg');
@@ -38,7 +38,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all));
   });

ffuf/releases.js

@@ -4,14 +4,14 @@ var github = require('../_common/github.js');
 var owner = 'ffuf';
 var repo = 'ffuf';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 5 for demonstration
     all.releases = all.releases.slice(0, 5);

fish/README.md

@@ -43,7 +43,7 @@ the file:
 ```sh
 #!/bin/bash
 
-echo "Who am I? I'm $(whoami)."
+echo "Who am I? I'm $(id -u -n)."
 ```
 
 You can also run bash explicitly:
@@ -99,7 +99,7 @@ You should use `chsh` to change your shell:
 ```sh
 #!/bin/sh
 
-sudo chsh -s "$(command -v fish)" "$(whoami)"
+sudo chsh -s "$(command -v fish)" "$(id -u -n)"
 ```
 
 If vim uses `fish` instead of `bash`, annoying errors will happen.

fish/releases.js

@@ -6,8 +6,8 @@ var repo = 'fish-shell';
 
 var ODDITIES = ['bundledpcre'];
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     all.releases = all.releases
       .map(function (rel) {
         for (let oddity of ODDITIES) {
@@ -30,7 +30,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 5 for demonstration
     all.releases = all.releases.slice(0, 5);

flutter/releases.js

@@ -1,9 +1,14 @@
 'use strict';
 
-var FLUTTER_OSES = ['macos', 'linux', 'windows'];
+let Fetcher = require('../_common/fetcher.js');
 
-// stable, beta, dev
-var channelMap = {};
+let FLUTTER_OSES = ['macos', 'linux', 'windows'];
+
+/**
+ * stable, beta, dev
+ * @type {Object.<String, Boolean>}
+ */
+let channelMap = {};
 
 // This can be spot-checked against
 // https://docs.flutter.dev/release/archive?tab=windows
@@ -53,23 +58,47 @@ var channelMap = {};
 //   ]
 // }
 
-module.exports = async function (request) {
+/**
+ * @typedef BuildInfo
+ * @prop {String} version
+ * @prop {String} [_version]
+ * @prop {Boolean} lts
+ * @prop {String} channel
+ * @prop {String} date
+ * @prop {String} download
+ * @prop {String} [_filename]
+ */
+
+module.exports = async function () {
   let all = {
     download: '',
+    /** @type {Array<BuildInfo>} */
     releases: [],
+    /** @type {Array<String>} */
     channels: [],
   };
 
   for (let osname of FLUTTER_OSES) {
-    let resp = await request({
-      url: `https://storage.googleapis.com/flutter_infra_release/releases/releases_${osname}.json`,
-      json: true,
-    });
+    let resp;
+    try {
+      let url = `https://storage.googleapis.com/flutter_infra_release/releases/releases_${osname}.json`;
+      resp = await Fetcher.fetch(url, {
+        headers: { Accept: 'application/json' },
+      });
+    } catch (e) {
+      /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error
+      let err = e;
+      if (err.code === 'E_FETCH_RELEASES') {
+        err.message = `failed to fetch 'flutter' release data for ${osname}: ${err.response.status} ${err.response.body}`;
+      }
+      throw e;
+    }
+    let data = JSON.parse(resp.body);
 
-    let body = resp.body;
-    all.download = `${body.base_url}/{{ download }}`;
+    let osBaseUrl = data.base_url;
+    let osReleases = data.releases;
 
-    for (let asset of body.releases) {
+    for (let asset of osReleases) {
       if (!channelMap[asset.channel]) {
         channelMap[asset.channel] = true;
       }
@@ -80,8 +109,8 @@ module.exports = async function (request) {
         lts: false,
         channel: asset.channel,
         date: asset.release_date.replace(/T.*/, ''),
-        //sha256: asset.sha256,
-        download: asset.archive,
+        download: `${osBaseUrl}/${asset.archive}`,
+        _filename: asset.archive,
       });
     }
   }
@@ -96,7 +125,7 @@ module.exports = async function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all.releases = all.releases.slice(25);
     console.info(JSON.stringify(all, null, 2));
   });

fzf/releases.js

@@ -4,14 +4,14 @@ var github = require('../_common/github.js');
 var owner = 'junegunn';
 var repo = 'fzf';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     all.releases = all.releases.slice(0, 10);
     //console.info(JSON.stringify(all));

gh/releases.js

@@ -4,14 +4,14 @@ var github = require('../_common/github.js');
 var owner = 'cli';
 var repo = 'cli';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 5 for demonstration
     all.releases = all.releases.slice(0, 5);

git/README.md

@@ -21,9 +21,10 @@ is a good place to get started if you're new to git.
 - Files
 - Commit Files
 - Ignore Files
+- Reasonable Defaults
 - Create Branch
-- Rebase by Default
 - Rebase
+- Auth via SSH
 - Auth via Token
 
 ### Files
@@ -66,9 +67,49 @@ This will branch from the branch you're currently on.
 git switch -c my-branch-name
 ```
 
-### How to rebase by default
+### Reasonable Defaults for Git Config
+
+- use SSH instead of HTTPS
+- default to 'main'
+- create on 'push'
+- stash on 'rebase'
+- default to 'rebase' (modern)
 
 ```sh
+#####################
+#    ENFORCE SSH    #
+#####################
+# replace HTTPS urls with SSH urls (to always use keys rather than tokens)
+
+git config --global url."ssh://git@example.com/".insteadOf "https://example.com/"
+git config --global url."ssh://git@github.com/".insteadOf "https://github.com/"
+
+######################
+#   DEFAULT BRANCH   #
+######################
+# Set the default branch for new repos (ex: 'main')
+
+git config --global init.defaultBranch 'main'
+
+######################
+# AUTOMATIC BRANCHES #
+######################
+# make 'git push' create branches if they don't exist on the remote
+
+git config --global push.autoSetupRemote true
+
+######################
+# REBASE AUTO-STASH  #
+######################
+# stash immediately before rebase and unstash immediately after
+
+git config --global rebase.autoStash true
+
+######################
+# REBASE BY DEFAULT  #
+######################
+# use 'rebase' rather than 'merge' or 'ff-only'
+
 git config --global pull.rebase true
 ```
 
@@ -102,6 +143,26 @@ git add ./my-merged-file
 git rebase --continue
 ```
 
+### How to authenticate git with SSH keys by default
+
+```sh
+# Git, Gitea, etc
+git config --global url."ssh://git@git.example.com/".insteadOf "https://git.example.com/"
+git config --global url."ssh://git@git.example.com.com/".insteadOf "git@git.example.com.com:"
+
+# GitHub
+git config --global url."ssh://git@github.com/".insteadOf "https://github.com/"
+git config --global url."ssh://git@github.com/".insteadOf "git@github.com:"
+
+# GitLab
+git config --global url."ssh://git@gitlab.com/".insteadOf "https://gitlab.com/"
+git config --global url."ssh://git@gitlab.com/".insteadOf "git@gitlab.com:"
+
+# BitBucket
+git config --global url."ssh://git@bitbucket.com/".insteadOf "https://bitbucket.com/"
+git config --global url."ssh://git@bitbucket.com/".insteadOf "git@bitbucket.com:"
+```
+
 ### How to authenticate git with deploy tokens
 
 Abbreviated from

git/releases.js

@@ -4,9 +4,9 @@ var github = require('../_common/github.js');
 var owner = 'git-for-windows';
 var repo = 'git';
 
-module.exports = function (request) {
+module.exports = function () {
   // TODO support mac and linux tarballs
-  return github(request, owner, repo).then(function (all) {
+  return github(null, owner, repo).then(function (all) {
     // See https://github.com/git-for-windows/git/wiki/MinGit
     // also consider https://github.com/git-for-windows/git/wiki/Silent-or-Unattended-Installation
     all.releases = all.releases
@@ -26,7 +26,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all, null, 2));
   });

gitdeploy/releases.js

@@ -4,14 +4,14 @@ var github = require('../_common/github.js');
 var owner = 'therootcompany';
 var repo = 'gitdeploy';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all));
   });

gitea/releases.js

@@ -6,8 +6,8 @@ var repo = 'gitea';
 
 var ODDITIES = ['-gogit-', '-docs-'];
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     // remove checksums and .deb
     all.releases = all.releases.filter(function (rel) {
       for (let oddity of ODDITIES) {
@@ -27,7 +27,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all));
   });

go-essentials/README.md

@@ -17,7 +17,6 @@ including:
 
 - [godoc](https://pkg.go.dev/golang.org/x/tools/cmd/godoc)
 - [gopls](https://pkg.go.dev/golang.org/x/tools/gopls)
-- [guru](https://pkg.go.dev/golang.org/x/tools/cmd/guru)
 - [golint](https://pkg.go.dev/golang.org/x/lint/golint)
 - [goimports](https://pkg.go.dev/golang.org/x/tools/cmd/goimports)
 - [gomvpkg](https://pkg.go.dev/golang.org/x/tools/cmd/gomvpkg)

go-essentials/install.ps1

@@ -16,10 +16,6 @@ Write-Output ""
 Write-Output gopls
 & go install golang.org/x/tools/gopls@latest
 
-Write-Output ""
-Write-Output guru
-& go install golang.org/x/tools/guru@latest
-
 Write-Output ""
 Write-Output golint
 & go install golang.org/x/lint/golint@latest

go-essentials/install.sh

@@ -3,7 +3,7 @@ set -e
 set -u
 
 __run_go_essentials() {
-    if ! command -v go 2> /dev/null; then
+    if ! command -v go > /dev/null; then
         "$HOME/.local/bin/webi" "go@${WEBI_TAG}"
     fi
 
@@ -44,10 +44,6 @@ __run_go_essentials() {
     echo gopls
     go "${my_install}" golang.org/x/tools/gopls@latest > /dev/null #2>/dev/null
 
-    echo ""
-    echo guru
-    go "${my_install}" golang.org/x/tools/cmd/guru@latest > /dev/null #2>/dev/null
-
     echo ""
     echo golint
     go "${my_install}" golang.org/x/lint/golint@latest > /dev/null #2>/dev/null

go/README.md

@@ -80,8 +80,7 @@ webi serviceman
 pushd ./hello/
 
 # swap 'hello' and './hello' for the name of your project and binary
-sudo env PATH="$PATH" \
-    serviceman add --system --username "$(whoami)" --name hello -- \
+serviceman add --name 'hello' -- \
     ./hello
 
 # Restart the logging service

go/releases.js

@@ -1,14 +1,21 @@
 'use strict';
 
-var osMap = {
+let Fetcher = require('../_common/fetcher.js');
+
+/** @type {Object.<String, String>} */
+let osMap = {
   darwin: 'macos',
 };
-var archMap = {
+/** @type {Object.<String, String>} */
+let archMap = {
   386: 'x86',
 };
 
 let ODDITIES = ['bootstrap', '-arm6.'];
 
+/**
+ * @param {String} filename
+ */
 function isOdd(filename) {
   for (let oddity of ODDITIES) {
     let isOddity = filename.includes(oddity);
@@ -18,7 +25,22 @@ function isOdd(filename) {
   }
 }
 
-function getAllReleases(request) {
+/**
+ * @typedef BuildInfo
+ * @prop {String} version
+ * @prop {String} [_version]
+ * @prop {String} arch
+ * @prop {String} channel
+ * @prop {String} date
+ * @prop {String} download
+ * @prop {String} ext
+ * @prop {String} [_filename]
+ * @prop {String} hash
+ * @prop {Boolean} lts
+ * @prop {String} os
+ */
+
+async function getDistributables() {
   /*
   {
     version: 'go1.13.8',
@@ -37,60 +59,71 @@ function getAllReleases(request) {
     ]
   };
   */
-  return request({
-    url: 'https://golang.org/dl/?mode=json&include=all',
-    json: true,
-  }).then((resp) => {
-    var goReleases = resp.body;
-    var all = {
-      releases: [],
-      download: 'https://dl.google.com/go/{{ download }}',
-    };
 
-    goReleases.forEach((release) => {
-      // strip 'go' prefix, standardize version
-      var parts = release.version.slice(2).split('.');
-      while (parts.length < 3) {
-        parts.push('0');
-      }
-      var version = parts.join('.');
-      // nix 'go' prefix
-      var fileversion = release.version.slice(2);
-
-      release.files.forEach((asset) => {
-        let odd = isOdd(asset.filename);
-        if (odd) {
-          return;
-        }
-
-        var filename = asset.filename;
-        var os = osMap[asset.os] || asset.os || '-';
-        var arch = archMap[asset.arch] || asset.arch || '-';
-        all.releases.push({
-          version: version,
-          _version: fileversion,
-          // all go versions >= 1.0.0 are effectively LTS
-          lts: (parts[0] > 0 && release.stable) || false,
-          channel: (release.stable && 'stable') || 'beta',
-          date: '1970-01-01', // the world may never know
-          os: os,
-          arch: arch,
-          ext: '', // let normalize run the split/test/join
-          hash: '-', // not ready to standardize this yet
-          download: filename,
-        });
-      });
+  let resp;
+  try {
+    let url = 'https://golang.org/dl/?mode=json&include=all';
+    resp = await Fetcher.fetch(url, {
+      headers: { Accept: 'application/json' },
     });
+  } catch (e) {
+    /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error
+    let err = e;
+    if (err.code === 'E_FETCH_RELEASES') {
+      err.message = `failed to fetch 'Go' release data: ${err.response.status} ${err.response.body}`;
+    }
+    throw e;
+  }
+  let goReleases = JSON.parse(resp.body);
 
-    return all;
-  });
+  let all = {
+    /** @type {Array<BuildInfo>} */
+    releases: [],
+    download: '',
+  };
+
+  for (let release of goReleases) {
+    // Strip 'go' prefix, standardize version
+    let parts = release.version.slice(2).split('.');
+    while (parts.length < 3) {
+      parts.push('0');
+    }
+    let version = parts.join('.');
+    let fileversion = release.version.slice(2);
+
+    for (let asset of release.files) {
+      if (isOdd(asset.filename)) {
+        continue;
+      }
+
+      let filename = asset.filename;
+      let os = osMap[asset.os] || asset.os || '-';
+      let arch = archMap[asset.arch] || asset.arch || '-';
+      let build = {
+        version: version,
+        _version: fileversion,
+        lts: (parts[0] > 0 && release.stable) || false,
+        channel: (release.stable && 'stable') || 'beta',
+        date: '1970-01-01', // the world may never know
+        os: os,
+        arch: arch,
+        ext: '', // let normalize run the split/test/join
+        hash: '-', // not ready to standardize this yet
+        download: `https://dl.google.com/go/${filename}`,
+      };
+      all.releases.push(build);
+    }
+  }
+
+  return all;
 }
 
-module.exports = getAllReleases;
+module.exports = getDistributables;
 
 if (module === require.main) {
-  getAllReleases(require('@root/request')).then(function (all) {
+  getDistributables().then(function (all) {
     all = require('../_webi/normalize.js')(all);
+    //@ts-expect-error
     all.releases = all.releases.slice(0, 10);
     console.info(JSON.stringify(all, null, 2));
   });

golang/README.md

@@ -85,8 +85,7 @@ webi serviceman
 pushd ./hello/
 
 # swap 'hello' and './hello' for the name of your project and binary
-sudo env PATH="$PATH" \
-    serviceman add --system --username "$(whoami)" --name hello -- \
+serviceman add --name 'hello' -- \
     ./hello
 
 # Restart the logging service

goreleaser/releases.js

@@ -4,14 +4,15 @@ var github = require('../_common/github.js');
 var owner = 'goreleaser';
 var repo = 'goreleaser';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
+    all._names = ['goreleaser', '1'];
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 5 for demonstration
     all.releases = all.releases.slice(0, 5);

gpg/releases.js

@@ -1,5 +1,7 @@
 'use strict';
 
+let Fetcher = require('../_common/fetcher.js');
+
 let ltsRe = /GnuPG-(2\.2\.[\d\.]+)/;
 
 function createRssMatcher() {
@@ -16,12 +18,42 @@ function createUrlMatcher() {
   );
 }
 
-async function getRawReleases(request) {
-  let matcher = createRssMatcher();
+/**
+ * @typedef BuildInfo
+ * @prop {String} version
+ * @prop {String} [_version]
+ * @prop {String} arch
+ * @prop {String} channel
+ * @prop {String} date
+ * @prop {String} download
+ * @prop {String} ext
+ * @prop {String} [_filename]
+ * @prop {String} hash
+ * @prop {Boolean} lts
+ * @prop {String} os
+ */
 
-  let resp = await request({
-    url: 'https://sourceforge.net/projects/gpgosx/rss?path=/',
-  });
+async function getRawReleases() {
+  let resp;
+  try {
+    let url = 'https://sourceforge.net/projects/gpgosx/rss?path=/';
+    resp = await Fetcher.fetch(url, {
+      headers: { Accept: 'application/rss+xml' },
+    });
+  } catch (e) {
+    /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error
+    let err = e;
+    if (err.code === 'E_FETCH_RELEASES') {
+      err.message = `failed to fetch 'gpg' release data: ${err.response.status} ${err.response.body}`;
+    }
+    throw e;
+  }
+  let contentType = resp.headers.get('Content-Type');
+  if (!contentType?.includes('xml')) {
+    throw new Error(`Unexpected content type: ${contentType}`);
+  }
+
+  let matcher = createRssMatcher();
   let links = [];
   for (;;) {
     let m = matcher.exec(resp.body);
@@ -30,62 +62,66 @@ async function getRawReleases(request) {
     }
     links.push(m[1]);
   }
+
   return links;
 }
 
+/**
+ * @param {Array<String>} links
+ */
 function transformReleases(links) {
   //console.log(JSON.stringify(links, null, 2));
   //console.log(links.length);
 
   let matcher = createUrlMatcher();
 
-  let releases = links
-    .map(function (link) {
-      let isLts = ltsRe.test(link);
-      let parts = link.match(matcher);
-      if (!parts || !parts[2]) {
-        return null;
-      }
-      let segs = parts[2].split('.');
-      let version = segs.slice(0, 3).join('.');
-      if (segs.length > 3) {
-        version += '+' + segs.slice(3);
-      }
-      let fileversion = segs.join('.');
+  let builds = [];
+  for (let link of links) {
+    let isLts = ltsRe.test(link);
+    let parts = link.match(matcher);
+    if (!parts || !parts[2]) {
+      continue;
+    }
 
-      return {
-        name: parts[1],
-        version: version,
-        _version: fileversion,
-        // all go versions >= 1.0.0 are effectively LTS
-        lts: isLts,
-        channel: 'stable',
-        // TODO <pubDate>Sat, 19 Nov 2016 16:17:33 UT</pubDate>
-        date: '1970-01-01', // the world may never know
-        os: 'macos',
-        arch: 'amd64',
-        ext: 'dmg',
-        download: link,
-      };
-    })
-    .filter(Boolean);
+    let segs = parts[2].split('.');
+    let version = segs.slice(0, 3).join('.');
+    if (segs.length > 3) {
+      version += '+' + segs.slice(3);
+    }
+    let fileversion = segs.join('.');
+
+    let build = {
+      name: parts[1],
+      version: version,
+      _version: fileversion,
+      lts: isLts,
+      channel: 'stable',
+      // TODO <pubDate>Sat, 19 Nov 2016 16:17:33 UT</pubDate>
+      date: '1970-01-01', // the world may never know
+      os: 'macos',
+      arch: 'amd64',
+      ext: 'dmg',
+      download: link,
+    };
+    builds.push(build);
+  }
 
   return {
     _names: ['GnuPG', 'gpgosx'],
-    releases: releases,
+    releases: builds,
   };
 }
 
-async function getAllReleases(request) {
-  let releases = await getRawReleases(request);
+async function getDistributables() {
+  let releases = await getRawReleases();
   let all = transformReleases(releases);
   return all;
 }
 
-module.exports = getAllReleases;
+module.exports = getDistributables;
 
 if (module === require.main) {
-  getAllReleases(require('@root/request')).then(function (all) {
+  getDistributables().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     all.releases = all.releases.slice(0, 10000);
     console.info(JSON.stringify(all, null, 2));

gprox/releases.js

@@ -4,14 +4,14 @@ var github = require('../_common/github.js');
 var owner = 'creedasaurus';
 var repo = 'gprox';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 5 for demonstration
     all.releases = all.releases.slice(0, 5);

grype/releases.js

@@ -4,14 +4,14 @@ var github = require('../_common/github.js');
 var owner = 'anchore';
 var repo = 'grype';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     // just select the first 5 for demonstration
     all.releases = all.releases.slice(0, 5);

hexyl/releases.js

@@ -4,14 +4,14 @@ var github = require('../_common/github.js');
 var owner = 'sharkdp';
 var repo = 'hexyl';
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     return all;
   });
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     all.releases = all.releases.slice(0, 10);
     //console.info(JSON.stringify(all));

hugo-extended/releases.js

@@ -4,8 +4,8 @@ var github = require('../_common/github.js');
 var owner = 'gohugoio';
 var repo = 'hugo';
 
-module.exports = async function (request) {
-  let all = await github(request, owner, repo);
+module.exports = async function () {
+  let all = await github(null, owner, repo);
 
   all.releases = all.releases.filter(function (rel) {
     let isExtended = rel.name.includes('_extended_');
@@ -25,7 +25,7 @@ module.exports = async function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all));
   });

hugo/releases.js

@@ -4,8 +4,8 @@ var github = require('../_common/github.js');
 var owner = 'gohugoio';
 var repo = 'hugo';
 
-module.exports = async function (request) {
-  let all = await github(request, owner, repo);
+module.exports = async function () {
+  let all = await github(null, owner, repo);
 
   all.releases = all.releases.filter(function (rel) {
     let isExtended = rel.name.includes('_extended_');
@@ -25,7 +25,7 @@ module.exports = async function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all));
   });

iterm2/releases.js

@@ -1,73 +1,91 @@
 'use strict';
 
-function getRawReleases(request) {
-  return request({ url: 'https://iterm2.com/downloads.html' }).then(
-    function (resp) {
-      var links = resp.body
-        .split(/[<>]+/g)
-        .map(function (str) {
-          var m = str.match(
-            /href="(https:\/\/iterm2\.com\/downloads\/.*\.zip)"/,
-          );
-          if (m && /iTerm2-[34]/.test(m[1])) {
-            return m[1];
-          }
-        })
-        .filter(Boolean);
-      return links;
-    },
-  );
+let Fetcher = require('../_common/fetcher.js');
+
+async function getRawReleases() {
+  let resp;
+  try {
+    let url = 'https://iterm2.com/downloads.html';
+    resp = await Fetcher.fetch(url, {
+      headers: { Accept: 'text/html' },
+    });
+  } catch (e) {
+    /** @type {Error & { code: string, response: { status: number, body: string } }} */ //@ts-expect-error
+    let err = e;
+    if (err.code === 'E_FETCH_RELEASES') {
+      err.message = `failed to fetch 'iterm2' release data: ${err.response.status} ${err.response.body}`;
+    }
+    throw e;
+  }
+
+  let contentType = resp.headers.get('Content-Type');
+  if (!contentType || !contentType.includes('text/html')) {
+    throw new Error(`Unexpected Content-Type: ${contentType}`);
+  }
+
+  let lines = resp.body.split(/[<>]+/g);
+
+  /** @type {Array<String>} */
+  let links = [];
+  for (let str of lines) {
+    let m = str.match(/href="(https:\/\/iterm2\.com\/downloads\/.*\.zip)"/);
+    if (m && /iTerm2-[34]/.test(m[1])) {
+      if (m[1]) {
+        links.push(m[1]);
+      }
+    }
+  }
+
+  return links;
 }
 
+/**
+ * @param {Array<String>} links
+ */
 function transformReleases(links) {
-  //console.log(JSON.stringify(links, null, 2));
-  //console.log(links.length);
+  let builds = [];
+  for (let link of links) {
+    let channel = /\/stable\//.test(link) ? 'stable' : 'beta';
+
+    let parts = link.replace(/.*\/iTerm2[-_]v?(\d_.*)\.zip/, '$1').split('_');
+    let version = parts.join('.').replace(/([_-])?beta/, '-beta');
+
+    // ex: 3.5.0-beta17 => 3_5_0beta17
+    // ex: 3.0.2-preview => 3_0_2-preview
+    let fileversion = version.replace(/\./g, '_');
+    fileversion = fileversion.replace(/-beta/g, 'beta');
+
+    let build = {
+      version: version,
+      _version: fileversion,
+      lts: 'stable' === channel,
+      channel: channel,
+      date: '1970-01-01', // the world may never know
+      os: 'macos',
+      arch: 'amd64',
+      ext: '', // let normalize run the split/test/join
+      download: link,
+    };
+    builds.push(build);
+  }
 
   return {
     _names: ['iTerm2', 'iterm2'],
-    releases: links
-      .map(function (link) {
-        var channel = /\/stable\//.test(link) ? 'stable' : 'beta';
-
-        var parts = link
-          .replace(/.*\/iTerm2[-_]v?(\d_.*)\.zip/, '$1')
-          .split('_');
-        var version = parts.join('.').replace(/([_-])?beta/, '-beta');
-
-        // ex: 3.5.0-beta17 => 3_5_0beta17
-        // ex: 3.0.2-preview => 3_0_2-preview
-        let fileversion = version.replace(/\./g, '_');
-        fileversion = fileversion.replace(/-beta/g, 'beta');
-
-        return {
-          version: version,
-          _version: fileversion,
-          // all go versions >= 1.0.0 are effectively LTS
-          lts: 'stable' === channel,
-          channel: channel,
-          date: '1970-01-01', // the world may never know
-          os: 'macos',
-          arch: 'amd64',
-          ext: '', // let normalize run the split/test/join
-          download: link,
-        };
-      })
-      .filter(Boolean),
+    releases: builds,
   };
 }
 
-function getAllReleases(request) {
-  return getRawReleases(request)
-    .then(transformReleases)
-    .then(function (all) {
-      return all;
-    });
+async function getDistributables() {
+  let rawReleases = await getRawReleases();
+  let all = transformReleases(rawReleases);
+
+  return all;
 }
 
-module.exports = getAllReleases;
+module.exports = getDistributables;
 
 if (module === require.main) {
-  getAllReleases(require('@root/request')).then(function (all) {
+  getDistributables().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     all.releases = all.releases.slice(0, 10000);
     console.info(JSON.stringify(all, null, 2));

jq/releases.js

@@ -15,8 +15,8 @@ function isOdd(build) {
   }
 }
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
+module.exports = function () {
+  return github(null, owner, repo).then(function (all) {
     let builds = [];
 
     for (let build of all.releases) {
@@ -35,7 +35,7 @@ module.exports = function (request) {
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
+  module.exports().then(function (all) {
     all = require('../_webi/normalize.js')(all);
     console.info(JSON.stringify(all));
     //console.info(JSON.stringify(all, null, 2));

jsconfig.json

@@ -0,0 +1,114 @@
+{
+  "compilerOptions": {
+    /* Visit https://aka.ms/tsconfig to read more about this file */
+
+    /* Projects */
+    // "incremental": true,                              /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
+    // "composite": true,                                /* Enable constraints that allow a TypeScript project to be used with project references. */
+    // "tsBuildInfoFile": "./.tsbuildinfo",              /* Specify the path to .tsbuildinfo incremental compilation file. */
+    // "disableSourceOfProjectReferenceRedirect": true,  /* Disable preferring source files instead of declaration files when referencing composite projects. */
+    // "disableSolutionSearching": true,                 /* Opt a project out of multi-project reference checking when editing. */
+    // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
+
+    /* Language and Environment */
+    "target": "es2022",                                  /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
+    // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
+    // "jsx": "preserve",                                /* Specify what JSX code is generated. */
+    // "experimentalDecorators": true,                   /* Enable experimental support for legacy experimental decorators. */
+    // "emitDecoratorMetadata": true,                    /* Emit design-type metadata for decorated declarations in source files. */
+    // "jsxFactory": "",                                 /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
+    // "jsxFragmentFactory": "",                         /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
+    // "jsxImportSource": "",                            /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
+    // "reactNamespace": "",                             /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
+    // "noLib": true,                                    /* Disable including any library files, including the default lib.d.ts. */
+    // "useDefineForClassFields": true,                  /* Emit ECMAScript-standard-compliant class fields. */
+    // "moduleDetection": "auto",                        /* Control what method is used to detect module-format JS files. */
+
+    /* Modules */
+    "module": "commonjs",                                /* Specify what module code is generated. */
+    // "rootDir": "./",                                  /* Specify the root folder within your source files. */
+    "moduleResolution": "node",                        /* Specify how TypeScript looks up a file from a given module specifier. */
+    // "baseUrl": "./",                                  /* Specify the base directory to resolve non-relative module names. */
+    // "paths": {},                                      /* Specify a set of entries that re-map imports to additional lookup locations. */
+    // "rootDirs": [],                                   /* Allow multiple folders to be treated as one when resolving modules. */
+    "typeRoots": ["./typings","./node_modules/@types"],  /* Specify multiple folders that act like './node_modules/@types'. */
+    // "types": [],                                      /* Specify type package names to be included without being referenced in a source file. */
+    // "allowUmdGlobalAccess": true,                     /* Allow accessing UMD globals from modules. */
+    // "moduleSuffixes": [],                             /* List of file name suffixes to search when resolving a module. */
+    // "allowImportingTsExtensions": true,               /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */
+    // "resolvePackageJsonExports": true,                /* Use the package.json 'exports' field when resolving package imports. */
+    // "resolvePackageJsonImports": true,                /* Use the package.json 'imports' field when resolving imports. */
+    // "customConditions": [],                           /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */
+    // "resolveJsonModule": true,                        /* Enable importing .json files. */
+    // "allowArbitraryExtensions": true,                 /* Enable importing files with any extension, provided a declaration file is present. */
+    // "noResolve": true,                                /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
+
+    /* JavaScript Support */
+    "allowJs": true,                                     /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
+    "checkJs": true,                                     /* Enable error reporting in type-checked JavaScript files. */
+    // "maxNodeModuleJsDepth": 1,                        /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
+
+    /* Emit */
+    // "declaration": true,                              /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
+    // "declarationMap": true,                           /* Create sourcemaps for d.ts files. */
+    // "emitDeclarationOnly": true,                      /* Only output d.ts files and not JavaScript files. */
+    // "sourceMap": true,                                /* Create source map files for emitted JavaScript files. */
+    // "inlineSourceMap": true,                          /* Include sourcemap files inside the emitted JavaScript. */
+    // "outFile": "./",                                  /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
+    // "outDir": "./",                                   /* Specify an output folder for all emitted files. */
+    // "removeComments": true,                           /* Disable emitting comments. */
+    "noEmit": true,                                      /* Disable emitting files from a compilation. */
+    // "importHelpers": true,                            /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
+    // "importsNotUsedAsValues": "remove",               /* Specify emit/checking behavior for imports that are only used for types. */
+    // "downlevelIteration": true,                       /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
+    // "sourceRoot": "",                                 /* Specify the root path for debuggers to find the reference source code. */
+    // "mapRoot": "",                                    /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSources": true,                            /* Include source code in the sourcemaps inside the emitted JavaScript. */
+    // "emitBOM": true,                                  /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
+    // "newLine": "crlf",                                /* Set the newline character for emitting files. */
+    // "stripInternal": true,                            /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
+    // "noEmitHelpers": true,                            /* Disable generating custom helper functions like '__extends' in compiled output. */
+    // "noEmitOnError": true,                            /* Disable emitting files if any type checking errors are reported. */
+    // "preserveConstEnums": true,                       /* Disable erasing 'const enum' declarations in generated code. */
+    // "declarationDir": "./",                           /* Specify the output directory for generated declaration files. */
+    // "preserveValueImports": true,                     /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
+
+    /* Interop Constraints */
+    // "isolatedModules": true,                          /* Ensure that each file can be safely transpiled without relying on other imports. */
+    // "verbatimModuleSyntax": true,                     /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */
+    // "allowSyntheticDefaultImports": true,             /* Allow 'import x from y' when a module doesn't have a default export. */
+    "esModuleInterop": true,                             /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */
+    // "preserveSymlinks": true,                         /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
+    "forceConsistentCasingInFileNames": true,            /* Ensure that casing is correct in imports. */
+
+    /* Type Checking */
+    "strict": true,                                      /* Enable all strict type-checking options. */
+    "noImplicitAny": true,                               /* Enable error reporting for expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,                         /* When type checking, take into account 'null' and 'undefined'. */
+    // "strictFunctionTypes": true,                      /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
+    // "strictBindCallApply": true,                      /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
+    // "strictPropertyInitialization": true,             /* Check for class properties that are declared but not set in the constructor. */
+    // "noImplicitThis": true,                           /* Enable error reporting when 'this' is given the type 'any'. */
+    // "useUnknownInCatchVariables": true,               /* Default catch clause variables as 'unknown' instead of 'any'. */
+    "alwaysStrict": true,                                /* Ensure 'use strict' is always emitted. */
+    // "noUnusedLocals": true,                           /* Enable error reporting when local variables aren't read. */
+    // "noUnusedParameters": true,                       /* Raise an error when a function parameter isn't read. */
+    // "exactOptionalPropertyTypes": true,               /* Interpret optional property types as written, rather than adding 'undefined'. */
+    // "noImplicitReturns": true,                        /* Enable error reporting for codepaths that do not explicitly return in a function. */
+    // "noFallthroughCasesInSwitch": true,               /* Enable error reporting for fallthrough cases in switch statements. */
+    // "noUncheckedIndexedAccess": true,                 /* Add 'undefined' to a type when accessed using an index. */
+    // "noImplicitOverride": true,                       /* Ensure overriding members in derived classes are marked with an override modifier. */
+    // "noPropertyAccessFromIndexSignature": true,       /* Enforces using indexed accessors for keys declared using an indexed type. */
+    // "allowUnusedLabels": true,                        /* Disable error reporting for unused labels. */
+    // "allowUnreachableCode": true,                     /* Disable error reporting for unreachable code. */
+
+    /* Completeness */
+    // "skipDefaultLibCheck": true,                      /* Skip type checking .d.ts files that are included with TypeScript. */
+    "skipLibCheck": true                                 /* Skip type checking all .d.ts files. */
+  },
+  "include": [
+  "*.js",
+  "*/*.js"
+],
+  "exclude": ["node_modules"]
+}

julia/README.md

@@ -0,0 +1,128 @@
+---
+title: Julia
+homepage: https://julialang.org/
+tagline: |
+  Julia: A Language for Data Science, Visualization, and Machine Learning
+---
+
+To update or switch versions, run `webi julia@stable` (or `@v1.10`, `@beta`,
+etc).
+
+## Cheat Sheet
+
+> Julia is a programming language for Data Science - a far better alternative to
+> (or plugin for) Python when performance matters.
+
+## Table of Contents
+
+- Files
+- Hello World
+- Compile for Distribution
+- Vim Plugin
+
+### Files
+
+These are the files / directories that are created and/or modified with this
+install:
+
+```text
+~/.config/envman/PATH.env
+~/.local/opt/julia/
+```
+
+### How to create a Hello World with Julia
+
+Try out the REPL by coping and pasting the code below (it'll print some nice
+ASCII art to the console):
+
+```sh
+julia -i
+```
+
+```julia
+function mandelbrot(a)
+    z = 0
+    for i=1:50
+        z = z^2 + a
+    end
+    return z
+end
+
+for y=1.0:-0.05:-1.0
+    for x=-2.0:0.0315:0.5
+        abs(mandelbrot(complex(x, y))) < 2 ? print("*") : print(" ")
+    end
+    println()
+end
+```
+
+Or write the program to `./mandelbrot.jl` and run it:
+
+```sh
+julia ./mandelbrot.jl
+```
+
+## How to Build a Distributable Binary with Julia
+
+Here's an example project that you can build:
+
+1. Clone and enter the example project
+   ```sh
+   git clone --depth=1 https://github.com/JuliaLang/PackageCompiler.jl ./PackageCompiler
+   pushd ./PackageCompiler/examples
+   ```
+2. Run Julia in project mode
+   ```sh
+   julia -q --project
+   ```
+3. Compile the binary
+   ```julia
+   using PackageCompiler
+   create_app("MyApp", "MyAppCompiled")
+   exit()
+   ```
+4. Test & Enjoy
+   ```sh
+   ./MyAppCompiled/bin/MyApp foo bar --julia-args -t4
+   ```
+
+See also:
+
+- <https://docs.juliahub.com/PackageCompiler/MMV8C/2.1.16/apps.html>
+- <https://github.com/JuliaLang/PackageCompiler.jl/tree/master/examples/MyApp>
+
+## How to Install the Language Server (for VSCode, Vim, etc)
+
+Open the Julia REPL and add `LanguageServer`:
+
+```sh
+julia -i
+```
+
+```jl
+using Pkg
+Pkg.add("LanguageServer")
+Pkg.add("SymbolServer")
+```
+
+You'll need to reinstall if you switch environments.
+
+## How to Install the Julia Vim Plugin
+
+`julia-vim` adds support for:
+
+- Latex-to-Unicode
+- jumping from between start and end of blocks with `%` \
+  (and other block jump sequences)
+
+```sh
+mkdir -p ~/.vim/pack/plugins/start
+git clone https://github.com/JuliaEditorSupport/julia-vim.git \
+    ~/.vim/pack/plugins/start/julia-vim
+```
+
+Usage Examples:
+
+- `\alpha<tab>` will produce `α` \
+  (meaning typing `\alpha` and hitting the `tab` key)
+- Hitting `%` on an `end` will take you to the `function` or `for`, etc

julia/install.ps1

@@ -0,0 +1,56 @@
+#!/usr/bin/env pwsh
+
+#################
+# Install julia #
+#################
+
+# Every package should define these variables
+$pkg_cmd_name = "julia"
+
+$pkg_dst_cmd = "$Env:USERPROFILE\.local\opt\julia\bin\julia.exe"
+$pkg_dst_bin = "$Env:USERPROFILE\.local\opt\julia\bin"
+$pkg_dst_dir = "$Env:USERPROFILE\.local\opt\julia"
+$pkg_dst = "$pkg_dst_dir"
+
+$pkg_src_cmd = "$Env:USERPROFILE\.local\opt\julia-v$Env:WEBI_VERSION\bin\julia.exe"
+$pkg_src_dir = "$Env:USERPROFILE\.local\opt\julia-v$Env:WEBI_VERSION"
+$pkg_src = "$pkg_src_dir"
+
+New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
+$pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
+
+# Fetch archive
+IF (!(Test-Path -Path "$pkg_download")) {
+    Write-Output "Downloading julia from $Env:WEBI_PKG_URL to $pkg_download"
+    & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
+    & Move-Item "$pkg_download.part" "$pkg_download"
+}
+
+IF (!(Test-Path -Path "$pkg_src_dir")) {
+    Write-Output "Installing julia"
+
+    # TODO: create package-specific temp directory
+    # Enter tmp
+    Push-Location .local\tmp
+
+    # Remove any leftover tmp cruft
+    Remove-Item -Path ".\julia*" -Recurse -ErrorAction Ignore
+
+    # Unpack archive file into this temporary directory
+    # Windows BSD-tar handles zip. Imagine that.
+    Write-Output "Unpacking $pkg_download"
+    & tar xf "$pkg_download"
+
+    # Settle unpacked archive into place
+    Write-Output "Install Location: $pkg_src_cmd"
+    Move-Item -Path ".\julia*" -Destination "$pkg_src_dir"
+
+    # Exit tmp
+    Pop-Location
+}
+
+Write-Output "Copying into '$pkg_dst' from '$pkg_src'"
+Remove-Item -Path "$pkg_dst" -Recurse -ErrorAction Ignore | Out-Null
+Copy-Item -Path "$pkg_src" -Destination "$pkg_dst" -Recurse
+webi_path_add "$pkg_dst_bin"
+& "$pkg_dst_cmd" --version

julia/install.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+
+# shellcheck disable=SC2034
+# "'pkg_cmd_name' appears unused. Verify it or export it."
+
+__init_julia() {
+    set -e
+    set -u
+
+    ##################
+    # Install julia #
+    ##################
+
+    # Every package should define these 6 variables
+    pkg_cmd_name="julia"
+
+    pkg_dst_cmd="$HOME/.local/opt/julia/bin/julia"
+    pkg_dst_dir="$HOME/.local/opt/julia"
+    pkg_dst="$pkg_dst_dir"
+
+    pkg_src_cmd="$HOME/.local/opt/julia-v$WEBI_VERSION/bin/julia"
+    pkg_src_dir="$HOME/.local/opt/julia-v$WEBI_VERSION"
+    pkg_src="$pkg_src_dir"
+
+    # pkg_install must be defined by every package
+    pkg_install() {
+        # ~/.local/opt/julia-v3.27.0/
+        mkdir -p "$(dirname "${pkg_src_dir}")"
+
+        # mv ./julia-*/ ~/.local/opt/julia-v3.27.0/
+        mv ./julia-*/ "${pkg_src_dir}"
+    }
+
+    # pkg_get_current_version is recommended, but not required
+    pkg_get_current_version() {
+        # 'julia --version' has output in this format:
+        #       julia version 1.10.0-rc1
+        # This trims it down to just the version number:
+        #       1.10.0-rc1
+        julia --version 2> /dev/null |
+            head -n 1 |
+            cut -d ' ' -f 3
+    }
+
+}
+
+__init_julia