ref: make ./builds-cacher.js one-off testable; add ./classify-one.js

Signed-off-by: Nikolay Nikolaev <nicknickolaev@gmail.com>

.github/workflows/node.js.yml

@@ -23,10 +23,15 @@ jobs:
           sh ./_scripts/install-ci-deps
           echo "${HOME}/.local/bin" >> $GITHUB_PATH
           echo "${HOME}/.local/opt/pwsh" >> $GITHUB_PATH
+      - run: |
+          git submodule init
+          git submodule update
       - run: shfmt --version
       - run: shellcheck -V
       - run: node --version
       - run: npm run fmt
       - run: npm ci
       - run: npm run lint
-      - run: npm run test
+      - env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: npm run test

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "_webi/build-classifier"]
+	path = _webi/build-classifier
+	url = https://github.com/webinstall/webi-build-classifier.git

README.md

@@ -98,9 +98,10 @@ You just fill in the blanks.
 Just create an empty directory and run the tests until you get a good result.
 
 ```sh
-git clone git@github.com:webinstall/packages.git
-pushd packages
-npm install
+git clone git@github.com:webinstall/webi-installers.git
+pushd ./webi-installers/
+git submodule update --init
+npm clean-install
 ```
 
 ```sh

_common/gitea.js

@@ -1,35 +1,46 @@
 'use strict';
 
-var ghRelease = require('./github.js');
+var GitHubish = require('./githubish.js');
 
 /**
- * Gets the releases for 'ripgrep'. This function could be trimmed down and made
- * for use with any github release.
+ * Lists Gitea Releases (w/ uploaded assets)
  *
- * @param request
- * @param {string} owner
- * @param {string} repo
- * @returns {PromiseLike<any> | Promise<any>}
+ * @param {any} _request - deprecated
+ * @param {String} owner
+ * @param {String} repo
+ * @param {String} baseurl
+ * @param {String} [username]
+ * @param {String} [token]
  */
-function getAllReleases(request, owner, repo, baseurl) {
-  if (!baseurl) {
-    return Promise.reject('missing baseurl');
-  }
-  return ghRelease(request, owner, repo, baseurl + '/api/v1').then(
-    function (all) {
-      return all;
-    },
-  );
+async function getAllReleases(
+  _request,
+  owner,
+  repo,
+  baseurl,
+  username = '',
+  token = '',
+) {
+  baseurl = `${baseurl}/api/v1`;
+  let all = await GitHubish.getAllReleases({
+    owner,
+    repo,
+    baseurl,
+    username,
+    token,
+  });
+  return all;
 }
 
 module.exports = getAllReleases;
 
 if (module === require.main) {
   getAllReleases(
-    require('@root/request'),
-    'coolaj86',
-    'go-pathman',
-    'https://git.coolaj86.com',
+    null,
+    'root',
+    'pathman',
+    'https://git.rootprojects.org',
+    '',
+    '',
   ).then(
     //getAllReleases(require('@root/request'), 'root', 'serviceman', 'https://git.rootprojects.org').then(
     function (all) {

_common/github.js

@@ -1,102 +1,41 @@
 'use strict';
 
-require('dotenv').config();
+require('dotenv').config({ path: '.env' });
+
+let GitHubish = require('./githubish.js');
 
 /**
  * Lists GitHub Releases (w/ uploaded assets)
  *
- * @param request
- * @param {string} owner
- * @param {string} repo
- * @returns {PromiseLike<any> | Promise<any>}
+ * @param {any} _request - deprecated
+ * @param {String} owner
+ * @param {String} repo
+ * @param {String} [baseurl]
+ * @param {String} [username]
+ * @param {String} [token]
  */
 async function getAllReleases(
-  request,
+  _request,
   owner,
   repo,
   baseurl = 'https://api.github.com',
+  username = process.env.GITHUB_USERNAME || '',
+  token = process.env.GITHUB_TOKEN || '',
 ) {
-  if (!owner) {
-    throw new Error('missing owner for repo');
-  }
-  if (!repo) {
-    throw new Error('missing repo name');
-  }
-
-  var req = {
-    url: `${baseurl}/repos/${owner}/${repo}/releases`,
-    json: true,
-  };
-
-  // TODO I really don't like global config, find a way to do better
-  if (process.env.GITHUB_USERNAME) {
-    req.auth = {
-      user: process.env.GITHUB_USERNAME,
-      pass: process.env.GITHUB_TOKEN,
-    };
-  }
-
-  let resp = await request(req);
-  if (!resp.ok) {
-    console.error('Bad Resp Headers:', resp.headers);
-    console.error('Bad Resp Body:', resp.body);
-    throw new Error('the elusive releases BOOGEYMAN strikes again');
-  }
-
-  let gHubResp = resp.body;
-  let all = {
-    releases: [],
-    // todo make this ':baseurl' + ':releasename'
-    download: '',
-  };
-
-  try {
-    gHubResp.forEach(transformReleases);
-  } catch (e) {
-    console.error(e.message);
-    console.error('Error Headers:', resp.headers);
-    console.error('Error Body:', resp.body);
-    throw e;
-  }
-
-  function transformReleases(release) {
-    for (let asset of release['assets']) {
-      let name = asset['name'];
-      let date = release['published_at']?.replace(/T.*/, '');
-      let download = asset['browser_download_url'];
-
-      // TODO tags aren't always semver / sensical
-      let version = release['tag_name'];
-      let channel;
-      if (release['prerelease']) {
-        // -rcX, -preview, -beta, etc will be checked in _webi/normalize.js
-        channel = 'beta';
-      }
-      let lts = /(\b|_)(lts)(\b|_)/.test(release['tag_name']);
-
-      all.releases.push({
-        name: name,
-        version: version,
-        lts: lts,
-        channel: channel,
-        date: date,
-        os: '', // will be guessed by download filename
-        arch: '', // will be guessed by download filename
-        ext: '', // will be normalized
-        download: download,
-      });
-    }
-  }
-
+  let all = await GitHubish.getAllReleases({
+    owner,
+    repo,
+    baseurl,
+    username,
+    token,
+  });
   return all;
 }
 
 module.exports = getAllReleases;
 
 if (module === require.main) {
-  getAllReleases(require('@root/request'), 'BurntSushi', 'ripgrep').then(
-    function (all) {
-      console.info(JSON.stringify(all, null, 2));
-    },
-  );
+  getAllReleases(null, 'BurntSushi', 'ripgrep').then(function (all) {
+    console.info(JSON.stringify(all, null, 2));
+  });
 }

_common/githubish.js

@@ -0,0 +1,124 @@
+'use strict';
+
+let GitHubish = module.exports;
+
+/**
+ * Lists GitHub-Like Releases (w/ uploaded assets)
+ *
+ * @param {Object} opts
+ * @param {String} opts.owner
+ * @param {String} opts.repo
+ * @param {String} opts.baseurl
+ * @param {String} [opts.username]
+ * @param {String} [opts.token]
+ */
+GitHubish.getAllReleases = async function ({
+  owner,
+  repo,
+  baseurl,
+  username = '',
+  token = '',
+}) {
+  if (!owner) {
+    throw new Error('missing owner for repo');
+  }
+  if (!repo) {
+    throw new Error('missing repo name');
+  }
+  if (!baseurl) {
+    throw new Error('missing baseurl');
+  }
+
+  let url = `${baseurl}/repos/${owner}/${repo}/releases`;
+  let opts = {
+    headers: {
+      'Content-Type': 'appplication/json',
+    },
+  };
+
+  if (token) {
+    let userpass = `${username}:${token}`;
+    let basicAuth = btoa(userpass);
+    Object.assign(opts.headers, {
+      Authorization: `Basic ${basicAuth}`,
+    });
+  }
+
+  let resp = await fetch(url, opts);
+  if (!resp.ok) {
+    let headers = Array.from(resp.headers);
+    console.error('Bad Resp Headers:', headers);
+    let text = await resp.text();
+    console.error('Bad Resp Body:', text);
+    let msg = `failed to fetch releases from '${baseurl}' with user '${username}'`;
+    throw new Error(msg);
+  }
+
+  let respText = await resp.text();
+  let gHubResp;
+  try {
+    gHubResp = JSON.parse(respText);
+  } catch (e) {
+    console.error('Bad Resp JSON:', respText);
+    console.error(e.message);
+    let msg = `failed to parse releases from '${baseurl}' with user '${username}'`;
+    throw new Error(msg);
+  }
+
+  let all = {
+    releases: [],
+    // todo make this ':baseurl' + ':releasename'
+    download: '',
+  };
+
+  try {
+    gHubResp.forEach(transformReleases);
+  } catch (e) {
+    console.error(e.message);
+    console.error('Error Headers:', resp.headers);
+    console.error('Error Body:', resp.body);
+    let msg = `failed to transform releases from '${baseurl}' with user '${username}'`;
+    throw new Error(msg);
+  }
+
+  function transformReleases(release) {
+    for (let asset of release['assets']) {
+      let name = asset['name'];
+      let date = release['published_at']?.replace(/T.*/, '');
+      let download = asset['browser_download_url'];
+
+      // TODO tags aren't always semver / sensical
+      let version = release['tag_name'];
+      let channel;
+      if (release['prerelease']) {
+        // -rcX, -preview, -beta, etc will be checked in _webi/normalize.js
+        channel = 'beta';
+      }
+      let lts = /(\b|_)(lts)(\b|_)/.test(release['tag_name']);
+
+      all.releases.push({
+        name: name,
+        version: version,
+        lts: lts,
+        channel: channel,
+        date: date,
+        os: '', // will be guessed by download filename
+        arch: '', // will be guessed by download filename
+        ext: '', // will be normalized
+        download: download,
+      });
+    }
+  }
+
+  return all;
+};
+
+if (module === require.main) {
+  GitHubish.getAllReleases({
+    owner: 'BurntSushi',
+    repo: 'ripgrep',
+    baseurl: 'https://api.github.com',
+  }).then(function (all) {
+    console.info(JSON.stringify(all, null, 2));
+  });
+}

_example/README.md

@@ -23,7 +23,7 @@ install:
 ```text
 ~/.config/envman/PATH.env
 ~/.local/bin/foo
-~/.local/opt/foo
+~/.local/opt/foo/
 ```
 
 ## Cheat Sheet

_example/releases.js

@@ -12,17 +12,19 @@ var repo = 'ripgrep';
 /**                                                                          **/
 /******************************************************************************/
 
-module.exports = function (request) {
-  return github(request, owner, repo).then(function (all) {
-    return all;
-  });
+module.exports = async function (request) {
+  let all = await github(request, owner, repo);
+  return all;
 };
 
 if (module === require.main) {
-  module.exports(require('@root/request')).then(function (all) {
-    all = require('../_webi/normalize.js')(all);
+  (async function () {
+    let request = require('@root/request');
+    let normalize = require('../_webi/normalize.js');
+    let all = await module.exports(request);
+    all = normalize(all);
     // just select the first 5 for demonstration
     all.releases = all.releases.slice(0, 5);
     console.info(JSON.stringify(all, null, 2));
-  });
+  })();
 }

_webi/builds-cacher-test.js

@@ -0,0 +1,73 @@
+'use strict';
+
+let Path = require('node:path');
+
+let BuildsCacher = require('./builds-cacher.js');
+// let Parallel = require('./parallel.js');
+
+var INSTALLERS_DIR = Path.join(__dirname, '..');
+var CACHE_DIR = Path.join(__dirname, '../_cache');
+
+async function main() {
+  let bc = BuildsCacher.create({
+    caches: CACHE_DIR,
+    installers: INSTALLERS_DIR,
+  });
+  bc.freshenRandomPackage(600 * 1000);
+
+  // let dirs = await bc.getProjectsByType();
+  // let projNames = Object.keys(dirs.valid);
+
+  let lastUpdate;
+
+  let projName = 'k9s';
+  {
+    let packages = await bc.getPackages({
+      //Releases: Releases,
+      name: projName,
+      date: new Date(),
+    });
+    lastUpdate = packages.updated;
+    console.info(
+      `Last update for '${projName}': ${packages.updated} (${packages.releases.length} assets)`,
+    );
+  }
+
+  console.info('Waiting 5s');
+  {
+    setTimeout(async function () {
+      let packages = await bc.getPackages({
+        //Releases: Releases,
+        name: projName,
+        date: new Date(),
+      });
+      console.info(
+        `Last update for '${projName}': ${packages.updated} (${packages.releases.length} assets)`,
+      );
+      if (lastUpdate < packages.updated) {
+        console.info(`PASS`);
+      } else {
+        console.info(`MAYBE fail`);
+      }
+    }, 5 * 1000);
+  }
+
+  //let parallel = 25;
+  //await Parallel.run(parallel, projNames, getAll);
+  //async function getAll(name) {
+  //  void (await bc.getPackages({
+  //    //Releases: Releases,
+  //    name: name,
+  //    date: new Date(),
+  //  }));
+  //}
+}
+
+main()
+  .then(function () {
+    console.log('Done');
+  })
+  .catch(function (e) {
+    console.error(e.stack || e);
+    process.exit(1);
+  });

_webi/builds-cacher.js

@@ -0,0 +1,995 @@
+'use strict';
+
+var BuildsCacher = module.exports;
+
+let Fs = require('node:fs/promises');
+let Path = require('node:path');
+
+let HostTargets = require('./build-classifier/host-targets.js');
+let Lexver = require('./build-classifier/lexver.js');
+let Triplet = require('./build-classifier/triplet.js');
+
+let request = require('@root/request');
+
+var ALIAS_RE = /^alias: (\w+)$/m;
+
+var LEGACY_ARCH_MAP = {
+  '*': 'ANYARCH',
+  arm64: 'aarch64',
+  armv6l: 'armv6',
+  armv7l: 'armv7',
+  amd64: 'x86_64',
+  mipsle: 'mipsel',
+  mips64le: 'mips64el',
+  mipsr6le: 'mipsr6el',
+  mips64r6le: 'mips64r6el',
+  // yes... el for arm and mips, but le for ppc
+  // (perhaps the joke got old?)
+  ppc64el: 'ppc64le',
+  386: 'x86',
+};
+var LEGACY_OS_MAP = {
+  '*': 'ANYOS',
+  macos: 'darwin',
+  posix: 'posix_2017',
+};
+
+var TERMS_META = [
+  // pattern
+  '{ARCH}',
+  '{EXT}',
+  '{LIBC}',
+  '{NAME}',
+  '{OS}',
+  '{VENDOR}',
+  // // os-/arch-indepedent
+  // 'ANYARCH',
+  // 'ANYOS',
+  // // libc
+  // 'none',
+  // channel
+  'beta',
+  'dev',
+  'preview',
+  'stable',
+];
+
+/** @typedef {String} TripletString - {arch}-{vendor}-{os}-{libc} */
+/** @typedef {String} VersionString */
+/** @typedef {Object.<VersionString, Array<BuildAsset>>} PackagesByRelease */
+
+/**
+ * @typedef ProjectInfo
+ * @prop {Array<BuildAsset>} releases
+ * @prop {Array<BuildAsset>} packages
+ * @prop {Object.<TripletString, PackagesByRelease>} releasesByTriplet
+ * @prop {Array<import('./build-classifier/types.js').ArchString>} arches
+ * @prop {Array<import('./build-classifier/types.js').OsString>} oses
+ * @prop {Array<import('./build-classifier/types.js').LibcString>} libcs
+ * @prop {Array<String>} channels
+ * @prop {Array<String>} formats
+ * @prop {Array<String>} triplets
+ * @prop {Array<String>} versions
+ * @prop {Array<String>} lexvers
+ * @prop {Object.<String, String>} lexversMap
+ */
+
+/**
+ * @typedef BuildAsset
+ * @prop {String} name
+ * @prop {String} version
+ * @prop {Boolean} lts
+ * @prop {String} date
+ * @prop {String} arch
+ * @prop {String} os
+ * @prop {String} libc
+ * @prop {String} ext
+ * @prop {String} download
+ */
+
+/**
+ * @typedef VersionTarget
+ * @prop {String} version
+ * @prop {Boolean} lts
+ * @prop {String} channel
+ */
+
+/** @typedef {TargetTriplet & HostTargetPartial} HostTarget */
+/** @typedef {import('./build-classifier/types.js').TargetTriplet} TargetTriplet */
+/**
+ * @typedef HostTargetPartial
+ * @prop {String} target.triplet - os-vendor-arch-libc
+ * @prop {Error} target.error
+ */
+
+async function getPartialHeader(path) {
+  let readme = `${path}/README.md`;
+  let head = await readFirstBytes(readme).catch(function (err) {
+    if (err.code !== 'ENOENT') {
+      console.warn(`warn: ${path}: ${err.message}`);
+    }
+    return null;
+  });
+
+  return head;
+}
+
+// let fsOpen = util.promisify(Fs.open);
+// let fsRead = util.promisify(Fs.read);
+async function readFirstBytes(path) {
+  let start = 0;
+  let n = 1024;
+  let fh = await Fs.open(path, 'r');
+  let buf = Buffer.alloc(n);
+  let result = await fh.read(buf, start, n);
+  let str = result.buffer.toString('utf8');
+  await fh.close();
+
+  return str;
+}
+
+let promises = {};
+async function getLatestBuilds(Releases, installersDir, cacheDir, name, date) {
+  console.info(`[INFO] getLatestBuilds: ${name}`);
+
+  if (!Releases) {
+    Releases = require(`${installersDir}/${name}/releases.js`);
+  }
+  // TODO update all releases files with module.exports.xxxx = 'foo';
+  if (!Releases.latest) {
+    Releases.latest = Releases;
+  }
+
+  let id = `${cacheDir}/${name}`;
+  if (!promises[id]) {
+    promises[id] = Promise.resolve();
+  }
+
+  promises[id] = promises[id].then(async function () {
+    return await getLatestBuildsInner(Releases, cacheDir, name, date);
+  });
+
+  return await promises[id];
+}
+
+async function getLatestBuildsInner(Releases, cacheDir, name, date) {
+  let data = await Releases.latest(request);
+
+  if (!date) {
+    date = new Date();
+  }
+  let isoDate = date.toISOString();
+  let yearMonth = isoDate.slice(0, 7);
+
+  // TODO hash file
+  let dataFile = `${cacheDir}/${yearMonth}/${name}.json`;
+  // TODO fsstat releases.js vs require-ing time as well
+  let tsFile = `${cacheDir}/${yearMonth}/${name}.updated.txt`;
+
+  let dirPath = Path.dirname(dataFile);
+  await Fs.mkdir(dirPath, { recursive: true });
+
+  let json = JSON.stringify(data, null, 2);
+  await Fs.writeFile(dataFile, json, 'utf8');
+
+  let seconds = date.valueOf();
+  let ms = seconds / 1000;
+  let msStr = ms.toFixed(3);
+  await Fs.writeFile(tsFile, msStr, 'utf8');
+
+  return data;
+}
+
+BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
+  let installersDir = installers;
+  let cacheDir = caches;
+
+  if (!ALL_TERMS) {
+    ALL_TERMS = Triplet.TERMS_PRIMARY_MAP;
+  }
+
+  let bc = {};
+  bc.ALL_TERMS = ALL_TERMS;
+  bc.orphanTerms = Object.assign({}, bc.ALL_TERMS);
+  bc.unknownTerms = {};
+  bc.usedTerms = {};
+  bc.formats = [];
+  bc._triplets = {};
+  bc._targetsByBuildIdCache = {};
+  bc._caches = {};
+  bc._staleAge = 15 * 60 * 1000;
+  bc._allFormats = {};
+  bc._allTriplets = {};
+
+  for (let term of TERMS_META) {
+    delete bc.orphanTerms[term];
+  }
+
+  bc.getProjectsByType = async function () {
+    let dirs = {
+      hidden: {},
+      errors: {},
+      alias: {},
+      invalid: {},
+      selfhosted: {},
+      valid: {},
+    };
+
+    let entries = await Fs.readdir(installersDir, { withFileTypes: true });
+    for (let entry of entries) {
+      let meta = await bc.getProjectTypeByEntry(entry);
+      if (meta.type === 'not_found') {
+        let err = meta.detail;
+        console.error('');
+        console.error('PROBLEM');
+        console.error(`    ${err.message}`);
+        console.error('');
+        console.error('SOLUTION');
+        console.error('    npm clean-install');
+        console.error('');
+        throw new Error(
+          '[SANITY FAIL] should never have missing modules in prod',
+        );
+      }
+      dirs[meta.type][entry.name] = meta.detail;
+    }
+
+    return dirs;
+  };
+
+  /**
+   * Get project type and detail - alias, selfhosted, valid (and the invalids)
+   * @param {String} name - filename
+   */
+  bc.getProjectType = async function (name) {
+    let filepath = Path.join(installersDir, name);
+    let entry;
+    try {
+      entry = await Fs.lstat(filepath);
+      Object.assign(entry, { name: name });
+    } catch (e) {
+      return { type: 'errors', detail: 'not found' };
+    }
+    let info = await bc.getProjectTypeByEntry(entry);
+
+    return info;
+  };
+
+  /**
+   * Get project type and detail - alias, selfhosted, valid (and the invalids)
+   * @param {fs.Stats|fs.Dirent} entry
+   */
+  bc.getProjectTypeByEntry = async function (entry) {
+    let path = Path.join(installersDir, entry.name);
+
+    // skip non-installer dirs
+    if (entry.isSymbolicLink()) {
+      let link = await Fs.readlink(path);
+      return { type: 'alias', detail: link };
+    }
+
+    if (!entry.isDirectory()) {
+      return { type: 'hidden', detail: '!directory' };
+    }
+    if (entry.name === 'node_modules') {
+      return { type: 'hidden', detail: 'node_modules' };
+    }
+    if (entry.name.startsWith('_')) {
+      return { type: 'hidden', detail: '_*' };
+    }
+    if (entry.name.startsWith('.')) {
+      return { type: 'hidden', detail: '.*' };
+    }
+    if (entry.name.startsWith('~')) {
+      return { type: 'hidden', detail: '~*' };
+    }
+    if (entry.name.endsWith('~')) {
+      return { type: 'hidden', detail: '*~' };
+    }
+
+    // skip invalid installers
+    let head = await getPartialHeader(path);
+    if (!head) {
+      return { type: 'invalid', detail: '!README.md' };
+    }
+
+    let alias = head.match(ALIAS_RE);
+    if (alias) {
+      let link = alias[1];
+      return { type: 'alias', detail: link };
+    }
+
+    let releasesPath = Path.join(path, 'releases.js');
+    try {
+      void require(releasesPath);
+    } catch (err) {
+      if (err.code !== 'MODULE_NOT_FOUND') {
+        return { type: 'errors', detail: err };
+      }
+
+      if (err.message.includes(`Cannot find module '${releasesPath}'`)) {
+        return { type: 'selfhosted', detail: true };
+      }
+
+      return { type: 'not_found', detail: err };
+    }
+
+    return { type: 'valid', detail: true };
+  };
+
+  // Typically a package is organized by release (ex: go has 1.20, 1.21, etc),
+  // but we will organize by the build (ex: go1.20-darwin-arm64.tar.gz, etc).
+  bc.getPackages = async function ({ Releases, name, date }) {
+    if (!date) {
+      date = new Date();
+    }
+    let isoDate = date.toISOString();
+    let yearMonth = isoDate.slice(0, 7);
+    let dataFile = `${cacheDir}/${yearMonth}/${name}.json`;
+    let tsFile = `${cacheDir}/${yearMonth}/${name}.updated.txt`;
+
+    let tsDate;
+    {
+      let secondsStr = await Fs.readFile(tsFile, 'ascii').catch(function (err) {
+        if (err.code !== 'ENOENT') {
+          throw err;
+        }
+        return '0';
+      });
+      secondsStr = secondsStr.trim();
+      let seconds = parseFloat(secondsStr) || 0;
+      let ms = seconds * 1000;
+      tsDate = new Date(ms);
+    }
+
+    let projInfo = bc._caches[name];
+
+    let meta = {
+      // version info
+      versions: projInfo?.versions || [],
+      lexvers: projInfo?.lexvers || [],
+      lexversMap: projInfo?.lexversMap || {},
+      // culled release assets
+      packages: projInfo?.packages || [],
+      releasesByTriplet: projInfo?.releasesByTriplet || {},
+      // target info
+      triplets: projInfo?.triplets || [],
+      oses: projInfo?.oses || [],
+      arches: projInfo?.arches || [],
+      libcs: projInfo?.libcs || [],
+      formats: projInfo?.formats || [],
+      // TODO channels: projInfo?.channels || [],
+    };
+
+    if (!projInfo) {
+      let json = await Fs.readFile(dataFile, 'ascii').catch(
+        async function (err) {
+          if (err.code !== 'ENOENT') {
+            throw err;
+          }
+
+          return null;
+        },
+      );
+
+      try {
+        projInfo = JSON.parse(json);
+      } catch (e) {
+        console.error(`error: ${dataFile}:\n\t${e.message}`);
+        projInfo = null;
+      }
+    }
+    if (!projInfo) {
+      projInfo = await getLatestBuilds(Releases, installersDir, cacheDir, name);
+    }
+    let latestProjInfo = await BuildsCacher.transformAndUpdate(
+      name,
+      projInfo,
+      meta,
+      tsDate,
+      bc,
+    );
+    bc._caches[name] = latestProjInfo;
+
+    process.nextTick(async function () {
+      let now = date.valueOf();
+      let age = now - projInfo.updated;
+
+      let fresh = age < bc._staleAge;
+      if (fresh) {
+        return;
+      }
+
+      projInfo = await getLatestBuilds(Releases, installersDir, cacheDir, name);
+      let latestProjInfo = BuildsCacher.transformAndUpdate(
+        name,
+        projInfo,
+        meta,
+        date,
+        bc,
+      );
+      bc._caches[name] = latestProjInfo;
+    });
+
+    return projInfo;
+  };
+
+  // Makes sure that packages are updated once an hour, on average
+  bc._staleNames = [];
+  bc._freshenTimeout = null;
+  bc.freshenRandomPackage = async function (minDelay) {
+    if (!minDelay) {
+      minDelay = 15 * 1000;
+    }
+
+    if (bc._staleNames.length === 0) {
+      let dirs = await bc.getProjectsByType();
+      bc._staleNames = Object.keys(dirs.valid);
+      bc._staleNames.sort(function () {
+        return 0.5 - Math.random();
+      });
+    }
+
+    let name = bc._staleNames.pop();
+    void (await bc.getPackages({
+      //Releases: Releases,
+      name: name,
+      date: new Date(),
+    }));
+    console.info(`[INFO] freshenRandomPackage: ${name}`);
+
+    let hour = 60 * 60 * 1000;
+    let delay = minDelay;
+    let spread = hour / bc._staleNames.length;
+    let seed = Math.random();
+    delay += seed * spread;
+
+    clearTimeout(bc._freshenTimeout);
+    bc._freshenTimeout = setTimeout(bc.freshenRandomPackage, delay);
+    bc._freshenTimeout.unref();
+  };
+
+  /**
+   * Given a list of acceptable formats, get the sorted list of of formats.
+   * Actually used (as per node _webi/lint-builds.js):
+   *     .7z
+   *     .app.zip
+   *     .dmg
+   *     .exe
+   *     .exe.xz
+   *     .git
+   *     .gz
+   *     .msi
+   *     .pkg
+   *     .pkg.tar.zst
+   *     .sh
+   *     .tar.gz
+   *     .tar.xz
+   *     .xz
+   *     .zip
+   */
+  bc.getSortedFormats = function (formats) {
+    /* jshint maxcomplexity: 25 */
+    formats.sort();
+    let id = formats.join(',');
+    if (bc._allFormats[id]) {
+      return bc._allFormats[id];
+    }
+
+    // we don't know how to handle any of these yet
+    // let exclude = [];
+    // let isAndroid = false;
+    // if (!isAndroid) {
+    //   exclude.push('.apk');
+    // }
+    // let isDebian = false;
+    // if (!isDebian) {
+    //   exclude.push('.deb');
+    // }
+    // let isEnterpriseLinux = false;
+    // if (!isEnterpriseLinux) {
+    //   exclude.push('.rpm');
+    // }
+    // let isArch = false;
+    // if (!isArch) {
+    //   exclude.push('.pkg.tar.zst');
+    // }
+
+    let hasExe = formats.includes('exe') || formats.includes('.exe');
+
+    /** @type {Array<String>} */
+    let exts = [];
+
+    let hasXz = formats.includes('xz') || formats.includes('.xz');
+    if (hasXz) {
+      exts.push('.tar.xz');
+      if (hasExe) {
+        exts.push('.exe.xz');
+      }
+      exts.push('.xz');
+    }
+    let hasZst = formats.includes('zst') || formats.includes('.zst');
+    if (hasZst) {
+      exts.push('.tar.zst');
+      exts.push('.zst');
+    }
+    let hasZip = formats.includes('zip') || formats.includes('.zip');
+    if (hasZip) {
+      exts.push('.zip');
+    }
+    let has7z = false;
+    if (has7z) {
+      exts.push('.7z');
+    }
+    // let hasBz2 = formats.includes('bz2') || formats.includes('.bz2');
+    // if (hasBz2) {
+    // 	exts.push('.bz2');
+    // }
+    if (hasExe) {
+      if (!hasZip) {
+        exts.push('.zip');
+      }
+      exts.push('.tar.gz');
+      exts.push('.gz');
+      exts.push('.exe');
+      exts.push('.msi');
+      //exts.push('.msixbundle');
+    } else {
+      exts.push('.tar.gz');
+      exts.push('.gz');
+      exts.push('.sh');
+    }
+    exts.push('.git');
+
+    // Fallbacks
+    // (we include everything to bubble an extract error over not found)
+    exts.push('.app.zip');
+    exts.push('.dmg');
+    exts.push('.pkg');
+    if (!hasXz) {
+      exts.push('.tar.xz');
+      if (hasExe) {
+        exts.push('.exe.xz');
+      }
+      exts.push('.xz');
+    }
+    if (!hasZip) {
+      if (!hasExe) {
+        exts.push('.zip');
+      }
+    }
+    if (!hasZst) {
+      exts.push('.tar.zst');
+      exts.push('.zst');
+    }
+    if (!has7z) {
+      exts.push('.7z');
+    }
+    // if (!hasRar) {
+    // 	exts.push('.rar');
+    // }
+    // exts.push('.tar.bz2');
+    // exts.push('.bz2');
+
+    bc._allFormats[id] = exts;
+    return exts;
+  };
+
+  bc.selectPackage = function (packages, formats) {
+    if (packages.length === 1) {
+      return packages[0];
+    }
+
+    let exts = bc.getSortedFormats(formats);
+    for (let ext of exts) {
+      for (let build of packages) {
+        if (build.ext === ext) {
+          return build;
+        }
+      }
+    }
+
+    return packages[0];
+  };
+
+  /**
+   * @param {ProjectInfo} projInfo
+   * @param {HostTarget} hostTarget
+   */
+  bc.enumerateLatestVersions = function (projInfo, hostTarget) {
+    let lexPrefix = '';
+    let matchInfo = Lexver.matchSorted(projInfo.lexvers, lexPrefix);
+    let verInfo = {
+      default: projInfo.lexversMap[matchInfo.default],
+      previous: projInfo.lexversMap[matchInfo.previous],
+      stable: projInfo.lexversMap[matchInfo.stable],
+      latest: projInfo.lexversMap[matchInfo.latest],
+    };
+
+    return verInfo;
+  };
+
+  /**
+   * @param {ProjectInfo} projInfo
+   * @param {HostTarget} hostTarget
+   * @param {VersionTarget} verTarget
+   */
+  bc.findMatchingPackages = function (projInfo, hostTarget, verTarget) {
+    let matchInfo = bc._enumerateVersions(projInfo, verTarget.version);
+    let triplets = bc._enumerateTriplets(hostTarget);
+    //console.log('dbg: matchInfo', matchInfo);
+
+    if (matchInfo) {
+      for (let _triplet of triplets) {
+        let targetReleases = projInfo.releasesByTriplet[_triplet];
+        if (!targetReleases) {
+          continue;
+        }
+
+        // Make sure that these releases are the expected version
+        // (ex: jq1.7 => darwin-arm64-libc, jq1.6 => darwin-x86_64-libc)
+        for (let matchver of matchInfo.matches) {
+          let ver = projInfo.lexversMap[matchver] || matchver;
+          let packages = targetReleases[ver];
+          if (!packages) {
+            continue;
+          }
+
+          let match = {
+            triplet: _triplet,
+            packages: packages,
+            latest: projInfo.versions[0],
+            version: ver,
+            versions: matchInfo,
+          };
+          return match;
+        }
+      }
+
+      return null;
+    }
+
+    for (let _triplet of triplets) {
+      let targetReleases = projInfo.releasesByTriplet[_triplet];
+      if (!targetReleases) {
+        continue;
+      }
+
+      let versions = Object.keys(targetReleases);
+      //console.log('dbg: targetRelease versions', versions);
+      let lexvers = [];
+      for (let version of versions) {
+        let lexPrefix = Lexver.parseVersion(version);
+        lexvers.push(lexPrefix);
+      }
+      lexvers.sort();
+      lexvers.reverse();
+      // TODO get the other matchInfo props
+
+      // Make sure that these releases are the expected version
+      // (ex: jq1.7 => darwin-arm64-libc, jq1.6 => darwin-x86_64-libc)
+      for (let matchver of lexvers) {
+        let ver = projInfo.lexversMap[matchver] || matchver;
+        let packages = targetReleases[ver];
+        //console.log('dbg: packages', packages);
+        if (!packages) {
+          continue;
+        }
+
+        let pkg = packages[0];
+        if (verTarget.lts) {
+          if (!pkg.lts) {
+            continue;
+          }
+
+          let match = {
+            triplet: _triplet,
+            packages: packages,
+            latest: projInfo.versions[0],
+            version: ver,
+            versions: matchInfo,
+          };
+          return match;
+        }
+
+        let wantChannel = verTarget.channel || 'stable';
+        let isChannel = pkg.channel || 'stable';
+        if (wantChannel === 'stable') {
+          if (isChannel !== 'stable') {
+            continue;
+          }
+        }
+        // latest, beta, alpha, rc, preview
+
+        let match = {
+          triplet: _triplet,
+          packages: packages,
+          latest: projInfo.versions[0],
+          version: ver,
+          versions: matchInfo,
+        };
+        return match;
+      }
+    }
+
+    return null;
+  };
+
+  bc._enumerateTriplets = function (hostTarget) {
+    let id = [hostTarget.os, hostTarget.arch, hostTarget.libc].join(',');
+    let triplets = bc._allTriplets[id] || [];
+    if (triplets.length > 0) {
+      return triplets;
+    }
+
+    let oses = [];
+    if (hostTarget.os === 'windows') {
+      oses = ['ANYOS', 'windows'];
+    } else if (hostTarget.os === 'android') {
+      oses = ['ANYOS', 'posix_2017', 'android', 'linux'];
+    } else {
+      oses = ['ANYOS', 'posix_2017', hostTarget.os];
+    }
+
+    let waterfall = HostTargets.WATERFALL[hostTarget.os] || {};
+    let arches = waterfall[hostTarget.arch] ||
+      HostTargets.WATERFALL.ANYOS[hostTarget.arch] || [hostTarget.arch];
+    arches = ['ANYARCH'].concat(arches);
+    let libcs = waterfall[hostTarget.libc] ||
+      HostTargets.WATERFALL.ANYOS[hostTarget.libc] || [hostTarget.libc];
+
+    for (let os of oses) {
+      for (let arch of arches) {
+        for (let libc of libcs) {
+          let triplet = `${os}-${arch}-${libc}`;
+          triplets.push(triplet);
+        }
+      }
+    }
+    bc._allTriplets[id] = triplets;
+
+    return triplets;
+  };
+
+  bc._enumerateVersions = function (projInfo, ver) {
+    if (!ver) {
+      return null;
+    }
+    let lexPrefix = Lexver.parsePrefix(ver);
+    let matchInfo = Lexver.matchSorted(projInfo.lexvers, lexPrefix);
+
+    return matchInfo;
+  };
+
+  return bc;
+};
+
+BuildsCacher._classify = function (bc, projInfo, build) {
+  /* jshint maxcomplexity: 25 */
+  let maybeInstallable = Triplet.maybeInstallable(projInfo, build);
+  if (!maybeInstallable) {
+    return null;
+  }
+
+  if (LEGACY_OS_MAP[build.os]) {
+    build.os = LEGACY_OS_MAP[build.os];
+  }
+  if (LEGACY_ARCH_MAP[build.arch]) {
+    build.arch = LEGACY_ARCH_MAP[build.arch];
+  }
+
+  // because some packages are shimmed to match a single download against
+  let preTarget = Object.assign({ os: '', arch: '', libc: '' }, build);
+
+  let targetId = `${preTarget.os}:${preTarget.arch}:${preTarget.libc}`;
+  let buildId = `${projInfo.name}:${targetId}@${build.download}`;
+  //console.log(`dbg: buildId`, buildId);
+  let target = bc._targetsByBuildIdCache[buildId];
+  if (target) {
+    Object.assign(build, { target: target, triplet: target.triplet });
+    return target;
+  }
+
+  let pattern = Triplet.toPattern(projInfo, build);
+  //console.log(`dbg: pattern`, pattern);
+  if (!pattern) {
+    let err = new Error(`no pattern generated for ${projInfo.name}`);
+    err.code = 'E_BUILD_NO_PATTERN';
+    target = { error: err };
+    bc._targetsByBuildIdCache[buildId] = target;
+    return target;
+  }
+
+  let rawTerms = pattern.split(/[_\{\}\/\.\-]+/g);
+  //console.log(`dbg: rawTerms`, rawTerms);
+  for (let term of rawTerms) {
+    delete bc.orphanTerms[term];
+    bc.usedTerms[term] = true;
+  }
+
+  // {NAME}/{NAME}-{VER}-Windows-x86_64_v2-musl.exe =>
+  //     {NAME}.windows.x86_64v2.musl.exe
+  let terms = Triplet.patternToTerms(pattern);
+  //console.log(`dbg: terms`, terms);
+  if (!terms.length) {
+    let err = new Error(`'${terms}' was trimmed to ''`);
+    target = { error: err };
+    bc._targetsByBuildIdCache[buildId] = target;
+    return target;
+  }
+
+  for (let term of terms) {
+    if (!term) {
+      continue;
+    }
+
+    if (bc.ALL_TERMS[term]) {
+      delete bc.orphanTerms[term];
+      bc.usedTerms[term] = true;
+      continue;
+    }
+
+    bc.unknownTerms[term] = true;
+  }
+
+  // {NAME}.windows.x86_64v2.musl.exe
+  //     windows-x86_64_v2-musl
+  target = { triplet: '' };
+  try {
+    void Triplet.termsToTarget(target, projInfo, build, terms);
+  } catch (e) {
+    console.error(`PACKAGE FORMAT CHANGE for '${projInfo.name}':`);
+    console.error(e.message);
+    console.error(build);
+    return null;
+  }
+
+  target.triplet = `${target.arch}-${target.vendor}-${target.os}-${target.libc}`;
+
+  {
+    // TODO I don't love this hidden behavior
+    // perhaps classify should just happen when the package is loaded
+    // (and the sanity error should be removed, or thrown after the loop is complete)
+    let hasTriplet = projInfo.triplets.includes(target.triplet);
+    if (!hasTriplet) {
+      projInfo.triplets.push(target.triplet);
+    }
+    let hasOs = projInfo.oses.includes(target.os);
+    if (!hasOs) {
+      projInfo.oses.push(target.os);
+    }
+    let hasArch = projInfo.arches.includes(target.arch);
+    if (!hasArch) {
+      projInfo.arches.push(target.arch);
+    }
+    let hasLibc = projInfo.libcs.includes(target.libc);
+    if (!hasLibc) {
+      projInfo.libcs.push(target.libc);
+    }
+
+    if (!build.ext) {
+      build.ext = Triplet.buildToPackageType(build);
+    }
+    if (build.ext) {
+      if (!build.ext.startsWith('.')) {
+        build.ext = `.${build.ext}`;
+      }
+    }
+    let hasExt = projInfo.formats.includes(build.ext);
+    if (!hasExt) {
+      projInfo.formats.push(build.ext);
+    }
+    let hasGlobalExt = bc.formats.includes(build.ext);
+    if (!hasGlobalExt) {
+      bc.formats.push(build.ext);
+    }
+  }
+
+  bc._triplets[target.triplet] = true;
+  bc._targetsByBuildIdCache[buildId] = target;
+
+  let triple = [target.arch, target.vendor, target.os, target.libc];
+  for (let term of triple) {
+    if (!bc.ALL_TERMS[term]) {
+      throw new Error(
+        `[SANITY FAIL] '${projInfo.name}' '${target.triplet}' generated unknown term '${term}'`,
+      );
+    }
+
+    delete bc.orphanTerms[term];
+    bc.usedTerms[term] = true;
+  }
+
+  return target;
+};
+
+BuildsCacher.transformAndUpdate = function (name, projInfo, meta, date, bc) {
+  meta.packages = [];
+
+  let updated = date.valueOf();
+
+  Object.assign(projInfo, { name, updated }, meta);
+  for (let build of projInfo.releases) {
+    let buildTarget = BuildsCacher._classify(bc, projInfo, build);
+    if (!buildTarget) {
+      // ignore known, non-package extensions
+      continue;
+    }
+
+    if (buildTarget.error) {
+      let err = buildTarget.error;
+      let code = err.code || '';
+      console.error(`[ERROR]: ${code} ${projInfo.name}: ${build.name}`);
+      console.error(`>>> ${err.message} <<<`);
+      console.error(projInfo);
+      console.error(build);
+      console.error(`^^^ ${err.message} ^^^`);
+      console.error(err.stack);
+      continue;
+    }
+
+    if (!build.name) {
+      build.name = build.download.replace(/.*\//, '');
+    }
+
+    build.target = buildTarget;
+    meta.packages.push(build);
+  }
+
+  BuildsCacher.updateReleasesByTriplet(meta);
+  BuildsCacher.updateAndSortVersions(projInfo, meta);
+
+  Object.assign(projInfo, { name, updated }, meta);
+  return projInfo;
+};
+
+// TODO
+//   - tag channels
+BuildsCacher.updateAndSortVersions = function (projInfo, meta) {
+  for (let build of projInfo.packages) {
+    let hasVersion = meta.versions.includes(build.version);
+    if (!hasVersion) {
+      build.lexver = Lexver.parseVersion(build.version);
+      meta.lexversMap[build.lexver] = build.version;
+    }
+  }
+
+  meta.lexvers = Object.keys(meta.lexversMap);
+  meta.lexvers.sort();
+  meta.lexvers.reverse();
+
+  meta.versions = [];
+  for (let lexver of meta.lexvers) {
+    let version = meta.lexversMap[lexver];
+    meta.versions.push(version);
+  }
+
+  projInfo.packages.sort(function (a, b) {
+    if (a.lexver > b.lexver) {
+      return -1;
+    }
+    if (a.lexver < b.lexver) {
+      return 1;
+    }
+    return 0;
+  });
+};
+
+BuildsCacher.updateReleasesByTriplet = function (meta) {
+  for (let build of meta.packages) {
+    let target = build.target;
+
+    let triplet = `${target.os}-${target.arch}-${target.libc}`;
+    if (!meta.releasesByTriplet[triplet]) {
+      meta.releasesByTriplet[triplet] = {};
+    }
+
+    let buildsByRelease = meta.releasesByTriplet[triplet];
+    if (!buildsByRelease[build.version]) {
+      buildsByRelease[build.version] = [];
+    }
+
+    let packages = buildsByRelease[build.version];
+    packages.push(build);
+  }
+};

_webi/builds.js

@@ -0,0 +1,41 @@
+'use strict';
+
+let Builds = module.exports;
+
+let Path = require('node:path');
+
+let BuildsCacher = require('./builds-cacher.js');
+// let HostTargets = require('./build-classifier/host-targets.js');
+let Parallel = require('./parallel.js');
+
+var INSTALLERS_DIR = Path.join(__dirname, '..');
+var CACHE_DIR = Path.join(__dirname, '../_cache');
+
+let bc = BuildsCacher.create({
+  caches: CACHE_DIR,
+  installers: INSTALLERS_DIR,
+});
+bc.freshenRandomPackage(600 * 1000);
+
+Builds.init = async function () {
+  bc.freshenRandomPackage(600 * 1000);
+
+  let dirs = await bc.getProjectsByType();
+  let projNames = Object.keys(dirs.valid);
+
+  let parallel = 25;
+  await Parallel.run(parallel, projNames, getAll);
+  async function getAll(name) {
+    void (await bc.getPackages({
+      //Releases: Releases,
+      name: name,
+      date: new Date(),
+    }));
+  }
+};
+
+Builds.enumerateLatestVersions = bc.enumerateLatestVersions;
+Builds.findMatchingPackages = bc.findMatchingPackages;
+Builds.getPackage = bc.getPackages;
+Builds.getProjectType = bc.getProjectType;
+Builds.selectPackage = bc.selectPackage;

_webi/classify-one.js

@@ -0,0 +1,92 @@
+'use strict';
+
+let Path = require('node:path');
+
+// let Builds = require('./builds.js');
+let BuildsCacher = require('./builds-cacher.js');
+let Triplet = require('./build-classifier/triplet.js');
+
+let request = require('@root/request');
+
+async function main() {
+  let projName = process.argv[2];
+  if (!projName) {
+    console.error(``);
+    console.error(`USAGE`);
+    console.error(``);
+    console.error(`    classify-one <project-name>`);
+    console.error(``);
+    console.error(`EXAMPLE`);
+    console.error(``);
+    console.error(`    classify-one caddy`);
+    console.error(``);
+    return;
+  }
+
+  let tsDate = new Date(0);
+  let meta = {
+    // version info
+    versions: [],
+    lexvers: [],
+    lexversMap: {},
+    // culled release assets
+    packages: [],
+    releasesByTriplet: {},
+    // target info
+    triplets: [],
+    oses: [],
+    arches: [],
+    libcs: [],
+    formats: [],
+    // TODO channels: [],
+  };
+
+  let installersDir = Path.join(__dirname, '..');
+  let Releases = require(`${installersDir}/${projName}/releases.js`);
+  if (!Releases.latest) {
+    Releases.latest = Releases;
+  }
+
+  let projInfo = await Releases.latest(request);
+
+  // let packages = await Builds.getPackage({ name: projName });
+  // console.log(packages);
+
+  let bc = {};
+  bc.ALL_TERMS = Triplet.TERMS_PRIMARY_MAP;
+  bc.orphanTerms = Object.assign({}, bc.ALL_TERMS);
+  bc.unknownTerms = {};
+  bc.usedTerms = {};
+  bc.formats = [];
+  bc._targetsByBuildIdCache = {};
+  bc._triplets = {};
+
+  let transformed = BuildsCacher.transformAndUpdate(
+    projName,
+    projInfo,
+    meta,
+    tsDate,
+    bc,
+  );
+
+  console.log(`[DEBUG] transformed`);
+  let sample = transformed.packages.slice(0, 20);
+  console.log('packages:', sample, ':packages');
+  console.log(
+    'releasesByTriplet:',
+    transformed.releasesByTriplet['linux-x86_64-none'][transformed.versions[0]],
+    ':releasesByTriplet',
+  );
+  console.log('versions:', transformed.versions, ':versions');
+  console.log('triplets:', transformed.triplets, ':triplets');
+  console.log('oses:', transformed.oses, ':oses');
+  console.log('arches:', transformed.arches, ':arches');
+  console.log('libcs:', transformed.libcs, ':libcs');
+  console.log('formats:', transformed.formats, ':formats');
+  console.log(Object.keys(transformed));
+}
+
+main().catch(function (err) {
+  console.error('Error:');
+  console.error(err);
+});

_webi/curl-pipe-bootstrap.tpl.sh

@@ -267,12 +267,23 @@ webi_bootstrap() { (
 fn_checksum() {
     a_filepath="${1}"
 
-    cmd_shasum='sha1sum'
-    if command -v shasum > /dev/null; then
-        cmd_shasum='shasum'
+    if command -v sha1sum > /dev/null; then
+        sha1sum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
+        return 0
     fi
 
-    $cmd_shasum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
+    if command -v shasum > /dev/null; then
+        shasum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
+        return 0
+    fi
+
+    if command -v sha1 > /dev/null; then
+        sha1 "${a_filepath}" | cut -d'=' -f2 | cut -c 2-9
+        return 0
+    fi
+
+    echo >&2 "    warn: no sha1 sum program"
+    date '+%F %H:%M'
 }
 
 ##############################################

_webi/installers.js

@@ -16,7 +16,7 @@ var BAD_SH_RE = /[<>'"`$\\]/;
 Installers.renderBash = async function (
   pkgdir,
   rel,
-  { baseurl, pkg, tag, ver, os = '', arch = '', libc = '', formats },
+  { baseurl, pkg, tag, ver, os = '', arch = '', libc = '', formats, latest },
 ) {
   if (!Array.isArray(formats)) {
     formats = [];
@@ -93,16 +93,17 @@ Installers.renderBash = async function (
     ['WEBI_GIT_TAG', rel.git_tag], // TODO replace with branch
     ['WEBI_LTS', rel.lts],
     ['WEBI_CHANNEL', rel.channel],
-    ['WEBI_EXT', rel.ext.replace(/tar.*/, 'tar')],
+    ['WEBI_EXT', rel.ext],
     ['WEBI_FORMATS', formats.join(',')],
     ['WEBI_PKG_URL', rel.download],
     ['WEBI_PKG_PATHNAME', pkgFile],
     ['WEBI_PKG_FILE', pkgFile], // TODO replace with pathname
     ['PKG_NAME', pkg],
-    ['PKG_OSES', rel.oses],
-    ['PKG_ARCHES', rel.arches],
-    ['PKG_LIBCS', rel.libcs],
-    ['PKG_FORMATS', (rel.formats || []).join(',')],
+    ['PKG_OSES', (rel.oses || []).join(' ')],
+    ['PKG_ARCHES', (rel.arches || []).join(' ')],
+    ['PKG_LIBCS', (rel.libcs || []).join(' ')],
+    ['PKG_FORMATS', (rel.formats || []).join(' ')],
+    ['PKG_LATEST', latest],
   ];
 
   for (let env of envReplacements) {

_webi/lint-builds.js

@@ -0,0 +1,338 @@
+#!/usr/bin/env node
+'use strict';
+
+let Fs = require('node:fs/promises');
+let Path = require('node:path');
+
+let BuildsCacher = require('./builds-cacher.js');
+let HostTargets = require('./build-classifier/host-targets.js');
+let Parallel = require('./parallel.js');
+
+var INSTALLERS_DIR = Path.join(__dirname, '..');
+var CACHE_DIR = Path.join(__dirname, '../_cache');
+
+let UserAgentsMap = require('./build-classifier/uas.json');
+let uas = Object.keys(UserAgentsMap);
+let uaTargetsMap = {};
+for (let ua of uas) {
+  let terms = ua.split(/[\s\/]+/g);
+  let target = {};
+  void HostTargets.termsToTarget(target, terms);
+  if (!target) {
+    continue;
+  }
+  if (target.errors.length) {
+    throw target.errors[0];
+  }
+  if (!target.os) {
+    // TODO make target null, or create error for this
+    console.warn(`no os for terms: ${terms}`);
+    //throw new Error(`terms: ${terms}`);
+    continue;
+  }
+  if (!target.arch) {
+    // TODO make target null, or create error for this
+    console.warn(`no arch for terms: ${terms}`);
+    //throw new Error(`terms: ${terms}`);
+    continue;
+  }
+  if (!target.libc) {
+    // TODO make target null, or create error for this
+    console.warn(`no libc for terms: ${terms}`);
+    //throw new Error(`terms: ${terms}`);
+    continue;
+  }
+  let triplet = `${target.os}-${target.arch}-${target.libc}`;
+  uaTargetsMap[triplet] = target;
+}
+let uaTargets = [];
+let triplets = Object.keys(uaTargetsMap);
+for (let triplet of triplets) {
+  let target = uaTargetsMap[triplet];
+  uaTargets.push(target);
+}
+
+function showDirs(dirs) {
+  {
+    let errors = Object.keys(dirs.errors);
+    console.error('');
+    console.error(`Errors: ${errors.length}`);
+    for (let name of errors) {
+      let err = dirs.errors[name];
+      console.error(`${name}/: ${err.message}`);
+    }
+  }
+
+  {
+    let hidden = Object.keys(dirs.hidden);
+    console.debug('');
+    console.debug(`Hidden: ${hidden.length}`);
+    for (let name of hidden) {
+      let kind = dirs.hidden[name];
+      if (kind === '!directory') {
+        console.debug(`    ${name}`);
+      } else {
+        console.debug(`    ${name}/`);
+      }
+    }
+  }
+
+  {
+    let alias = Object.keys(dirs.alias);
+    console.debug('');
+    console.debug(`Alias: ${alias.length}`);
+    for (let name of alias) {
+      let kind = dirs.alias[name];
+      if (kind === 'symlink') {
+        console.debug(`    ${name} => ...`);
+      } else {
+        console.debug(`    ${name}/`);
+      }
+    }
+  }
+
+  {
+    let invalids = Object.keys(dirs.invalid);
+    console.warn('');
+    console.warn(`Invalid: ${invalids.length}`);
+    for (let name of invalids) {
+      console.warn(`    ${name}/`);
+    }
+  }
+
+  {
+    let selfhosted = Object.keys(dirs.selfhosted);
+    console.info('');
+    console.info(`Self-Hosted: ${selfhosted.length}`);
+    for (let name of selfhosted) {
+      console.info(`    ${name}/`);
+    }
+  }
+
+  {
+    let valids = Object.keys(dirs.valid);
+    console.info('');
+    console.info(`Found: ${valids.length}`);
+    for (let name of valids) {
+      console.info(`    ${name}/`);
+    }
+  }
+}
+
+let bc = BuildsCacher.create({
+  caches: CACHE_DIR,
+  installers: INSTALLERS_DIR,
+});
+
+async function main() {
+  /* jshint maxcomplexity: 25 */
+  // TODO
+  //     node ./_webi/lint-builds.js caddy@beta 'x86_64/unknown Darwin libc'
+  //
+  //let [projName, userAgent] = process.argv.slice(2);
+  let projName = process.argv[2];
+  // create test case for zoxide, goreleaser, go, yq, caddy, rg
+
+  let dirs = await bc.getProjectsByType();
+  if (!projName) {
+    showDirs(dirs);
+    console.info('');
+  }
+
+  bc.freshenRandomPackage(600 * 1000);
+
+  let rows = [];
+  let triples = [];
+  let valids = Object.keys(dirs.valid);
+
+  if (projName) {
+    if (!valids.includes(projName)) {
+      throw new Error(`'${projName}' is not a valid installable project`);
+    }
+    valids = [projName];
+  }
+  //valids = ['atomicparsley', 'caddy', 'macos'];
+  //valids = ['atomicparsley'];
+
+  console.info('');
+  console.info(`Fetching project release assets`);
+  let parallel = 25;
+  let projects = [];
+  await Parallel.run(parallel, valids, getAll);
+  async function getAll(name, i) {
+    console.info(`    ${name}`);
+    let projInfo = await bc.getPackages({
+      //Releases: Releases,
+      name: name,
+      date: new Date(),
+    });
+    projects[i] = projInfo;
+  }
+
+  console.info(`Classifying build assets for...`);
+  for (let projInfo of projects) {
+    console.info(`    ${projInfo.name}`);
+
+    let nStr = projInfo.releases.length.toString();
+    let n = nStr.padStart(5, ' ');
+    let row = `##### ${n}\t${projInfo.name}\tv`;
+    rows.push(row);
+
+    // ignore known, non-package extensions
+    for (let build of projInfo.releases) {
+      let target = bc.classify(projInfo, build);
+      if (!target) {
+        // non-build file
+        continue;
+      }
+
+      if (target.error) {
+        let e = target.error;
+        if (e.code === 'E_BUILD_NO_PATTERN') {
+          console.warn(`>>> ${e.message} <<<`);
+          console.warn(projInfo);
+          console.warn(build);
+          console.warn(`^^^ ${e.message} ^^^`);
+        }
+        throw e;
+      }
+
+      if (target.unknownTerms?.length) {
+        let msg = `${projInfo.name}: unrecognized term(s) '${target.unknownTerms}' in '${build.download}'`;
+        let err = new Error(msg);
+        throw err;
+      }
+
+      triples.push(target.triplet);
+      // if (!build.version) {
+      //   throw new Error(`no version for ${pkg.name} ${build.name}`);
+      // }
+      // // For debug printing versions
+      // console.error(build.version);
+      rows.push(`${target.triplet}\t${projInfo.name}\t${build.version}`);
+    }
+  }
+
+  console.info(`Fetching builds for`);
+  for (let projInfo of projects) {
+    console.info('');
+    console.info('');
+    console.info(`    ${projInfo.name}`);
+
+    for (let target of uaTargets) {
+      let libc = target.libc || 'libc';
+      let hostTriplet = `${target.os}-${target.arch}-${libc}`;
+      console.info('');
+      console.info(`    target: ${hostTriplet}`);
+      let match = bc.findMatchingPackages(projInfo, target, {
+        ver: '',
+      });
+      if (!match) {
+        console.info(
+          `    project: ${projInfo.name}: missing build for os '${target.os}'`,
+        );
+        continue;
+      }
+
+      if (!match.releases) {
+        console.info(
+          `    project: ${projInfo.name}: missing build for os '${target.os}-${target.arch}-${libc}'`,
+        );
+      } else if (match.triplet === hostTriplet) {
+        let releaseNames = Object.keys(match.releases);
+        console.info(`    selected ${releaseNames.length}`);
+      } else {
+        let releaseNames = Object.keys(match.releases);
+        console.info(
+          `    selected ${releaseNames.length} (${match.triplet} fallback)`,
+        );
+      }
+    }
+  }
+
+  let tsv = rows.join('\n');
+  console.info('');
+  console.info('#rows', rows.length);
+  await Fs.writeFile('builds.tsv', tsv, 'utf8');
+
+  console.info('');
+  console.info('Triplets Detected:');
+  let triplets = Object.keys(bc._triplets);
+  if (triplets.length) {
+    triplets.sort();
+    console.info('   ', triplets.join('\n    '));
+  } else {
+    console.info('    (none)');
+  }
+
+  console.info('');
+  console.info('New / Unknown Terms:');
+  let unknowns = Object.keys(bc.unknownTerms);
+  if (unknowns.length) {
+    unknowns.sort();
+    console.warn('   ', unknowns.join('\n    '));
+  } else {
+    console.info('    (none)');
+  }
+
+  console.info('');
+  console.info('Unused Terms:');
+  let unuseds = Object.keys(bc.orphanTerms);
+  if (unuseds.length) {
+    unuseds.sort();
+    console.warn('   ', unuseds.join('\n    '));
+  } else {
+    console.info('    (none)');
+  }
+
+  console.info('');
+  console.info('Formats:');
+  if (bc.formats.length) {
+    let formats = bc.formats.slice();
+    formats.sort();
+    if (!formats[0]) {
+      formats[0] = '(bin)';
+    }
+    console.warn('   ', formats.join('\n    '));
+  } else {
+    console.info('    (none)');
+  }
+
+  // sort -u -k1 builds.tsv | rg -v '^#|^https?:' | rg -i arm
+  // cut -f1 builds.tsv | sort -u -k1 | rg -v '^#|^https?:' | rg -i arm
+}
+
+if (module === require.main) {
+  let times = [];
+  let now = Date.now();
+  main()
+    .then(async function () {
+      let then = Date.now();
+      let delta = then - now;
+      times.push(delta);
+      now = then;
+      await main();
+      then = Date.now();
+      delta = then - now;
+      times.push(delta);
+    })
+    .then(function () {
+      console.info('');
+      console.info('Run times');
+      for (let delta of times) {
+        let s = delta / 1000;
+        console.info(`    ${s}`);
+      }
+
+      function forceExit() {
+        console.warn(`warn: dangling event loop reference`);
+        process.exit(0);
+      }
+      let exitTimeout = setTimeout(forceExit, 250);
+      exitTimeout.unref();
+    })
+    .catch(function (err) {
+      console.error(err.stack || err);
+      process.exit(1);
+    });
+}

_webi/package-install.tpl.sh

@@ -127,6 +127,7 @@ __bootstrap_webi() {
             echo ""
             echo "    $(t_err "Error: no '${PKG_NAME:-"Unknown Package"}@${WEBI_TAG:-"Unknown Tag"}' release for '${WEBI_OS:-"Unknown OS"}' (${WEBI_LIBC:-"Unknown Libc"}) on '${WEBI_ARCH:-"Unknown CPU"}' as one of '${WEBI_FORMATS:-"Unknown File Type"}'")"
             echo ""
+            echo "        Latest Version: ${PKG_LATEST}"
             echo "        CPUs: $PKG_ARCHES"
             echo "        OSes: $PKG_OSES"
             echo "        libcs: $PKG_LIBCS"
@@ -196,20 +197,32 @@ __bootstrap_webi() {
         my_dl_rel="$(
             fn_sub_home "${WEBI_PKG_PATH}/${WEBI_PKG_FILE}"
         )"
-        if [ "tar" = "$WEBI_EXT" ]; then
+        if test "$WEBI_EXT" = "tar.zst"; then
+            echo "    Extracting $(t_path "${my_dl_rel}")"
+            unzstd -c --keep "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" | tar xf -
+        elif test "$WEBI_EXT" = "tar.xz"; then
+            echo "    Extracting $(t_path "${my_dl_rel}")"
+            unxz -c -k "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" | tar xf -
+        elif test "$WEBI_EXT" = "tar.gz"; then
+            echo "    Extracting $(t_path "${my_dl_rel}")"
+            tar xzf "${WEBI_PKG_PATH}/$WEBI_PKG_FILE"
+        elif test "$WEBI_EXT" = "tar.bz2"; then
+            echo "    Extracting $(t_path "${my_dl_rel}")"
+            tar xjf "${WEBI_PKG_PATH}/$WEBI_PKG_FILE"
+        elif test "$WEBI_EXT" = "tar"; then
             echo "    Extracting $(t_path "${my_dl_rel}")"
             tar xf "${WEBI_PKG_PATH}/$WEBI_PKG_FILE"
-        elif [ "zip" = "$WEBI_EXT" ]; then
+        elif test "$WEBI_EXT" = "zip" || test "$WEBI_EXT" = "app.zip"; then
             echo "    Extracting $(t_path "${my_dl_rel}")"
             unzip "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" > __unzip__.log
-        elif [ "exe" = "$WEBI_EXT" ]; then
+        elif test "$WEBI_EXT" = "exe"; then
             echo "    Moving $(t_path "${my_dl_rel}")"
             echo "      to $(t_path "$(fn_sub_home "$(pwd)")")"
             mv "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" .
-        elif [ "git" = "$WEBI_EXT" ]; then
+        elif test "$WEBI_EXT" = "git"; then
             echo "    Moving $(t_path "${my_dl_rel}")"
             mv "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" .
-        elif [ "xz" = "$WEBI_EXT" ]; then
+        elif test "$WEBI_EXT" = "xz"; then
             echo "    Inflating $(t_path "${my_dl_rel}")"
             unxz -c "${WEBI_PKG_PATH}/$WEBI_PKG_FILE" > "$(basename "$WEBI_PKG_FILE")"
         else
@@ -222,8 +235,6 @@ __bootstrap_webi() {
     webi_path_add() {
         my_path="${1}"
 
-        fn_envman_init
-
         # \v was chosen as it is extremely unlikely for a filename
         # \1 could be an even better choice, but needs more testing.
         # (currently tested working on: linux & mac)
@@ -386,7 +397,6 @@ __bootstrap_webi() {
     export _webi_tmp="${_webi_tmp:-"$HOME/.local/opt/webi-tmp.d"}"
 
     mkdir -p "${WEBI_PKG_PATH}"
-    mkdir -p "$HOME/.local/bin"
     mkdir -p "$HOME/.local/opt"
 
     if test -e ~/.local/bin; then
@@ -395,6 +405,7 @@ __bootstrap_webi() {
         echo "    Creating$(t_path ' ~/.local/bin')"
         mkdir -p "$HOME/.local/bin"
     fi
+    fn_envman_init
 
     ##
     ##
@@ -775,12 +786,23 @@ webi_upgrade() { (
 fn_checksum() {
     a_filepath="${1}"
 
-    cmd_shasum='sha1sum'
-    if command -v shasum > /dev/null; then
-        cmd_shasum='shasum'
+    if command -v sha1sum > /dev/null; then
+        sha1sum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
+        return 0
     fi
 
-    $cmd_shasum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
+    if command -v shasum > /dev/null; then
+        shasum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
+        return 0
+    fi
+
+    if command -v sha1 > /dev/null; then
+        sha1 "${a_filepath}" | cut -d'=' -f2 | cut -c 2-9
+        return 0
+    fi
+
+    echo >&2 "    warn: no sha1 sum program"
+    date '+%F %H:%M'
 }
 
 ##############################################

_webi/parallel.js

@@ -0,0 +1,44 @@
+'use strict';
+
+var Parallel = module.exports;
+Parallel.run = async function (limit, arr, fn) {
+  let index = 0;
+  let actives = [];
+  let results = [];
+  limit = Math.min(limit, arr.length);
+
+  function launch() {
+    let _index = index;
+    let p = fn(arr[_index], _index, arr);
+
+    // some tasks may be synchronous
+    // so we must push before removing
+    actives.push(p);
+
+    p.then(function _resolve(result) {
+      let i = actives.indexOf(p);
+      actives.splice(i, 1);
+      results[_index] = result;
+    });
+
+    index += 1;
+  }
+
+  // start tasks in parallel, up to limit
+  for (; actives.length < limit; ) {
+    launch();
+  }
+
+  // keep the task queue full
+  for (; index < arr.length; ) {
+    // wait for one task to complete
+    await Promise.race(actives);
+    // add one task again
+    launch();
+  }
+
+  // wait for all remaining tasks
+  await Promise.all(actives);
+
+  return results;
+};

_webi/serve-installer.js

@@ -2,17 +2,14 @@
 
 var InstallerServer = module.exports;
 
-var Fs = require('fs/promises');
-var path = require('path');
+let Fs = require('fs/promises');
+let Path = require('path');
 
-var uaDetect = require('./ua-detect.js');
-var Projects = require('./projects.js');
-var Installers = require('./installers.js');
+let HostTargets = require('./build-classifier/host-targets.js');
+let Builds = require('./builds.js');
+let Installers = require('./installers.js');
 
-// handlers caching and transformation, probably should be broken down
-var Releases = require('./transform-releases.js');
-
-InstallerServer.INSTALLERS_DIR = path.join(__dirname, '..');
+InstallerServer.INSTALLERS_DIR = Path.join(__dirname, '..');
 InstallerServer.serveInstaller = async function (
   baseurl,
   ua,
@@ -22,113 +19,194 @@ InstallerServer.serveInstaller = async function (
   formats,
   libc,
 ) {
-  let [rel, opts] = await InstallerServer.helper({
-    ua,
-    pkg,
+  let unameAgent = ua;
+  let projectName = pkg;
+  let [rel, tmplParams] = await InstallerServer.helper({
+    unameAgent,
+    projectName,
     tag,
     formats,
     libc,
   });
-  Object.assign(opts, {
+  Object.assign(tmplParams, {
     baseurl,
   });
 
-  var pkgdir = path.join(InstallerServer.INSTALLERS_DIR, pkg);
+  var pkgdir = Path.join(InstallerServer.INSTALLERS_DIR, projectName);
   if ('ps1' === ext) {
-    return Installers.renderPowerShell(pkgdir, rel, opts);
+    return Installers.renderPowerShell(pkgdir, rel, tmplParams);
   }
-  return Installers.renderBash(pkgdir, rel, opts);
+  return Installers.renderBash(pkgdir, rel, tmplParams);
 };
-InstallerServer.helper = async function ({ ua, pkg, tag, formats, libc }) {
-  // TODO put some of this in a middleware? or common function?
 
-  // TODO maybe move package/version/lts/channel detection into getReleases
-  var ver = tag.replace(/^v/, '');
-  var lts;
-  var channel;
+// TODO put some of this in a middleware? or common function?
+// TODO maybe move package/version/lts/channel detection into getReleases
+InstallerServer.helper = async function ({
+  unameAgent,
+  projectName,
+  tag,
+  formats,
+  libc,
+}) {
+  console.log(`dbg: Installer User-Agent: ${unameAgent}`);
 
-  switch (ver) {
-    case 'latest':
-      ver = '';
-      channel = 'stable';
-      break;
-    case 'lts':
-      lts = true;
-      channel = 'stable';
-      ver = '';
-      break;
-    case 'stable':
-      channel = 'stable';
-      ver = '';
-      break;
-    case 'beta':
-      channel = 'beta';
-      ver = '';
-      break;
-    case 'dev':
-      channel = 'dev';
-      ver = '';
-      break;
+  let releaseTarget = toReleaseTarget(tag);
+  let hostFormats = formats;
+  let terms = unameAgent.split(/[\s\/]+/g);
+  let hostTarget = {};
+  try {
+    void HostTargets.termsToTarget(hostTarget, terms);
+  } catch (e) {
+    // if we can't guarantee the results...
+    // "in the face of ambiguity, refuse the temptation to guess"
+    throw e;
+  }
+  console.log(`dbg: Installer Host Target:`);
+  console.log(hostTarget);
+
+  if (!hostTarget.os) {
+    throw new Error(`OS could not be identified by User-Agent '${unameAgent}'`);
   }
 
-  var myOs = uaDetect.os(ua);
-  var myArch = uaDetect.arch(ua);
-  var myLibc;
-  if (libc) {
-    myLibc = uaDetect.libc(libc);
+  console.log(`dbg: Get Project Installer Type for '${projectName}':`);
+  let proj = await Builds.getProjectType(projectName);
+  console.log(proj);
+
+  let validTypes = ['alias', 'selfhosted', 'valid'];
+  if (!validTypes.includes(proj.type)) {
+    let msg = `'${projectName}' doesn't have an installer: '${proj.type}': '${proj.detail}'`;
+    let err = new Error(msg);
+    err.code = 'ENOENT';
+    throw err;
   }
-  if (!myLibc) {
-    myLibc = uaDetect.libc(ua);
-  }
-  if (!myLibc) {
-    myLibc = 'libc';
+  if (proj.type === 'alias') {
+    projectName = proj.detail;
   }
 
-  let cfg = await Projects.get(pkg);
-  let releaseQuery = {
-    pkg: cfg.alias || pkg,
-    ver,
-    os: myOs,
-    arch: myArch,
-    libc: myLibc,
-    lts,
-    channel,
-    // TODO use formats for sorting, not exclusion
-    // (it's better to install xz or report an error to install zip)
-    formats,
+  let tmplParams = {
+    pkg: projectName,
+    tag: tag,
+    os: hostTarget.os,
+    arch: hostTarget.arch,
+    libc: hostTarget.libc,
+    formats: hostFormats,
     limit: 1,
   };
+  Object.assign(tmplParams, releaseTarget);
+  console.log('tmplParams', tmplParams);
 
-  let rels = await Releases.getReleases(releaseQuery);
-
-  var rel = rels.releases[0];
-  var opts = {
-    pkg: cfg.alias || pkg,
-    ver,
-    tag,
-    os: myOs,
-    arch: myArch,
-    libc: myLibc,
-    lts,
-    channel,
-    formats,
-    limit: 1,
+  let errPackage = {
+    name: 'doesntexist.ext',
+    version: '0.0.0',
+    lts: '-',
+    channel: 'error',
+    date: '1970-01-01',
+    os: hostTarget.os || '-',
+    arch: hostTarget.arch || '-',
+    libc: hostTarget.libc || '-',
+    ext: 'err',
+    download: 'https://example.com/doesntexist.ext',
+    comment:
+      'No matches found. Could be bad or missing version info' +
+      ',' +
+      "Check query parameters. Should be something like '/api/releases/{package}@{version}.tab?os={macos|linux|windows|-}&arch={amd64|x86|aarch64|arm64|armv7l|-}&libc={musl|gnu|msvc|libc|static}&limit=10'",
   };
-  rel = Object.assign(
-    {
-      oses: rels.oses,
-      arches: rels.arches,
-      libcs: rels.libcs,
-      formats: rels.formats,
-    },
-    rel,
+
+  if (proj.type === 'selfhosted') {
+    return [errPackage, tmplParams];
+  }
+
+  let projInfo = await Builds.getPackage({
+    name: projectName,
+    date: new Date(),
+  });
+  let latest = projInfo.versions[0];
+  Object.assign(tmplParams, { latest });
+  //console.log('projInfo', projInfo);
+
+  let buildTargetInfo = {
+    triplets: projInfo.triplets,
+    oses: projInfo.oses,
+    arches: projInfo.arches,
+    libcs: projInfo.libcs,
+    formats: projInfo.formats,
+  };
+
+  let hasOs = projInfo.oses.includes(hostTarget.os);
+  if (!hasOs) {
+    hasOs = projInfo.oses.includes('ANYOS');
+  }
+  if (!hasOs) {
+    let pkg1 = Object.assign(buildTargetInfo, errPackage);
+    return [pkg1, tmplParams];
+  }
+
+  let targetRelease = Builds.findMatchingPackages(
+    projInfo,
+    hostTarget,
+    releaseTarget,
   );
+  // { triplet: `${os}-${arch}-${libc}`, packages: targetPackages
+  // , latest: projInfo.versions[0], versions: matchInfo
+  // }
 
-  return [rel, opts];
+  if (!targetRelease?.packages) {
+    let pkg1 = Object.assign(buildTargetInfo, errPackage);
+    return [pkg1, tmplParams];
+  }
+
+  let buildPkg = Builds.selectPackage(targetRelease.packages, hostFormats);
+  let ext = buildPkg.ext || '.exe';
+  if (ext.startsWith('.')) {
+    ext = ext.slice(1);
+  }
+
+  let version = targetRelease.version;
+  if (version.startsWith('v')) {
+    version = version.slice(1);
+  }
+
+  buildPkg = Object.assign(buildTargetInfo, buildPkg, { ext, version });
+  console.log('dbg: buildPkg', buildPkg);
+  console.log('dbg: tmplParams', tmplParams);
+  return [buildPkg, tmplParams];
 };
 
-var CURL_PIPE_PS1_BOOT = path.join(__dirname, 'curl-pipe-bootstrap.tpl.ps1');
-var CURL_PIPE_SH_BOOT = path.join(__dirname, 'curl-pipe-bootstrap.tpl.sh');
+let channelNames = [
+  'stable',
+  // 'hotfix',
+  'latest',
+  'rc',
+  'preview',
+  'pre',
+  'dev',
+  'beta',
+  'alpha',
+];
+
+function toReleaseTarget(tag) {
+  tag = tag.replace(/^v/, '');
+
+  let releaseTarget = {
+    channel: '',
+    lts: false,
+    version: '',
+  };
+
+  if (tag === 'lts') {
+    releaseTarget.lts = true;
+    releaseTarget.channel = 'stable';
+  } else if (channelNames.includes(tag)) {
+    releaseTarget.channel = tag;
+  } else {
+    releaseTarget.version = tag;
+  }
+
+  return releaseTarget;
+}
+
+var CURL_PIPE_PS1_BOOT = Path.join(__dirname, 'curl-pipe-bootstrap.tpl.ps1');
+var CURL_PIPE_SH_BOOT = Path.join(__dirname, 'curl-pipe-bootstrap.tpl.sh');
 var BAD_SH_RE = /[<>'"`$\\]/;
 
 InstallerServer.getPosixCurlPipeBootstrap = async function ({
@@ -150,7 +228,6 @@ InstallerServer.getPosixCurlPipeBootstrap = async function ({
     let name = env[0];
     let value = env[1];
 
-    // TODO create REs once, in higher scope
     let envRe = new RegExp(
       `^[ \\t]*#?[ \\t]*(export[ \\t])?[ \\t]*(${name})=.*`,
       'm',
@@ -198,9 +275,6 @@ InstallerServer.getPwshCurlPipeBootstrap = async function ({
     let tplRe = new RegExp(`{{ (${name}) }}`, 'g');
     bootTxt = bootTxt.replace(tplRe, `${value}`);
 
-    // let envRe = new RegExp(`^[ \\t]*#?[ \\t]*($$${name})[ \\t]*=.*`, 'im');
-    // bootTxt = bootTxt.replace(envRe, `$$${name} = '${value}'`);
-
     let setRe = new RegExp(
       `(#[ \\t]*)?(\\$${name})[ \\t]*=[ \\t]['"].*['"][ \\t]`,
       'im',

_webi/test.js

@@ -32,7 +32,7 @@ if (/\b-?-h(elp)?\b/.test(process.argv.join(' '))) {
 var os = require('os');
 var fs = require('fs');
 var path = require('path');
-var Releases = require('./transform-releases.js');
+var Builds = require('./builds.js');
 var Installers = require('./installers.js');
 var ServeInstaller = require('./serve-installer.js');
 
@@ -65,7 +65,8 @@ console.info('Has the necessary files?');
   });
 
 console.info('');
-Releases.get(path.join(process.cwd(), pkgdir)).then(async function (all) {
+let projName = pkgdir.split('/').filter(Boolean).pop();
+Builds.getPackage({ name: projName }).then(async function (/*projInfo*/) {
   var pkgname = path.basename(pkgdir.replace(/\/$/, ''));
   var nodeOs = os.platform();
   var nodeOsRelease = os.release();
@@ -82,8 +83,8 @@ Releases.get(path.join(process.cwd(), pkgdir)).then(async function (all) {
   var formats = ['exe', 'xz', 'tar', 'zip', 'git'];
 
   let [rel, opts] = await ServeInstaller.helper({
-    ua: `${nodeOs}/${nodeOsRelease} ${nodeArch}/unknown ${nodeLibc}`,
-    pkg: pkgname,
+    unameAgent: `${nodeOs}/${nodeOsRelease} ${nodeArch}/unknown ${nodeLibc}`,
+    projectName: pkgname,
     tag: pkgtag || '',
     formats: formats,
     libc: nodeLibc,

brew/brew-update-service-install

@@ -6,15 +6,23 @@ main() { (
     sed '1,/^#~\/.local\/bin\/brew-updater/d' "${0}" > ~/.local/bin/brew-update-hourly
     chmod a+x ~/.local/bin/brew-update-hourly
 
+    echo "Checking for serviceman..."
+    if ! command -v serviceman > /dev/null; then
+        "$HOME/.local/bin/webi" serviceman
+        export PATH="$HOME/.local/bin:$PATH"
+        serviceman --version
+    fi
+
     env PATH="$PATH" serviceman add --user \
         --workdir ~/.local/opt/brew/ \
         --name sh.brew.updater -- \
         ~/.local/bin/brew-update-hourly
 ); }
 
-if main; then
-    exit 0
+if ! main; then
+    exit 1
 fi
+exit 0
 
 #~/.local/bin/brew-updater
 #!/bin/sh

cilium/README.md

@@ -0,0 +1,47 @@
+---
+title: cilium
+homepage: https://github.com/cilium/cilium-cli
+tagline: |
+  cilium: manage & troubleshoot Kubernetes clusters running Cilium
+---
+
+To update or switch versions, run `webi cilium@stable` (or `@v2`, `@beta`,etc).
+
+### Files
+
+These are the files / directories that are created and/or modified with this
+install:
+
+```text
+~/.config/envman/PATH.env
+~/.local/bin/cilium
+~/.local/opt/cilium/
+```
+
+## Cheat Sheet
+
+> Cilium is an open source, cloud native solution for providing, securing, and
+> observing network connectivity between workloads, fueled by the revolutionary
+> Kernel technology eBPF.
+
+Quick Start User Guide:
+
+<https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/#k8s-install-quick>
+
+To install the default version of the Cilium image:
+
+```sh
+cilium install
+```
+
+To upgrade to a specific version of the Cilium image:
+
+```sh
+cilium upgrade --version v1.15.3
+```
+
+To check the status of the current Cilium deployment:
+
+```sh
+cilium status
+```

cilium/install.ps1

@@ -0,0 +1,45 @@
+#!/usr/bin/env pwsh
+
+##################
+# Install cilium #
+##################
+
+$pkg_cmd_name = "cilium"
+
+$pkg_dst_cmd = "$Env:USERPROFILE\.local\bin\cilium.exe"
+$pkg_dst = "$pkg_dst_cmd"
+
+$pkg_src_cmd = "$Env:USERPROFILE\.local\opt\cilium-v$Env:WEBI_VERSION\bin\cilium.exe"
+$pkg_src_bin = "$Env:USERPROFILE\.local\opt\cilium-v$Env:WEBI_VERSION\bin"
+$pkg_src_dir = "$Env:USERPROFILE\.local\opt\cilium-v$Env:WEBI_VERSION"
+$pkg_src = "$pkg_src_cmd"
+
+New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
+$pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
+
+IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) {
+    Write-Output "Downloading cilium from $Env:WEBI_PKG_URL to $pkg_download"
+    & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
+    & Move-Item "$pkg_download.part" "$pkg_download"
+}
+
+IF (!(Test-Path -Path "$pkg_src_cmd")) {
+    Write-Output "Installing cilium"
+    Push-Location .local\tmp
+
+    Remove-Item -Path ".\cilium-v*" -Recurse -ErrorAction Ignore
+    Remove-Item -Path ".\cilium.exe" -Recurse -ErrorAction Ignore
+
+    Write-Output "Unpacking $pkg_download"
+    & tar xf "$pkg_download"
+
+    Write-Output "Install Location: $pkg_src_cmd"
+    New-Item "$pkg_src_bin" -ItemType Directory -Force
+    Move-Item -Path ".\cilium-*\cilium.exe" -Destination "$pkg_src_bin"
+
+    Pop-Location
+}
+
+Write-Output "Copying into '$pkg_dst_cmd' from '$pkg_src_cmd'"
+Remove-Item -Path "$pkg_dst_cmd" -Recurse -ErrorAction Ignore
+Copy-Item -Path "$pkg_src" -Destination "$pkg_dst" -Recurse

cilium/install.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+__init_cilium() {
+    set -e
+    set -u
+
+    ##################
+    # Install cilium #
+    ##################
+
+    pkg_cmd_name="cilium"
+
+    pkg_dst_cmd="$HOME/.local/bin/cilium"
+    pkg_dst="$pkg_dst_cmd"
+
+    pkg_src_cmd="$HOME/.local/opt/cilium-v$WEBI_VERSION/bin/cilium"
+    pkg_src_dir="$HOME/.local/opt/cilium-v$WEBI_VERSION"
+    pkg_src="$pkg_src_cmd"
+
+    WEBI_SINGLE=true
+
+    # pkg_install must be defined by every package
+    pkg_install() {
+        # ~/.local/opt/cilium-v0.16.16/bin
+        mkdir -p "$(dirname "${pkg_src_cmd}")"
+
+        # mv ./hugo ~/.local/opt/cilium-v0.16.16/bin/
+        mv ./cilium "${pkg_src_cmd}"
+    }
+
+    pkg_get_current_version() {
+        cilium version 2> /dev/null |
+            head -n 1 |
+            cut -d ' ' -f 2
+    }
+
+}
+
+__init_cilium

cilium/releases.js

@@ -0,0 +1,22 @@
+'use strict';
+
+var github = require('../_common/github.js');
+var owner = 'cilium';
+var repo = 'cilium-cli';
+
+module.exports = async function (request) {
+  let all = await github(request, owner, repo);
+  return all;
+};
+
+if (module === require.main) {
+  (async function () {
+    let request = require('@root/request');
+    let normalize = require('../_webi/normalize.js');
+    let all = await module.exports(request);
+    all = normalize(all);
+    // just select the first 5 for demonstration
+    all.releases = all.releases.slice(0, 5);
+    console.info(JSON.stringify(all, null, 2));
+  })();
+}

commandlinetools/README.md

@@ -0,0 +1,55 @@
+---
+title: XCode Command Line Tools
+homepage: https://webinstall.dev/commandlinetools
+tagline: |
+  The XCode Command Line Tools include git, swift, make, clang, and other developer tools
+---
+
+## Cheat Sheet
+
+> The developer tools provided by Apple for macOS.
+
+- git
+- swift
+- make
+- clang
+- etc
+
+This is also part of [webi-essentials](../webi-essentials/).
+
+## Table of Contents
+
+- Files
+- Manual Install
+  - macOS
+  - Linux
+  - Alpine
+  - Windows
+
+### Files
+
+These are the files / directories that are created and/or modified with this
+install:
+
+```sh
+/Library/Developer/CommandLineTools/
+```
+
+### How to Install Manually
+
+It's very easy to start the installer:
+
+```sh
+xcode-select --install
+```
+
+The trick is to also have a mechanism to know when it has finished:
+
+```sh
+while ! test -x /Library/Developer/CommandLineTools/usr/bin/git ||
+    ! test -x /Library/Developer/CommandLineTools/usr/bin/make; do
+    sleep 0.25
+done
+
+echo "Command Line Tools Installed"
+```

commandlinetools/install.sh

@@ -0,0 +1,62 @@
+#!/bin/sh
+set -e
+set -u
+
+fn_install_xcode_commandlinetools() { (
+    b_os="$(uname -s)"
+    if test "${b_os}" != 'Darwin'; then
+        echo >&2 'XCode Command Line Tools are for macOS only'
+        return 1
+    fi
+
+    # streamline the output to be pretty
+    fn_check_pkg '/Library/Developer/CommandLineTools/usr/bin/clang' 'clang'
+    fn_check_pkg '/Library/Developer/CommandLineTools/usr/bin/git' 'git'
+    fn_check_pkg '/Library/Developer/CommandLineTools/usr/bin/make' 'make'
+    echo >&2 ""
+
+    # git
+    if xcode-select -p > /dev/null 2> /dev/null; then
+        echo ""
+        return 0
+    fi
+
+    cmd_xcode_cli_tools_install="xcode-select --install"
+    echo "    Running $(t_cmd "${cmd_xcode_cli_tools_install}")"
+    $cmd_xcode_cli_tools_install 2> /dev/null
+    echo ""
+    echo ">>> $(t_attn 'ACTION REQUIRED') <<<"
+    echo ""
+    echo "        $(t_attn "Click") '$(t_bold 'Install')' $(t_attn "in the pop-up")"
+    echo "        (it may appear $(t_em 'under') this window)"
+    echo ""
+    echo "^^^ $(t_attn 'ACTION REQUIRED') ^^^"
+    echo ""
+    printf "    waiting %s to finish installing Command Line Developer Tools ..." "$(t_em 'for you')"
+    while ! test -x /Library/Developer/CommandLineTools/usr/bin/git ||
+        ! test -x /Library/Developer/CommandLineTools/usr/bin/make; do
+        sleep 0.25
+    done
+    echo " $(t_info 'OK')"
+    echo "    Installed to $(t_path '/Library/Developer/CommandLineTools/')"
+    sleep 1
+); }
+
+fn_check_pkg() { (
+    a_pkg="${1}"
+    a_pkgname="${2:-$a_pkg}"
+
+    printf >&2 '    %s %s %s' \
+        "$(t_dim "Checking for")" \
+        "$(t_pkg "${a_pkgname}")" \
+        "$(t_dim "...")"
+
+    if command -v "${a_pkg}" > /dev/null; then
+        echo >&2 " $(t_dim 'OK')"
+        return 0
+    fi
+
+    echo >&2 ' missing'
+); }
+
+fn_install_xcode_commandlinetools

dashcore-utils/dash.example.conf

@@ -1,7 +1,14 @@
+# for exposing RPCs, building APIs
 txindex=1
 addressindex=1
 timestampindex=1
 spentindex=1
+# listen as server (explicit default)
+listen=1
+# because its already run as a service (systemd, openrc)
+daemon=0
+# for evonodes
+#server=1
 
 [main]
 rpcuser=RPCUSER_MAIN

dashcore-utils/dashd-hd-service-install

@@ -17,7 +17,7 @@ fn_usage() { (
 ); }
 
 fn_datadir_help() { (
-    my_vol="${1:-}"
+    my_vol="${1}"
     my_user="$(
         id -n -u
     )"
@@ -25,6 +25,7 @@ fn_datadir_help() { (
         id -n -g
     )"
 
+    my_mount="$(dirname "${my_vol}")"
     echo >&2 ""
     echo >&2 "ERROR"
     echo >&2 "        '${my_vol}' is not writable"
@@ -32,8 +33,8 @@ fn_datadir_help() { (
     echo >&2 "SOLUTION"
     echo >&2 "        1. Mount a large (50gb+) volume"
     echo >&2 ""
-    echo >&2 "              sudo mkdir -p /mnt/EXAMPLE"
-    echo >&2 "              sudo mount /dev/sdx1 /mnt/EXAMPLE"
+    echo >&2 "              sudo mkdir -p ${my_mount}"
+    echo >&2 "              sudo mount /dev/sdx1 ${my_mount}"
     echo >&2 ""
     echo >&2 "        2. Create a 'dashcore' inside of it"
     echo >&2 ""
@@ -111,7 +112,7 @@ fn_srv_install() { (
         echo "Installing 'serviceman'..."
         echo ""
         {
-            curl -fsSL "${WEBI_HOST}/serviceman" | sh
+            "$HOME/.local/bin/webi" serviceman
         } > /dev/null
 
         # shellcheck disable=SC1090

dashcore-utils/install.sh

@@ -5,32 +5,38 @@ set -u
 __install_dashcore_utils() {
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dash-qt-hd" \
-        "$HOME/.local/bin/dash-qt-hd"
+        "$HOME/.local/bin/dash-qt-hd" \
+        "dash-qt-hd"
     chmod a+x "$HOME/.local/bin/dash-qt-hd"
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dash-qt-testnet" \
-        "$HOME/.local/bin/dash-qt-testnet"
+        "$HOME/.local/bin/dash-qt-testnet" \
+        "dash-qt-testnet"
     chmod a+x "$HOME/.local/bin/dash-qt-testnet"
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dashd-hd" \
-        "$HOME/.local/bin/dashd-hd"
+        "$HOME/.local/bin/dashd-hd" \
+        "dashd-hd"
     chmod a+x "$HOME/.local/bin/dashd-hd"
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dashd-testnet" \
-        "$HOME/.local/bin/dashd-testnet"
+        "$HOME/.local/bin/dashd-testnet" \
+        "dashd-testnet"
     chmod a+x "$HOME/.local/bin/dashd-testnet"
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dashd-hd-service-install" \
-        "$HOME/.local/bin/dashd-hd-service-install"
+        "$HOME/.local/bin/dashd-hd-service-install" \
+        "dashd-hd-service-install"
     chmod a+x "$HOME/.local/bin/dashd-hd-service-install"
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dashd-testnet-service-install" \
-        "$HOME/.local/bin/dashd-testnet-service-install"
+        "$HOME/.local/bin/dashd-testnet-service-install" \
+        "dashd-testnet-service-install"
     chmod a+x "$HOME/.local/bin/dashd-testnet-service-install"
 
     if ! test -e "${HOME}/.dashcore"; then
@@ -44,7 +50,8 @@ __install_dashcore_utils() {
 
     webi_download \
         "$WEBI_HOST/packages/dashcore-utils/dash.example.conf" \
-        "$HOME/.dashcore/dash.example.conf"
+        "$HOME/.dashcore/dash.example.conf" \
+        "dash.example.conf"
 
     if ! grep -q rpcuser ~/.dashcore/dash.conf; then
         cat ~/.dashcore/dash.example.conf >> ~/.dashcore/dash.conf

dashd/README.md

@@ -24,7 +24,7 @@ install:
 ~/.config/envman/PATH.env
 ~/.dashcore/dash.conf
 ~/.dashcore/wallets/
-~/.local/bin/bin/dashd-hd-service-install
+~/.local/bin/dashd-hd-service-install
 ~/.local/opt/dashcore/
 /mnt/<BLK_VOL>/dashcore/
 ```

flutter/releases.js

@@ -66,10 +66,10 @@ module.exports = async function (request) {
       json: true,
     });
 
-    let body = resp.body;
-    all.download = `${body.base_url}/{{ download }}`;
+    let osBaseUrl = resp.body.base_url;
+    let osReleases = resp.body.releases;
 
-    for (let asset of body.releases) {
+    for (let asset of osReleases) {
       if (!channelMap[asset.channel]) {
         channelMap[asset.channel] = true;
       }
@@ -81,7 +81,8 @@ module.exports = async function (request) {
         channel: asset.channel,
         date: asset.release_date.replace(/T.*/, ''),
         //sha256: asset.sha256,
-        download: asset.archive,
+        download: `${osBaseUrl}/${asset.archive}`,
+        _filename: asset.archive,
       });
     }
   }

go-essentials/install.sh

@@ -3,7 +3,7 @@ set -e
 set -u
 
 __run_go_essentials() {
-    if ! command -v go 2> /dev/null; then
+    if ! command -v go > /dev/null; then
         "$HOME/.local/bin/webi" "go@${WEBI_TAG}"
     fi
 

go/releases.js

@@ -44,7 +44,7 @@ function getAllReleases(request) {
     var goReleases = resp.body;
     var all = {
       releases: [],
-      download: 'https://dl.google.com/go/{{ download }}',
+      download: '',
     };
 
     goReleases.forEach((release) => {
@@ -77,7 +77,7 @@ function getAllReleases(request) {
           arch: arch,
           ext: '', // let normalize run the split/test/join
           hash: '-', // not ready to standardize this yet
-          download: filename,
+          download: `https://dl.google.com/go/${filename}`,
         });
       });
     });

julia/README.md

@@ -0,0 +1,128 @@
+---
+title: Julia
+homepage: https://julialang.org/
+tagline: |
+  Julia: A Language for Data Science, Visualization, and Machine Learning
+---
+
+To update or switch versions, run `webi julia@stable` (or `@v1.10`, `@beta`,
+etc).
+
+## Cheat Sheet
+
+> Julia is a programming language for Data Science - a far better alternative to
+> (or plugin for) Python when performance matters.
+
+## Table of Contents
+
+- Files
+- Hello World
+- Compile for Distribution
+- Vim Plugin
+
+### Files
+
+These are the files / directories that are created and/or modified with this
+install:
+
+```text
+~/.config/envman/PATH.env
+~/.local/opt/julia/
+```
+
+### How to create a Hello World with Julia
+
+Try out the REPL by coping and pasting the code below (it'll print some nice
+ASCII art to the console):
+
+```sh
+julia -i
+```
+
+```julia
+function mandelbrot(a)
+    z = 0
+    for i=1:50
+        z = z^2 + a
+    end
+    return z
+end
+
+for y=1.0:-0.05:-1.0
+    for x=-2.0:0.0315:0.5
+        abs(mandelbrot(complex(x, y))) < 2 ? print("*") : print(" ")
+    end
+    println()
+end
+```
+
+Or write the program to `./mandelbrot.jl` and run it:
+
+```sh
+julia ./mandelbrot.jl
+```
+
+## How to Build a Distributable Binary with Julia
+
+Here's an example project that you can build:
+
+1. Clone and enter the example project
+   ```sh
+   git clone --depth=1 https://github.com/JuliaLang/PackageCompiler.jl ./PackageCompiler
+   pushd ./PackageCompiler/examples
+   ```
+2. Run Julia in project mode
+   ```sh
+   julia -q --project
+   ```
+3. Compile the binary
+   ```julia
+   using PackageCompiler
+   create_app("MyApp", "MyAppCompiled")
+   exit()
+   ```
+4. Test & Enjoy
+   ```sh
+   ./MyAppCompiled/bin/MyApp foo bar --julia-args -t4
+   ```
+
+See also:
+
+- <https://docs.juliahub.com/PackageCompiler/MMV8C/2.1.16/apps.html>
+- <https://github.com/JuliaLang/PackageCompiler.jl/tree/master/examples/MyApp>
+
+## How to Install the Language Server (for VSCode, Vim, etc)
+
+Open the Julia REPL and add `LanguageServer`:
+
+```sh
+julia -i
+```
+
+```jl
+using Pkg
+Pkg.add("LanguageServer")
+Pkg.add("SymbolServer")
+```
+
+You'll need to reinstall if you switch environments.
+
+## How to Install the Julia Vim Plugin
+
+`julia-vim` adds support for:
+
+- Latex-to-Unicode
+- jumping from between start and end of blocks with `%` \
+  (and other block jump sequences)
+
+```sh
+mkdir -p ~/.vim/pack/plugins/start
+git clone https://github.com/JuliaEditorSupport/julia-vim.git \
+    ~/.vim/pack/plugins/start/julia-vim
+```
+
+Usage Examples:
+
+- `\alpha<tab>` will produce `α` \
+  (meaning typing `\alpha` and hitting the `tab` key)
+- Hitting `%` on an `end` will take you to the `function` or `for`, etc

julia/install.ps1

@@ -0,0 +1,56 @@
+#!/usr/bin/env pwsh
+
+#################
+# Install julia #
+#################
+
+# Every package should define these variables
+$pkg_cmd_name = "julia"
+
+$pkg_dst_cmd = "$Env:USERPROFILE\.local\opt\julia\bin\julia.exe"
+$pkg_dst_bin = "$Env:USERPROFILE\.local\opt\julia\bin"
+$pkg_dst_dir = "$Env:USERPROFILE\.local\opt\julia"
+$pkg_dst = "$pkg_dst_dir"
+
+$pkg_src_cmd = "$Env:USERPROFILE\.local\opt\julia-v$Env:WEBI_VERSION\bin\julia.exe"
+$pkg_src_dir = "$Env:USERPROFILE\.local\opt\julia-v$Env:WEBI_VERSION"
+$pkg_src = "$pkg_src_dir"
+
+New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null
+$pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE"
+
+# Fetch archive
+IF (!(Test-Path -Path "$pkg_download")) {
+    Write-Output "Downloading julia from $Env:WEBI_PKG_URL to $pkg_download"
+    & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part"
+    & Move-Item "$pkg_download.part" "$pkg_download"
+}
+
+IF (!(Test-Path -Path "$pkg_src_dir")) {
+    Write-Output "Installing julia"
+
+    # TODO: create package-specific temp directory
+    # Enter tmp
+    Push-Location .local\tmp
+
+    # Remove any leftover tmp cruft
+    Remove-Item -Path ".\julia*" -Recurse -ErrorAction Ignore
+
+    # Unpack archive file into this temporary directory
+    # Windows BSD-tar handles zip. Imagine that.
+    Write-Output "Unpacking $pkg_download"
+    & tar xf "$pkg_download"
+
+    # Settle unpacked archive into place
+    Write-Output "Install Location: $pkg_src_cmd"
+    Move-Item -Path ".\julia*" -Destination "$pkg_src_dir"
+
+    # Exit tmp
+    Pop-Location
+}
+
+Write-Output "Copying into '$pkg_dst' from '$pkg_src'"
+Remove-Item -Path "$pkg_dst" -Recurse -ErrorAction Ignore | Out-Null
+Copy-Item -Path "$pkg_src" -Destination "$pkg_dst" -Recurse
+webi_path_add "$pkg_dst_bin"
+& "$pkg_dst_cmd" --version

julia/install.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+
+# shellcheck disable=SC2034
+# "'pkg_cmd_name' appears unused. Verify it or export it."
+
+__init_julia() {
+    set -e
+    set -u
+
+    ##################
+    # Install julia #
+    ##################
+
+    # Every package should define these 6 variables
+    pkg_cmd_name="julia"
+
+    pkg_dst_cmd="$HOME/.local/opt/julia/bin/julia"
+    pkg_dst_dir="$HOME/.local/opt/julia"
+    pkg_dst="$pkg_dst_dir"
+
+    pkg_src_cmd="$HOME/.local/opt/julia-v$WEBI_VERSION/bin/julia"
+    pkg_src_dir="$HOME/.local/opt/julia-v$WEBI_VERSION"
+    pkg_src="$pkg_src_dir"
+
+    # pkg_install must be defined by every package
+    pkg_install() {
+        # ~/.local/opt/julia-v3.27.0/
+        mkdir -p "$(dirname "${pkg_src_dir}")"
+
+        # mv ./julia-*/ ~/.local/opt/julia-v3.27.0/
+        mv ./julia-*/ "${pkg_src_dir}"
+    }
+
+    # pkg_get_current_version is recommended, but not required
+    pkg_get_current_version() {
+        # 'julia --version' has output in this format:
+        #       julia version 1.10.0-rc1
+        # This trims it down to just the version number:
+        #       1.10.0-rc1
+        julia --version 2> /dev/null |
+            head -n 1 |
+            cut -d ' ' -f 3
+    }
+
+}
+
+__init_julia

julia/releases.js

@@ -0,0 +1,145 @@
+'use strict';
+
+var osMap = {
+  winnt: 'windows',
+  mac: 'darwin',
+};
+var archMap = {
+  armv7l: 'armv7',
+  i686: 'x86',
+  powerpc64le: 'ppc64le',
+};
+
+async function getAllReleases() {
+  let all = {
+    releases: [],
+    download: '',
+    _names: ['julia', 'macaarch64'],
+  };
+
+  let resp = await fetch(
+    'https://julialang-s3.julialang.org/bin/versions.json',
+    {
+      headers: {
+        Accept: 'application/json',
+      },
+    },
+  );
+  let buildsByVersion = await resp.json();
+
+  /*
+      {
+        "url": "https://julialang-s3.julialang.org/bin/mac/aarch64/1.9/julia-1.9.4-macaarch64.tar.gz",
+        "triplet": "aarch64-apple-darwin14",
+        "kind": "archive",
+        "arch": "aarch64",
+        "asc": "-----BEGIN PGP SIGNATURE-----\n\niQIzBAABCAAdFiEENnPfUp2QSUd/drN1ZuPH3APW5JUFAmVTQBcACgkQZuPH3APW\n5JWUqw//QF/CJLAxXZdcXqpBulLUs/AX+x/8aERGcKxZqfeYOwA5efOzma8sASa/\nUzYCLp9E31x/RMDoZah6vPRRjBR+uVI6CLlXCCCmbAJP3lD2vlcY9LKe2/7s3Ba8\nhwITRaL6R5zNr+YfSHW1Hoj2tWgAQh9Y+Te7bP3jzwp5dlFygFO0pzoN+aeJbPNA\nbgT0ry8tgh78/tgNjgt4Ev3E2t3ehhrDGK4tgkkKieO6sdFz8jOacZVZkR1kLVEg\nMBIqmqZfk+5/HMf/6gHwd5GOXW8+GakN7vYXO+9VFETA2EiD5Z5k4Edq/VrNCn4O\npC6WHpBmVBBYX4aQtHkJyQaV8PtFd1j9338jUWlDaa6BVtX2hjRtU1k1oLZB1TTX\nl4awzYgFqdCRnFmOtzTdMDBcfedOiIHdTyxXPjJCX3i0GXmeuk89e5dE4P6sTT9n\n24GeBVQgMaXuNorg9L0oKrsQ8RDT20yEnVbfhy4Cvoq7dNIks6IxLZt10tjJFp1j\n0oJ5f6KucGyqFM9UhXRcuLj8Z+Q+JDzBs5c2pPe/bEzv6nChRNv252e5dve17esg\nK7tHkhXzM+6wl60oyRtpWghOubXyBDsNu1MH3qC9lWy3wmWuMN7no+yX0vGFyhMT\naxjLJSeYdccKD3SuzYotp3XwBKk05PFX9lWy0vuIjVj1sGWcES8=\n=G1tO\n-----END PGP SIGNATURE-----\n",
+        "sha256": "67542975e86102eec95bc4bb7c30c5d8c7ea9f9a0b388f0e10f546945363b01a",
+        "size": 119559478,
+        "version": "1.9.4",
+        "os": "mac",
+        "extension": "tar.gz"
+      }
+   */
+
+  let versions = Object.keys(buildsByVersion);
+  for (let version of versions) {
+    let release = buildsByVersion[version];
+
+    // let odd = isOdd(asset.filename);
+    // if (odd) {
+    //   return;
+    // }
+
+    for (let build of release.files) {
+      // // For debugging
+      // let paths = build.url.split('/');
+      // let extlen = build.extension.length + 1;
+      // let name = paths.at(-1);
+      // name = name.replace(`-${build.version}`, '');
+      // name = name.slice('julia-'.length);
+      // name = name.slice(0, -extlen);
+      // console.log(`name: ${name}`);
+
+      // console.log(`triplet: ${build.triplet}`);
+      // console.log(`arch: ${build.arch}`);
+      // console.log(`os: ${build.os}`);
+      // console.log(`kind: ${build.kind}`);
+      // console.log(`extension: ${build.extension}`);
+      // console.log(`version: ${build.version}`);
+
+      if (build.kind === 'installer') {
+        continue;
+      }
+
+      let arch = archMap[build.arch] || build.arch || '-';
+      let os = osMap[build.os] || build.os || '-';
+      let libc = '';
+      let hardMusl = /\b(musl)\b/.test(build.url);
+      if (hardMusl) {
+        libc = 'musl';
+      } else if (os === 'linux') {
+        libc = 'gnu';
+      }
+
+      let webiBuild = {
+        version: build.version,
+        _version: build.version,
+        lts: false,
+        channel: '', // autodetect by filename (-beta1, -alpha1, -rc1)
+        date: '1970-01-01', // the world may never know
+        os: os,
+        arch: arch,
+        libc: libc,
+        _musl: hardMusl,
+        ext: '', // let normalize run the split/test/join
+        hash: '-', // build.sha256 not ready to standardize this yet
+        download: build.url,
+      };
+      all.releases.push(webiBuild);
+    }
+  }
+
+  all.releases.sort(sortByVersion);
+
+  return all;
+}
+
+function sortByVersion(a, b) {
+  let [aVer, aPre] = a.version.split('-');
+  let [bVer, bPre] = b.version.split('-');
+
+  let aVers = aVer.split('.');
+  let bVers = bVer.split('.');
+  for (let i = 0; i < 3; i += 1) {
+    aVers[i] = aVers[i].padStart(4, '0');
+    bVers[i] = bVers[i].padStart(4, '0');
+  }
+
+  aVer = aVers.join('.');
+  if (aPre) {
+    aVer += `-${aPre}`;
+  }
+  bVer = bVers.join('.');
+  if (bPre) {
+    bVer += `-${aPre}`;
+  }
+
+  if (aVer > bVer) {
+    return -1;
+  }
+  if (aVer < bVer) {
+    return 1;
+  }
+  return 0;
+}
+
+module.exports = getAllReleases;
+
+if (module === require.main) {
+  getAllReleases().then(function (all) {
+    all = require('../_webi/normalize.js')(all);
+    all.releases = all.releases.slice(0, 10);
+    console.info(JSON.stringify(all, null, 2));
+  });
+}

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@webinstall/webi-installers",
-  "version": "1.0.1",
+  "version": "1.1.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@webinstall/webi-installers",
-      "version": "1.0.1",
+      "version": "1.1.1",
       "license": "MPL-2.0",
       "dependencies": {
         "@root/request": "^1.9.2",

package.json

@@ -1,10 +1,11 @@
 {
   "name": "@webinstall/webi-installers",
   "private": true,
-  "version": "1.0.1",
+  "version": "1.1.1",
   "description": "webinstall script repository",
   "main": "_webi/",
   "scripts": {
+    "bump": "npm version -m \"chore(release): bump to v%s\"",
     "fmt": "npm run prettier && npm run shfmt && npm run pwsh-fmt",
     "lint": "npm run shellcheck && npm run jshint && npm run pwsh-lint",
     "prepare": "npm run tooling-init && npm run git-hooks-init",

psscriptanalyzer/install.ps1

@@ -13,7 +13,7 @@ function Repair-MissingCommand {
         Return
     }
 
-    & $HOME\.local\bin\webi-pwsh.ps1 $Package
+    & "$HOME\.local\bin\webi-pwsh.ps1" $Package
     $null = Sync-EnvPath
 }
 

pwsh-essentials/install.ps1

@@ -15,7 +15,7 @@ function Repair-MissingCommand {
         Return
     }
 
-    & $HOME\.local\bin\webi-pwsh.ps1 $Package
+    & "$HOME\.local\bin\webi-pwsh.ps1" $Package
     $null = Sync-EnvPath
 }
 
@@ -47,7 +47,7 @@ function Install-PwshEssential {
     Repair-MissingCommand -Name "PowerShell Core" -Package "pwsh" -Command "pwsh"
 
     # Fetch PSScriptAnalyzer (fmt, lint, fix)
-    & $HOME\.local\bin\webi-pwsh.ps1 psscriptanalyzer
+    & "$HOME\.local\bin\webi-pwsh.ps1" psscriptanalyzer
 
     # Fetch shorthand commands to fmt, lint, & fix
     $ScriptNames = , "pwsh-fmt", "pwsh-fix", "pwsh-lint"

pwsh/install.sh

@@ -45,8 +45,10 @@ __init_pwsh() {
     }
 
     pkg_done_message() {
-        echo "Installed 'pwsh' at $pkg_dst"
-        pwsh -V
+        # We print the version here to ensure the install completed without
+        # errors - no missing libraries, not incompatible arch, etc
+        echo ""
+        "$pkg_dst_cmd" -V
     }
 }
 

pyenv/README.md

@@ -7,22 +7,6 @@ tagline: |
 
 To update run `pyenv update`.
 
-### Files
-
-These are the files / directories that are created and/or modified with this
-install:
-
-```sh
-~/.config/envman/PATH.env
-~/.pyenv/bin/
-~/.pyenv/shims/
-
-# pyenv also loads shell hooks via
-~/.bashrc
-~/.config/fish/config.fish
-~/.zshrc
-```
-
 ### How to Install pyenv on macOS
 
 Install Xcode tools first:
@@ -38,11 +22,21 @@ Make sure that you already have the necessary build tools installed:
 ```sh
 # required
 sudo apt update
-sudo apt install -y build-essential zlib1g-dev libssl-dev
+sudo apt install -y build-essential zlib1g-dev libssl-dev \
+                    libreadline-dev liblzma-dev
 
 # recommended
-sudo apt install -y libreadline-dev libsqlite3-dev \
-                    libffi-dev libbz2-dev liblzma-dev
+sudo apt install -y libsqlite3-dev libffi-dev libbz2-dev
+```
+
+### How to Install pyenv on Alpine
+
+```sh
+sudo apk add \
+    bash \
+    build-base \
+    git \
+    zlib-dev
 ```
 
 ## Cheat Sheet
@@ -82,6 +76,34 @@ Revert back to your system python:
 pyenv global system
 ```
 
+## Table of Contents
+
+- Files
+- List Installable Versions
+- Install a Specific Version
+- Install Local to a Project
+- `virtualenv`s
+  - List Existing
+  - Create New
+  - Activate / Deactivate
+  - Delete
+
+### Files
+
+These are the files / directories that are created and/or modified with this
+install:
+
+```sh
+~/.config/envman/PATH.env
+~/.pyenv/bin/
+~/.pyenv/shims/
+
+# pyenv also loads shell hooks via
+~/.bashrc
+~/.config/fish/config.fish
+~/.zshrc
+```
+
 ### List all available python version
 
 ```sh

pyenv/install.sh

@@ -4,6 +4,13 @@ __init_pyenv() {
     set -e
     set -u
 
+    b_os="$(uname -s)"
+    if test "${b_os}" = 'Darwin'; then
+        if ! test -x /Library/Developer/CommandLineTools/usr/bin/git; then
+            "$HOME/.local/bin/webi" commandlinetools
+        fi
+    fi
+
     curl -fsSL https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
 
     if [ ! -f ~/.bashrc ] || ! grep -q 'pyenv init' ~/.bashrc; then

sd/install.ps1

@@ -45,7 +45,7 @@ IF (!(Test-Path -Path "$pkg_src_cmd")) {
     # Settle unpacked archive into place
     Write-Output "Install Location: $pkg_src_cmd"
     New-Item "$pkg_src_bin" -ItemType Directory -Force | Out-Null
-    Move-Item -Path "b\*\release\sd.exe" -Destination "$pkg_src_bin"
+    Move-Item -Path "sd-*\sd.exe" -Destination "$pkg_src_bin"
     # Exit tmp
     Pop-Location
 }

ssh-pubkey/ssh-pubkey

@@ -2,43 +2,100 @@
 set -e
 set -u
 
+fn_warn_rsa() { (
+    echo 'WARNING'
+    echo '    ~/.ssh/id_rsa is less than the required 3072 bits'
+    echo '    (some modern services will reject your key)'
+
+    echo ''
+    echo 'SOLUTION'
+    echo '    Generate a new key, and update accordingly'
+
+    my_ts="$(date -u "+%F_%H.%M.%S")"
+
+    echo ''
+    echo '    # OPTION 1: Generate a more efficient, ED25519 256-bit key'
+    echo '    #           and update your ~/.ssh/config accordingly'
+    echo "    mv ~/.ssh/id_rsa ~/.ssh/id_rsa.${my_ts}.bak"
+    # shellcheck disable=SC2016
+    echo '    ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519 -q -N ""'
+    echo "    echo 'Host *
+        IdentityFile ~/.ssh/id_ed25519
+        IdentityFile ~/.ssh/id_rsa.${my_ts}.bak' >> ~/.ssh/config"
+
+    echo ''
+    echo '    # OPTION 2: Generate a larger, 4096-bit RSA key"'
+    echo '    #           and update your ~/.ssh/config accordingly'
+    echo "    mv ~/.ssh/id_rsa ~/.ssh/id_rsa.${my_ts}.bak"
+    # shellcheck disable=SC2016
+    echo '    ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_rsa -q -N ""'
+    echo "    echo 'Host *
+        IdentityFile ~/.ssh/id_rsa
+        IdentityFile ~/.ssh/id_rsa.${my_ts}.bak' >> ~/.ssh/config"
+); }
+
 main() {
 
-    if [ ! -d "$HOME/.ssh" ]; then
-        mkdir -p "$HOME/.ssh/"
-        chmod 0700 "$HOME/.ssh/"
+    if ! test -d ~/.ssh; then
+        mkdir -p ~/.ssh/
+        chmod 0700 ~/.ssh/
     fi
 
-    if [ ! -f "$HOME/.ssh/config" ]; then
+    if ! test -f ~/.ssh/config; then
         # for the benefit of VSCode
-        touch "$HOME/.ssh/config"
-        chmod 0644 "$HOME/.ssh/config"
+        touch ~/.ssh/config
+        chmod 0644 ~/.ssh/config
     fi
 
-    if [ ! -f "$HOME/.ssh/authorized_keys" ]; then
-        touch "$HOME/.ssh/authorized_keys"
-        chmod 0600 "$HOME/.ssh/authorized_keys"
+    if ! test -f ~/.ssh/authorized_keys; then
+        touch ~/.ssh/authorized_keys
+        chmod 0600 ~/.ssh/authorized_keys
     fi
 
-    if [ ! -f "$HOME/.ssh/id_rsa" ]; then
-        ssh-keygen -b 2048 -t rsa -f "$HOME/.ssh/id_rsa" -q -N ""
-        echo >&2 ""
-    fi
-
-    if [ ! -f "$HOME/.ssh/id_rsa.pub" ]; then
-        ssh-keygen -y -f "$HOME/.ssh/id_rsa" > "$HOME/.ssh/id_rsa.pub"
+    my_keytype=''
+    if test -f ~/.ssh/id_ed25519; then
+        my_keytype='ed25519'
+        if ! test -f ~/.ssh/id_ed25519.pub; then
+            echo >&2 ""
+            ssh-keygen -y -f ~/.ssh/id_ed25519 > ~/.ssh/id_ed25519.pub
+        fi
+    elif test -f ~/.ssh/id_rsa; then
+        my_keytype='rsa'
+        if ! test -f ~/.ssh/id_rsa.pub; then
+            echo >&2 ""
+            ssh-keygen -y -f ~/.ssh/id_rsa > ~/.ssh/id_rsa.pub
+        fi
+    elif test -f ~/.ssh/id_ecda; then
+        my_keytype='ecdsa'
+        if ! test -f ~/.ssh/id_ecdsa.pub; then
+            echo >&2 ""
+            ssh-keygen -y -f ~/.ssh/id_ecda > ~/.ssh/id_ecda.pub
+        fi
+    else
+        my_keytype='rsa'
         echo >&2 ""
+        echo >&2 "Generating public/private rsa key pair."
+        ssh-keygen -b 4096 -t rsa -f ~/.ssh/id_rsa -q -N ""
+        ssh-keygen -y -f ~/.ssh/id_rsa > ~/.ssh/id_rsa.pub
     fi
 
     # TODO use the comment (if any) for the name of the file
     echo >&2 ""
     #shellcheck disable=SC2088
-    echo >&2 "~/Downloads/id_rsa.$(whoami).pub":
+    echo >&2 "~/Downloads/id_${my_keytype}.$(whoami).pub":
     echo >&2 ""
-    rm -f "$HOME/Downloads/id_rsa.$(whoami).pub"
-    cp -r "$HOME/.ssh/id_rsa.pub" "$HOME/Downloads/id_rsa.$(whoami).pub"
-    cat "$HOME/Downloads/id_rsa.$(whoami).pub"
+    rm -f "$HOME/Downloads/id_${my_keytype}.$(whoami).pub"
+    cp -RPp "$HOME/.ssh/id_${my_keytype}.pub" "$HOME/Downloads/id_${my_keytype}.$(whoami).pub"
+    cat "$HOME/Downloads/id_${my_keytype}.$(whoami).pub"
     echo >&2 ""
+
+    if test -f ~/.ssh/id_rsa; then
+        my_rsa_chars="$(cat ~/.ssh/id_rsa)"
+        if test "${#my_rsa_chars}" -lt 2500; then
+            fn_warn_rsa >&2
+            echo >&2 ""
+        fi
+    fi
 }
 
 main

vim-devicons/README.md

@@ -1,6 +1,6 @@
 ---
 title: vim-devicons
-homepage: https://github.com/ryanoasis/devicons
+homepage: https://github.com/ryanoasis/vim-devicons
 tagline: |
   VimDevIcons: Adds Icons to Your Plugins.
 ---

vim-things.sh

@@ -1,12 +0,0 @@
-vim README.md
-vim _webi/template.ps1
-vim _webi/template.sh
-vim brew/install.sh
-vim git/install.ps1
-vim golang/install.ps1
-vim golang/install.sh
-vim node/install.ps1
-vim postgres/install.sh
-vim sass/install.ps1
-vim webi/install.sh
-vim zig/install.ps1

webi-essentials/install.sh

@@ -1,7 +1,6 @@
 #!/bin/sh
 set -e
 set -u
-#set -x
 
 fn_install_webi_essentials_macos() { (
     # streamline the output to be pretty
@@ -181,7 +180,7 @@ _install_webi_essentials() { (
         fi
     fi
 
-    printf '%s' "    $(t_dim 'Checking for apt/apk ...')"
+    printf '%s' "    $(t_dim 'Checking for apt/apk/pkg_add ...')"
     if command -v apt > /dev/null; then
         echo " $(t_pkg 'apt')"
         _install_webi_essentials_apt "${cmd_sudo}" "${b_pkgs}"
@@ -196,6 +195,13 @@ _install_webi_essentials() { (
         return 0
     fi
 
+    if command -v pkg_add > /dev/null; then
+        echo " $(t_pkg 'pkg_add')"
+        _install_webi_essentials_pkg_add "${cmd_sudo}" "${b_pkgs}"
+        echo "    $(t_dim 'OK')"
+        return 0
+    fi
+
     echo " $(t_dim 'none')"
     _install_webi_essentials_webi "${cmd_sudo}" "${b_pkgs}"
     echo "    $(t_dim 'OK')"
@@ -231,11 +237,21 @@ _install_webi_essentials_apk() { (
     ${cmd_sudo} apk add --no-cache ${b_pkgs}
 ); }
 
+_install_webi_essentials_pkg_add() { (
+    cmd_sudo="${1}"
+    b_pkgs="${2}"
+
+    echo "    $(t_dim 'Running') $(t_cmd "${cmd_sudo}pkg_add")"
+    fn_polite_sudo "${cmd_sudo}" "    $(t_cmd "pkg_add ${b_pkgs}")"
+    # shellcheck disable=SC2086
+    ${cmd_sudo} pkg_add ${b_pkgs}
+); }
+
 _install_webi_essentials_webi() { (
     cmd_sudo="${1}"
     b_pkgs="${2}"
 
-    if test "${b_pkgs}" == 'xz'; then
+    if test "${b_pkgs}" = 'xz'; then
         ~/.local/bin/webi xz
         return 0
     fi

webi/releases.js

@@ -1,43 +0,0 @@
-'use strict';
-
-module.exports = async function () {
-  return {
-    releases: [
-      {
-        name: 'webi.tar',
-        version: '0.0.0',
-        lts: false,
-        channel: 'stable',
-        date: '',
-        os: 'linux',
-        arch: 'amd64',
-        ext: 'tar',
-        download: '',
-      },
-      {
-        name: 'webi.tar',
-        version: '0.0.0',
-        lts: false,
-        channel: 'stable',
-        date: '',
-        os: 'macos',
-        arch: 'amd64',
-        ext: 'tar',
-        download: '',
-      },
-      {
-        name: 'webi.tar',
-        version: '0.0.0',
-        lts: false,
-        channel: 'stable',
-        date: '',
-        os: 'windows',
-        arch: 'amd64',
-        ext: 'tar',
-        download: '',
-      },
-    ],
-    formats: [],
-    downloads: '',
-  };
-};

webi/webi-pwsh.ps1

@@ -133,7 +133,7 @@ Push-Location $Env:USERPROFILE
 New-Item -Path "$HOME\.local\bin\" -ItemType Directory -Force | Out-Null
 
 # See note on Set-ExecutionPolicy above
-Set-Content -Path "$HOME\.local\bin\webi.bat" -Value "@echo off`r`npowershell -ExecutionPolicy Bypass %USERPROFILE%\.local\bin\webi-pwsh.ps1 %*"
+Set-Content -Path "$HOME\.local\bin\webi.bat" -Value "@echo off`r`npowershell -ExecutionPolicy Bypass -File `"%USERPROFILE%\.local\bin\webi-pwsh.ps1`" %*"
 # Backwards-compat bugfix: remove old webi-pwsh.ps1 location
 Remove-Item -Path "$HOME\.local\bin\webi.ps1" -Recurse -ErrorAction Ignore
 if (!(Test-Path -Path "$HOME\.local\opt")) {
@@ -199,7 +199,7 @@ $UrlParams = "formats=zip,exe,tar,git&libc=msvc"
 $PkgInstallPwsh = "$HOME\.local\tmp\$exename.install.ps1"
 Invoke-DownloadUrl -Force -URL $PKG_URL -Params $UrlParams -Path $PkgInstallPwsh
 
-powershell $HOME\.local\tmp\${exename}.install.ps1
+powershell -File "$HOME\.local\tmp\${exename}.install.ps1"
 
 IF ($IsWebiParent) {
     Write-Host ""

webi/webi.sh

@@ -63,10 +63,11 @@ __webi_main() {
     export WEBI_URL
     set -e
 
+    my_uname_o="$(uname -o 2> /dev/null || echo '')"
     my_libc=''
     if ldd /bin/ls 2> /dev/null | grep -q 'musl' 2> /dev/null; then
         my_libc='musl'
-    elif uname -o | grep -q 'GNU' || uname -s | grep -q 'Linux'; then
+    elif echo "${my_uname_o}" | grep -q 'GNU' || uname -s | grep -q 'Linux'; then
         my_libc='gnu'
     else
         my_libc='libc'
@@ -141,16 +142,30 @@ __webi_main() {
     }
 
     fn_checksum() {
-        cmd_shasum='sha1sum'
-        if command -v shasum > /dev/null; then
-            cmd_shasum='shasum'
+        a_filepath="${1}"
+
+        if command -v sha1sum > /dev/null; then
+            sha1sum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
+            return 0
         fi
-        $cmd_shasum "${0}" | cut -d' ' -f1 | cut -c 1-8
+
+        if command -v shasum > /dev/null; then
+            shasum "${a_filepath}" | cut -d' ' -f1 | cut -c 1-8
+            return 0
+        fi
+
+        if command -v sha1 > /dev/null; then
+            sha1 "${a_filepath}" | cut -d'=' -f2 | cut -c 2-9
+            return 0
+        fi
+
+        echo >&2 "    warn: no sha1 sum program"
+        date '+%F %H:%M'
     }
 
     version() {
         my_checksum="$(
-            fn_checksum
+            fn_checksum "${0}"
         )"
         my_version=v1.2.8
         printf "\e[35mwebi\e[32m %s\e[0m Copyright 2020+ AJ ONeal\n" "${my_version} (${my_checksum})"

xcode-cli

@@ -0,0 +1 @@
+commandlinetools