Compare commits

..

2 Commits

Author SHA1 Message Date
AJ ONeal
b96f457496 wip: feat: log whenever directories are created 2023-11-20 00:05:35 +00:00
AJ ONeal
0b3f73f661 wip: feat: add convenience functions to webi script 2023-11-20 00:05:35 +00:00
33 changed files with 726 additions and 2736 deletions

View File

@@ -23,15 +23,10 @@ jobs:
sh ./_scripts/install-ci-deps
echo "${HOME}/.local/bin" >> $GITHUB_PATH
echo "${HOME}/.local/opt/pwsh" >> $GITHUB_PATH
- run: |
git submodule init
git submodule update
- run: shfmt --version
- run: shellcheck -V
- run: node --version
- run: npm run fmt
- run: npm ci
- run: npm run lint
- env:
GITHUB_TOKEN: ${{ github.token }}
run: npm run test
- run: npm run test

3
.gitmodules vendored
View File

@@ -1,3 +0,0 @@
[submodule "_webi/build-classifier"]
path = _webi/build-classifier
url = https://github.com/webinstall/webi-build-classifier.git

View File

@@ -6,12 +6,11 @@ var Crypto = require('crypto');
var util = require('util');
var exec = util.promisify(require('child_process').exec);
var Fs = require('node:fs/promises');
var FsSync = require('node:fs');
var Path = require('node:path');
var repoBaseDir = process.env.REPO_BASE_DIR || '';
if (!repoBaseDir) {
repoBaseDir = Path.resolve('./_repos');
repoBaseDir = Path.resolve('./repos');
// for stderr
console.error(`[Warn] REPO_BASE_DIR= not set, ${repoBaseDir}`);
}
@@ -21,29 +20,8 @@ var Repos = {};
Repos.clone = async function (repoPath, gitUrl) {
let uuid = Crypto.randomUUID();
let tmpPath = `${repoPath}.${uuid}.tmp`;
let bakPath = `${repoPath}.${uuid}.dup`;
await exec(`git clone --bare --filter=tree:0 ${gitUrl} ${tmpPath}`);
try {
FsSync.accessSync(repoPath);
return;
} catch (e) {
if (e.code !== 'ENOENT') {
throw e;
}
}
// sync to avoid race conditions
try {
FsSync.renameSync(repoPath, bakPath);
} catch (e) {
if (e.code !== 'ENOENT') {
throw e;
}
}
FsSync.renameSync(tmpPath, repoPath);
await Fs.rm(bakPath, { force: true, recursive: true });
await Fs.rename(tmpPath, repoPath);
};
Repos.checkExists = async function (repoPath) {

View File

@@ -1,958 +0,0 @@
'use strict';
var BuildsCacher = module.exports;
let Fs = require('node:fs/promises');
let Path = require('node:path');
let HostTargets = require('./build-classifier/host-targets.js');
let Lexver = require('./build-classifier/lexver.js');
let Triplet = require('./build-classifier/triplet.js');
let request = require('@root/request');
var ALIAS_RE = /^alias: (\w+)$/m;
var LEGACY_ARCH_MAP = {
'*': 'ANYARCH',
arm64: 'aarch64',
armv6l: 'armv6',
armv7l: 'armv7',
amd64: 'x86_64',
mipsle: 'mipsel',
mips64le: 'mips64el',
mipsr6le: 'mipsr6el',
mips64r6le: 'mips64r6el',
// yes... el for arm and mips, but le for ppc
// (perhaps the joke got old?)
ppc64el: 'ppc64le',
386: 'x86',
};
var LEGACY_OS_MAP = {
'*': 'ANYOS',
macos: 'darwin',
posix: 'posix_2017',
};
var TERMS_META = [
// pattern
'{ARCH}',
'{EXT}',
'{LIBC}',
'{NAME}',
'{OS}',
'{VENDOR}',
// // os-/arch-indepedent
// 'ANYARCH',
// 'ANYOS',
// // libc
// 'none',
// channel
'beta',
'dev',
'preview',
'stable',
];
/** @typedef {String} TripletString - {arch}-{vendor}-{os}-{libc} */
/** @typedef {String} VersionString */
/** @typedef {Object.<VersionString, Array<BuildAsset>>} PackagesByRelease */
/**
* @typedef ProjectInfo
* @prop {Array<BuildAsset>} releases
* @prop {Array<BuildAsset>} packages
* @prop {Object.<TripletString, PackagesByRelease>} releasesByTriplet
* @prop {Array<import('./build-classifier/types.js').ArchString>} arches
* @prop {Array<import('./build-classifier/types.js').OsString>} oses
* @prop {Array<import('./build-classifier/types.js').LibcString>} libcs
* @prop {Array<String>} channels
* @prop {Array<String>} formats
* @prop {Array<String>} triplets
* @prop {Array<String>} versions
* @prop {Array<String>} lexvers
* @prop {Object.<String, String>} lexversMap
*/
/**
* @typedef BuildAsset
* @prop {String} name
* @prop {String} version
* @prop {Boolean} lts
* @prop {String} date
* @prop {String} arch
* @prop {String} os
* @prop {String} libc
* @prop {String} ext
* @prop {String} download
*/
/**
* @typedef VersionTarget
* @prop {String} version
* @prop {Boolean} lts
* @prop {String} channel
*/
/** @typedef {TargetTriplet & HostTargetPartial} HostTarget */
/** @typedef {import('./build-classifier/types.js').TargetTriplet} TargetTriplet */
/**
* @typedef HostTargetPartial
* @prop {String} target.triplet - os-vendor-arch-libc
* @prop {Error} target.error
*/
async function getPartialHeader(path) {
let readme = `${path}/README.md`;
let head = await readFirstBytes(readme).catch(function (err) {
if (err.code !== 'ENOENT') {
console.warn(`warn: ${path}: ${err.message}`);
}
return null;
});
return head;
}
// let fsOpen = util.promisify(Fs.open);
// let fsRead = util.promisify(Fs.read);
async function readFirstBytes(path) {
let start = 0;
let n = 1024;
let fh = await Fs.open(path, 'r');
let buf = Buffer.alloc(n);
let result = await fh.read(buf, start, n);
let str = result.buffer.toString('utf8');
await fh.close();
return str;
}
let promises = {};
async function getLatestBuilds(Releases, installersDir, cacheDir, name, date) {
if (!Releases) {
Releases = require(`${installersDir}/${name}/releases.js`);
}
// TODO update all releases files with module.exports.xxxx = 'foo';
if (!Releases.latest) {
Releases.latest = Releases;
}
let id = `${cacheDir}/${name}`;
if (!promises[id]) {
promises[id] = Promise.resolve();
}
promises[id] = promises[id].then(async function () {
return await getLatestBuildsInner(Releases, cacheDir, name, date);
});
return await promises[id];
}
async function getLatestBuildsInner(Releases, cacheDir, name, date) {
let data = await Releases.latest(request);
if (!date) {
date = new Date();
}
let isoDate = date.toISOString();
let yearMonth = isoDate.slice(0, 7);
// TODO hash file
let dataFile = `${cacheDir}/${yearMonth}/${name}.json`;
// TODO fsstat releases.js vs require-ing time as well
let tsFile = `${cacheDir}/${yearMonth}/${name}.updated.txt`;
let dirPath = Path.dirname(dataFile);
await Fs.mkdir(dirPath, { recursive: true });
let json = JSON.stringify(data, null, 2);
await Fs.writeFile(dataFile, json, 'utf8');
let seconds = date.valueOf();
let ms = seconds / 1000;
let msStr = ms.toFixed(3);
await Fs.writeFile(tsFile, msStr, 'utf8');
return data;
}
BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
let installersDir = installers;
let cacheDir = caches;
if (!ALL_TERMS) {
ALL_TERMS = Triplet.TERMS_PRIMARY_MAP;
}
let bc = {};
bc.usedTerms = {};
bc.orphanTerms = Object.assign({}, ALL_TERMS);
bc.unknownTerms = {};
bc.formats = [];
bc._triplets = {};
bc._targetsByBuildIdCache = {};
bc._caches = {};
bc._staleAge = 15 * 60 * 1000;
bc._allFormats = {};
bc._allTriplets = {};
for (let term of TERMS_META) {
delete bc.orphanTerms[term];
}
bc.getProjectsByType = async function () {
let dirs = {
hidden: {},
errors: {},
alias: {},
invalid: {},
selfhosted: {},
valid: {},
};
let entries = await Fs.readdir(installersDir, { withFileTypes: true });
for (let entry of entries) {
let meta = await bc.getProjectTypeByEntry(entry);
if (meta.type === 'not_found') {
let err = meta.detail;
console.error('');
console.error('PROBLEM');
console.error(` ${err.message}`);
console.error('');
console.error('SOLUTION');
console.error(' npm clean-install');
console.error('');
throw new Error(
'[SANITY FAIL] should never have missing modules in prod',
);
}
dirs[meta.type][entry.name] = meta.detail;
}
return dirs;
};
/**
* Get project type and detail - alias, selfhosted, valid (and the invalids)
* @param {String} name - filename
*/
bc.getProjectType = async function (name) {
let filepath = Path.join(installersDir, name);
let entry;
try {
entry = await Fs.lstat(filepath);
Object.assign(entry, { name: name });
} catch (e) {
return { type: 'errors', detail: 'not found' };
}
let info = await bc.getProjectTypeByEntry(entry);
return info;
};
/**
* Get project type and detail - alias, selfhosted, valid (and the invalids)
* @param {fs.Stats|fs.Dirent} entry
*/
bc.getProjectTypeByEntry = async function (entry) {
let path = Path.join(installersDir, entry.name);
// skip non-installer dirs
if (entry.isSymbolicLink()) {
let link = await Fs.readlink(path);
return { type: 'alias', detail: link };
}
if (!entry.isDirectory()) {
return { type: 'hidden', detail: '!directory' };
}
if (entry.name === 'node_modules') {
return { type: 'hidden', detail: 'node_modules' };
}
if (entry.name.startsWith('_')) {
return { type: 'hidden', detail: '_*' };
}
if (entry.name.startsWith('.')) {
return { type: 'hidden', detail: '.*' };
}
if (entry.name.startsWith('~')) {
return { type: 'hidden', detail: '~*' };
}
if (entry.name.endsWith('~')) {
return { type: 'hidden', detail: '*~' };
}
// skip invalid installers
let head = await getPartialHeader(path);
if (!head) {
return { type: 'invalid', detail: '!README.md' };
}
let alias = head.match(ALIAS_RE);
if (alias) {
let link = alias[1];
return { type: 'alias', detail: link };
}
let releasesPath = Path.join(path, 'releases.js');
try {
void require(releasesPath);
} catch (err) {
if (err.code !== 'MODULE_NOT_FOUND') {
return { type: 'errors', detail: err };
}
if (err.message.includes(`Cannot find module '${releasesPath}'`)) {
return { type: 'selfhosted', detail: true };
}
return { type: 'not_found', detail: err };
}
return { type: 'valid', detail: true };
};
// Typically a package is organized by release (ex: go has 1.20, 1.21, etc),
// but we will organize by the build (ex: go1.20-darwin-arm64.tar.gz, etc).
bc.getPackages = async function ({ Releases, name, date }) {
if (!date) {
date = new Date();
}
let isoDate = date.toISOString();
let yearMonth = isoDate.slice(0, 7);
let dataFile = `${cacheDir}/${yearMonth}/${name}.json`;
// let secondsStr = await Fs.readFile(tsFile, 'ascii').catch(function (err) {
// if (err.code !== 'ENOENT') {
// throw err;
// }
// return '0';
// });
// secondsStr = secondsStr.trim();
// let seconds = parseFloat(secondsStr) || 0;
// let age = now - seconds;
let projInfo = bc._caches[name];
let meta = {
// version info
versions: projInfo?.versions || [],
lexvers: projInfo?.lexvers || [],
lexversMap: projInfo?.lexversMap || {},
// culled release assets
packages: projInfo?.packages || [],
releasesByTriplet: projInfo?.releasesByTriplet || {},
// target info
triplets: projInfo?.triplets || [],
oses: projInfo?.oses || [],
arches: projInfo?.arches || [],
libcs: projInfo?.libcs || [],
formats: projInfo?.formats || [],
// TODO channels: projInfo?.channels || [],
};
if (!projInfo) {
let json = await Fs.readFile(dataFile, 'ascii').catch(
async function (err) {
if (err.code !== 'ENOENT') {
throw err;
}
return null;
},
);
try {
projInfo = JSON.parse(json);
} catch (e) {
console.error(`error: ${dataFile}:\n\t${e.message}`);
projInfo = null;
}
}
if (!projInfo) {
projInfo = await getLatestBuilds(Releases, installersDir, cacheDir, name);
}
process.nextTick(async function () {
let now = date.valueOf();
let age = now - projInfo.updated;
if (age < bc._staleAge) {
return;
}
projInfo = await getLatestBuilds(Releases, installersDir, cacheDir, name);
transformAndUpdate(name, projInfo, meta, date);
});
transformAndUpdate(name, projInfo, meta, date);
return projInfo;
};
function transformAndUpdate(name, projInfo, meta, date) {
meta.packages = [];
let updated = date.valueOf();
Object.assign(projInfo, { name, updated }, meta);
for (let build of projInfo.releases) {
let buildTarget = bc.classify(projInfo, build);
if (!buildTarget) {
// ignore known, non-package extensions
continue;
}
if (buildTarget.error) {
let err = buildTarget.error;
let code = err.code || '';
console.error(`[ERROR]: ${code} ${projInfo.name}: ${build.name}`);
console.error(`>>> ${err.message} <<<`);
console.error(projInfo);
console.error(build);
console.error(`^^^ ${err.message} ^^^`);
console.error(err.stack);
continue;
}
if (!build.name) {
build.name = build.download.replace(/.*\//, '');
}
build.target = buildTarget;
meta.packages.push(build);
}
updateReleasesByTriplet(meta);
updateAndSortVersions(projInfo, meta);
Object.assign(projInfo, { name, updated }, meta);
bc._caches[name] = projInfo;
}
function updateReleasesByTriplet(meta) {
for (let build of meta.packages) {
let target = build.target;
let triplet = `${target.os}-${target.arch}-${target.libc}`;
if (!meta.releasesByTriplet[triplet]) {
meta.releasesByTriplet[triplet] = {};
}
let buildsByRelease = meta.releasesByTriplet[triplet];
if (!buildsByRelease[build.version]) {
buildsByRelease[build.version] = [];
}
let packages = buildsByRelease[build.version];
packages.push(build);
}
}
// TODO
// - tag channels
function updateAndSortVersions(projInfo, meta) {
for (let build of projInfo.packages) {
let hasVersion = meta.versions.includes(build.version);
if (!hasVersion) {
build.lexver = Lexver.parseVersion(build.version);
meta.lexversMap[build.lexver] = build.version;
}
}
meta.lexvers = Object.keys(meta.lexversMap);
meta.lexvers.sort();
meta.lexvers.reverse();
meta.versions = [];
for (let lexver of meta.lexvers) {
let version = meta.lexversMap[lexver];
meta.versions.push(version);
}
projInfo.packages.sort(function (a, b) {
if (a.lexver > b.lexver) {
return -1;
}
if (a.lexver < b.lexver) {
return 1;
}
return 0;
});
}
// Makes sure that packages are updated once an hour, on average
bc._staleNames = [];
bc._freshenTimeout = null;
bc.freshenRandomPackage = async function (minDelay) {
if (!minDelay) {
minDelay = 15 * 1000;
}
if (bc._staleNames.length === 0) {
let dirs = await bc.getProjectsByType();
bc._staleNames = Object.keys(dirs.valid);
bc._staleNames.sort(function () {
return 0.5 - Math.random();
});
}
let name = bc._staleNames.pop();
void (await bc.getPackages({
//Releases: Releases,
name: name,
date: new Date(),
}));
let hour = 60 * 60 * 1000;
let delay = minDelay;
let spread = hour / bc._staleNames.length;
let seed = Math.random();
delay += seed * spread;
clearTimeout(bc._freshenTimeout);
bc._freshenTimeout = setTimeout(bc.freshenRandomPackage, delay);
bc._freshenTimeout.unref();
};
bc.classify = function (projInfo, build) {
/* jshint maxcomplexity: 25 */
let maybeInstallable = Triplet.maybeInstallable(projInfo, build);
if (!maybeInstallable) {
return null;
}
if (LEGACY_OS_MAP[build.os]) {
build.os = LEGACY_OS_MAP[build.os];
}
if (LEGACY_ARCH_MAP[build.arch]) {
build.arch = LEGACY_ARCH_MAP[build.arch];
}
// because some packages are shimmed to match a single download against
let preTarget = Object.assign({ os: '', arch: '', libc: '' }, build);
let targetId = `${preTarget.os}:${preTarget.arch}:${preTarget.libc}`;
let buildId = `${projInfo.name}:${targetId}@${build.download}`;
let target = bc._targetsByBuildIdCache[buildId];
if (target) {
Object.assign(build, { target: target, triplet: target.triplet });
return target;
}
let pattern = Triplet.toPattern(projInfo, build);
if (!pattern) {
let err = new Error(`no pattern generated for ${projInfo.name}`);
err.code = 'E_BUILD_NO_PATTERN';
target = { error: err };
bc._targetsByBuildIdCache[buildId] = target;
return target;
}
let rawTerms = pattern.split(/[_\{\}\/\.\-]+/g);
for (let term of rawTerms) {
delete bc.orphanTerms[term];
bc.usedTerms[term] = true;
}
// {NAME}/{NAME}-{VER}-Windows-x86_64_v2-musl.exe =>
// {NAME}.windows.x86_64v2.musl.exe
let terms = Triplet.patternToTerms(pattern);
if (!terms.length) {
let err = new Error(`'${terms}' was trimmed to ''`);
target = { error: err };
bc._targetsByBuildIdCache[buildId] = target;
return target;
}
for (let term of terms) {
if (!term) {
continue;
}
if (ALL_TERMS[term]) {
delete bc.orphanTerms[term];
bc.usedTerms[term] = true;
continue;
}
bc.unknownTerms[term] = true;
}
// {NAME}.windows.x86_64v2.musl.exe
// windows-x86_64_v2-musl
target = { triplet: '' };
void Triplet.termsToTarget(target, projInfo, build, terms);
target.triplet = `${target.arch}-${target.vendor}-${target.os}-${target.libc}`;
{
// TODO I don't love this hidden behavior
// perhaps classify should just happen when the package is loaded
// (and the sanity error should be removed, or thrown after the loop is complete)
let hasTriplet = projInfo.triplets.includes(target.triplet);
if (!hasTriplet) {
projInfo.triplets.push(target.triplet);
}
let hasOs = projInfo.oses.includes(target.os);
if (!hasOs) {
projInfo.oses.push(target.os);
}
let hasArch = projInfo.arches.includes(target.arch);
if (!hasArch) {
projInfo.arches.push(target.arch);
}
let hasLibc = projInfo.libcs.includes(target.libc);
if (!hasLibc) {
projInfo.libcs.push(target.libc);
}
if (!build.ext) {
build.ext = Triplet.buildToPackageType(build);
}
if (build.ext) {
if (!build.ext.startsWith('.')) {
build.ext = `.${build.ext}`;
}
}
let hasExt = projInfo.formats.includes(build.ext);
if (!hasExt) {
projInfo.formats.push(build.ext);
}
let hasGlobalExt = bc.formats.includes(build.ext);
if (!hasGlobalExt) {
bc.formats.push(build.ext);
}
}
bc._triplets[target.triplet] = true;
bc._targetsByBuildIdCache[buildId] = target;
let triple = [target.arch, target.vendor, target.os, target.libc];
for (let term of triple) {
if (!ALL_TERMS[term]) {
throw new Error(
`[SANITY FAIL] '${projInfo.name}' '${target.triplet}' generated unknown term '${term}'`,
);
}
delete bc.orphanTerms[term];
bc.usedTerms[term] = true;
}
return target;
};
/**
* Given a list of acceptable formats, get the sorted list of of formats.
* Actually used (as per node _webi/lint-builds.js):
* .7z
* .app.zip
* .dmg
* .exe
* .exe.xz
* .git
* .gz
* .msi
* .pkg
* .pkg.tar.zst
* .sh
* .tar.gz
* .tar.xz
* .xz
* .zip
*/
bc.getSortedFormats = function (formats) {
/* jshint maxcomplexity: 25 */
formats.sort();
let id = formats.join(',');
if (bc._allFormats[id]) {
return bc._allFormats[id];
}
// we don't know how to handle any of these yet
// let exclude = [];
// let isAndroid = false;
// if (!isAndroid) {
// exclude.push('.apk');
// }
// let isDebian = false;
// if (!isDebian) {
// exclude.push('.deb');
// }
// let isEnterpriseLinux = false;
// if (!isEnterpriseLinux) {
// exclude.push('.rpm');
// }
// let isArch = false;
// if (!isArch) {
// exclude.push('.pkg.tar.zst');
// }
let hasExe = formats.includes('exe') || formats.includes('.exe');
/** @type {Array<String>} */
let exts = [];
let hasXz = formats.includes('xz') || formats.includes('.xz');
if (hasXz) {
exts.push('.tar.xz');
if (hasExe) {
exts.push('.exe.xz');
}
exts.push('.xz');
}
let hasZst = formats.includes('zst') || formats.includes('.zst');
if (hasZst) {
exts.push('.tar.zst');
exts.push('.zst');
}
let hasZip = formats.includes('zip') || formats.includes('.zip');
if (hasZip) {
exts.push('.zip');
}
let has7z = false;
if (has7z) {
exts.push('.7z');
}
// let hasBz2 = formats.includes('bz2') || formats.includes('.bz2');
// if (hasBz2) {
// exts.push('.bz2');
// }
if (hasExe) {
if (!hasZip) {
exts.push('.zip');
}
exts.push('.tar.gz');
exts.push('.gz');
exts.push('.exe');
exts.push('.msi');
//exts.push('.msixbundle');
} else {
exts.push('.tar.gz');
exts.push('.gz');
exts.push('.sh');
}
exts.push('.git');
// Fallbacks
exts.push('.app.zip');
exts.push('.dmg');
exts.push('.pkg');
if (!hasXz) {
exts.push('.tar.xz');
if (hasExe) {
exts.push('.exe.xz');
}
exts.push('.xz');
}
if (!hasZip) {
if (!hasExe) {
exts.push('.zip');
}
}
if (!hasZst) {
exts.push('.tar.zst');
exts.push('.zst');
}
if (!has7z) {
exts.push('.7z');
}
// if (!hasRar) {
// exts.push('.rar');
// }
// exts.push('.tar.bz2');
// exts.push('.bz2');
bc._allFormats[id] = exts;
return exts;
};
bc.selectPackage = function (packages, formats) {
if (packages.length === 1) {
return packages[0];
}
let exts = bc.getSortedFormats(formats);
for (let ext of exts) {
for (let build of packages) {
if (build.ext === ext) {
return build;
}
}
}
return packages[0];
};
/**
* @param {ProjectInfo} projInfo
* @param {HostTarget} hostTarget
*/
bc.enumerateLatestVersions = function (projInfo, hostTarget) {
let lexPrefix = '';
let matchInfo = Lexver.matchSorted(projInfo.lexvers, lexPrefix);
let verInfo = {
default: projInfo.lexversMap[matchInfo.default],
previous: projInfo.lexversMap[matchInfo.previous],
stable: projInfo.lexversMap[matchInfo.stable],
latest: projInfo.lexversMap[matchInfo.latest],
};
return verInfo;
};
/**
* @param {ProjectInfo} projInfo
* @param {HostTarget} hostTarget
* @param {VersionTarget} verTarget
*/
bc.findMatchingPackages = function (projInfo, hostTarget, verTarget) {
let matchInfo = bc._enumerateVersions(projInfo, verTarget.version);
let triplets = bc._enumerateTriplets(hostTarget);
//console.log('dbg: matchInfo', matchInfo);
if (matchInfo) {
for (let _triplet of triplets) {
let targetReleases = projInfo.releasesByTriplet[_triplet];
if (!targetReleases) {
continue;
}
// Make sure that these releases are the expected version
// (ex: jq1.7 => darwin-arm64-libc, jq1.6 => darwin-x86_64-libc)
for (let matchver of matchInfo.matches) {
let ver = projInfo.lexversMap[matchver] || matchver;
let packages = targetReleases[ver];
if (!packages) {
continue;
}
let match = {
triplet: _triplet,
packages: packages,
latest: projInfo.versions[0],
version: ver,
versions: matchInfo,
};
return match;
}
}
return null;
}
for (let _triplet of triplets) {
let targetReleases = projInfo.releasesByTriplet[_triplet];
if (!targetReleases) {
continue;
}
let versions = Object.keys(targetReleases);
//console.log('dbg: targetRelease versions', versions);
let lexvers = [];
for (let version of versions) {
let lexPrefix = Lexver.parseVersion(version);
lexvers.push(lexPrefix);
}
lexvers.sort();
lexvers.reverse();
// TODO get the other matchInfo props
// Make sure that these releases are the expected version
// (ex: jq1.7 => darwin-arm64-libc, jq1.6 => darwin-x86_64-libc)
for (let matchver of lexvers) {
let ver = projInfo.lexversMap[matchver] || matchver;
let packages = targetReleases[ver];
//console.log('dbg: packages', packages);
if (!packages) {
continue;
}
let pkg = packages[0];
if (verTarget.lts) {
if (!pkg.lts) {
continue;
}
let match = {
triplet: _triplet,
packages: packages,
latest: projInfo.versions[0],
version: ver,
versions: matchInfo,
};
return match;
}
let wantChannel = verTarget.channel || 'stable';
let isChannel = pkg.channel || 'stable';
if (wantChannel === 'stable') {
if (isChannel !== 'stable') {
continue;
}
}
// latest, beta, alpha, rc, preview
let match = {
triplet: _triplet,
packages: packages,
latest: projInfo.versions[0],
version: ver,
versions: matchInfo,
};
return match;
}
}
return null;
};
bc._enumerateTriplets = function (hostTarget) {
let id = [hostTarget.os, hostTarget.arch, hostTarget.libc].join(',');
let triplets = bc._allTriplets[id] || [];
if (triplets.length > 0) {
return triplets;
}
let oses = [];
if (hostTarget.os === 'windows') {
oses = ['ANYOS', 'windows'];
} else if (hostTarget.os === 'android') {
oses = ['ANYOS', 'posix_2017', 'android', 'linux'];
} else {
oses = ['ANYOS', 'posix_2017', hostTarget.os];
}
let waterfall = HostTargets.WATERFALL[hostTarget.os] || {};
let arches = waterfall[hostTarget.arch] ||
HostTargets.WATERFALL.ANYOS[hostTarget.arch] || [hostTarget.arch];
arches = ['ANYARCH'].concat(arches);
let libcs = waterfall[hostTarget.libc] ||
HostTargets.WATERFALL.ANYOS[hostTarget.libc] || [hostTarget.libc];
for (let os of oses) {
for (let arch of arches) {
for (let libc of libcs) {
let triplet = `${os}-${arch}-${libc}`;
triplets.push(triplet);
}
}
}
bc._allTriplets[id] = triplets;
return triplets;
};
bc._enumerateVersions = function (projInfo, ver) {
if (!ver) {
return null;
}
let lexPrefix = Lexver.parsePrefix(ver);
let matchInfo = Lexver.matchSorted(projInfo.lexvers, lexPrefix);
return matchInfo;
};
return bc;
};

View File

@@ -1,41 +0,0 @@
'use strict';
let Builds = module.exports;
let Path = require('node:path');
let BuildsCacher = require('./builds-cacher.js');
// let HostTargets = require('./build-classifier/host-targets.js');
let Parallel = require('./parallel.js');
var INSTALLERS_DIR = Path.join(__dirname, '..');
var CACHE_DIR = Path.join(__dirname, '../_cache');
let bc = BuildsCacher.create({
caches: CACHE_DIR,
installers: INSTALLERS_DIR,
});
bc.freshenRandomPackage(600 * 1000);
Builds.init = async function () {
bc.freshenRandomPackage(600 * 1000);
let dirs = await bc.getProjects();
let projNames = Object.keys(dirs.valid);
let parallel = 25;
await Parallel.run(parallel, projNames, getAll);
async function getAll(name) {
void (await bc.getPackages({
//Releases: Releases,
name: name,
date: new Date(),
}));
}
};
Builds.enumerateLatestVersions = bc.enumerateLatestVersions;
Builds.findMatchingPackages = bc.findMatchingPackages;
Builds.getPackage = bc.getPackages;
Builds.getProjectType = bc.getProjectType;
Builds.selectPackage = bc.selectPackage;

View File

@@ -27,42 +27,18 @@ fn_show_welcome() { (
echo ""
echo " $(t_attn 'Success')? Star it! $(t_url 'https://github.com/webinstall/webi-installers')"
echo " $(t_attn 'Problem')? Report it: $(t_url 'https://github.com/webinstall/webi-installers/issues')"
echo " $(t_dim "(your system is") $(t_host "$(fn_get_os)")/$(t_host "$(uname -m)") $(t_dim "with") $(t_host "$(fn_get_libc)") $(t_dim "&") $(t_host "$(fn_get_http_client_name)")$(t_dim ")")"
echo " $(t_dim "(your system is") $(t_host "$(uname -s)")/$(t_host "$(uname -m)") $(t_dim "with") $(t_host "$(fn_get_libc)") $(t_dim "&") $(t_host "$(fn_get_http_client_name)")$(t_dim ")")"
sleep 0.2
); }
fn_get_os() { (
# Ex:
# GNU/Linux
# Android
# Linux (often Alpine, musl)
# Darwin
b_os="$(uname -o 2> /dev/null || echo '')"
b_sys="$(uname -s)"
if test -z "${b_os}" || test "${b_os}" = "${b_sys}"; then
# ex: 'Darwin' (and plain, non-GNU 'Linux')
echo "${b_sys}"
return 0
fi
if echo "${b_os}" | grep -q "${b_sys}"; then
# ex: 'GNU/Linux'
echo "${b_os}"
return 0
fi
# ex: 'Android/Linux'
echo "${b_os}/${b_sys}"
); }
fn_get_libc() { (
# Ex:
# musl
# libc
if ldd /bin/ls 2> /dev/null | grep -q 'musl' 2> /dev/null; then
echo 'musl'
elif fn_get_os | grep -q 'GNU|Linux'; then
elif uname -o | grep -q 'GNU' || uname -s | grep -q 'Linux'; then
echo 'gnu'
else
echo 'libc'
@@ -150,14 +126,9 @@ fn_wget() { (
a_url="${1}"
a_path="${2}"
cmd_wget="wget -c -q --user-agent"
cmd_wget="wget -q --user-agent"
if fn_is_tty; then
cmd_wget="wget -c -q --show-progress --user-agent"
fi
# busybox wget doesn't support --show-progress
# See <https://github.com/webinstall/webi-installers/pull/772>
if readlink "$(command -v wget)" | grep -q busybox; then
cmd_wget="wget --user-agent"
cmd_wget="wget -q --show-progress --user-agent"
fi
b_triple_ua="$(fn_get_target_triple_user_agent)"
@@ -166,9 +137,9 @@ fn_wget() { (
b_agent="webi/wget+curl ${b_triple_ua}"
fi
if ! $cmd_wget "${b_agent}" "${a_url}" -O "${a_path}"; then
if ! $cmd_wget "${b_agent}" -c "${a_url}" -O "${a_path}"; then
echo >&2 " $(t_err "failed to download (wget)") '$(t_url "${a_url}")'"
echo >&2 " $cmd_wget '${b_agent}' '${a_url}' -O '${a_path}'"
echo >&2 " $cmd_wget '${b_agent}' -c '${a_url}' -O '${a_path}'"
echo >&2 " $(wget -V)"
return 1
fi
@@ -201,9 +172,9 @@ fn_curl() { (
fn_get_target_triple_user_agent() { (
# Ex:
# x86_64/unknown GNU/Linux/5.15.107-2-pve gnu
# x86_64/unknown Linux/5.15.107-2-pve gnu
# arm64/unknown Darwin/22.6.0 libc
echo "$(uname -m)/unknown $(fn_get_os)/$(uname -r) $(fn_get_libc)"
echo "$(uname -m)/unknown $(uname -s)/$(uname -r) $(fn_get_libc)"
); }
fn_download_to_path() { (
@@ -211,10 +182,10 @@ fn_download_to_path() { (
a_path="${2}"
mkdir -p "$(dirname "${a_path}")"
if command -v curl > /dev/null; then
fn_curl "${a_url}" "${a_path}.part"
elif command -v wget > /dev/null; then
if command -v wget > /dev/null; then
fn_wget "${a_url}" "${a_path}.part"
elif command -v curl > /dev/null; then
fn_curl "${a_url}" "${a_path}.part"
else
echo >&2 " $(t_err "failed to detect HTTP client (curl, wget)")"
return 1
@@ -254,6 +225,11 @@ webi_bootstrap() { (
echo " Updating $(t_path "${b_path_rel}")"
fi
b_download_dir="$(dirname "${a_path}")"
if ! test -w "${b_download_dir}"; then
echo " Creating $(t_path "${b_download_dir}/")"
mkdir -p "${b_download_dir}"
fi
echo " Downloading $(t_url "${b_webi_file_url}")"
echo " to $(t_path "${b_path_rel}")"
fn_download_to_path "${b_webi_file_url}" "${a_path}"

View File

@@ -1,338 +0,0 @@
#!/usr/bin/env node
'use strict';
let Fs = require('node:fs/promises');
let Path = require('node:path');
let BuildsCacher = require('./builds-cacher.js');
let HostTargets = require('./build-classifier/host-targets.js');
let Parallel = require('./parallel.js');
var INSTALLERS_DIR = Path.join(__dirname, '..');
var CACHE_DIR = Path.join(__dirname, '../_cache');
let UserAgentsMap = require('./build-classifier/uas.json');
let uas = Object.keys(UserAgentsMap);
let uaTargetsMap = {};
for (let ua of uas) {
let terms = ua.split(/[\s\/]+/g);
let target = {};
void HostTargets.termsToTarget(target, terms);
if (!target) {
continue;
}
if (target.errors.length) {
throw target.errors[0];
}
if (!target.os) {
// TODO make target null, or create error for this
console.warn(`no os for terms: ${terms}`);
//throw new Error(`terms: ${terms}`);
continue;
}
if (!target.arch) {
// TODO make target null, or create error for this
console.warn(`no arch for terms: ${terms}`);
//throw new Error(`terms: ${terms}`);
continue;
}
if (!target.libc) {
// TODO make target null, or create error for this
console.warn(`no libc for terms: ${terms}`);
//throw new Error(`terms: ${terms}`);
continue;
}
let triplet = `${target.os}-${target.arch}-${target.libc}`;
uaTargetsMap[triplet] = target;
}
let uaTargets = [];
let triplets = Object.keys(uaTargetsMap);
for (let triplet of triplets) {
let target = uaTargetsMap[triplet];
uaTargets.push(target);
}
function showDirs(dirs) {
{
let errors = Object.keys(dirs.errors);
console.error('');
console.error(`Errors: ${errors.length}`);
for (let name of errors) {
let err = dirs.errors[name];
console.error(`${name}/: ${err.message}`);
}
}
{
let hidden = Object.keys(dirs.hidden);
console.debug('');
console.debug(`Hidden: ${hidden.length}`);
for (let name of hidden) {
let kind = dirs.hidden[name];
if (kind === '!directory') {
console.debug(` ${name}`);
} else {
console.debug(` ${name}/`);
}
}
}
{
let alias = Object.keys(dirs.alias);
console.debug('');
console.debug(`Alias: ${alias.length}`);
for (let name of alias) {
let kind = dirs.alias[name];
if (kind === 'symlink') {
console.debug(` ${name} => ...`);
} else {
console.debug(` ${name}/`);
}
}
}
{
let invalids = Object.keys(dirs.invalid);
console.warn('');
console.warn(`Invalid: ${invalids.length}`);
for (let name of invalids) {
console.warn(` ${name}/`);
}
}
{
let selfhosted = Object.keys(dirs.selfhosted);
console.info('');
console.info(`Self-Hosted: ${selfhosted.length}`);
for (let name of selfhosted) {
console.info(` ${name}/`);
}
}
{
let valids = Object.keys(dirs.valid);
console.info('');
console.info(`Found: ${valids.length}`);
for (let name of valids) {
console.info(` ${name}/`);
}
}
}
let bc = BuildsCacher.create({
caches: CACHE_DIR,
installers: INSTALLERS_DIR,
});
async function main() {
/* jshint maxcomplexity: 25 */
// TODO
// node ./_webi/lint-builds.js caddy@beta 'x86_64/unknown Darwin libc'
//
//let [projName, userAgent] = process.argv.slice(2);
let projName = process.argv[2];
// create test case for zoxide, goreleaser, go, yq, caddy, rg
let dirs = await bc.getProjectsByType();
if (!projName) {
showDirs(dirs);
console.info('');
}
bc.freshenRandomPackage(600 * 1000);
let rows = [];
let triples = [];
let valids = Object.keys(dirs.valid);
if (projName) {
if (!valids.includes(projName)) {
throw new Error(`'${projName}' is not a valid installable project`);
}
valids = [projName];
}
//valids = ['atomicparsley', 'caddy', 'macos'];
//valids = ['atomicparsley'];
console.info('');
console.info(`Fetching project release assets`);
let parallel = 25;
let projects = [];
await Parallel.run(parallel, valids, getAll);
async function getAll(name, i) {
console.info(` ${name}`);
let projInfo = await bc.getPackages({
//Releases: Releases,
name: name,
date: new Date(),
});
projects[i] = projInfo;
}
console.info(`Classifying build assets for...`);
for (let projInfo of projects) {
console.info(` ${projInfo.name}`);
let nStr = projInfo.releases.length.toString();
let n = nStr.padStart(5, ' ');
let row = `##### ${n}\t${projInfo.name}\tv`;
rows.push(row);
// ignore known, non-package extensions
for (let build of projInfo.releases) {
let target = bc.classify(projInfo, build);
if (!target) {
// non-build file
continue;
}
if (target.error) {
let e = target.error;
if (e.code === 'E_BUILD_NO_PATTERN') {
console.warn(`>>> ${e.message} <<<`);
console.warn(projInfo);
console.warn(build);
console.warn(`^^^ ${e.message} ^^^`);
}
throw e;
}
if (target.unknownTerms?.length) {
let msg = `${projInfo.name}: unrecognized term(s) '${target.unknownTerms}' in '${build.download}'`;
let err = new Error(msg);
throw err;
}
triples.push(target.triplet);
// if (!build.version) {
// throw new Error(`no version for ${pkg.name} ${build.name}`);
// }
// // For debug printing versions
// console.error(build.version);
rows.push(`${target.triplet}\t${projInfo.name}\t${build.version}`);
}
}
console.info(`Fetching builds for`);
for (let projInfo of projects) {
console.info('');
console.info('');
console.info(` ${projInfo.name}`);
for (let target of uaTargets) {
let libc = target.libc || 'libc';
let hostTriplet = `${target.os}-${target.arch}-${libc}`;
console.info('');
console.info(` target: ${hostTriplet}`);
let match = bc.findMatchingPackages(projInfo, target, {
ver: '',
});
if (!match) {
console.info(
` project: ${projInfo.name}: missing build for os '${target.os}'`,
);
continue;
}
if (!match.releases) {
console.info(
` project: ${projInfo.name}: missing build for os '${target.os}-${target.arch}-${libc}'`,
);
} else if (match.triplet === hostTriplet) {
let releaseNames = Object.keys(match.releases);
console.info(` selected ${releaseNames.length}`);
} else {
let releaseNames = Object.keys(match.releases);
console.info(
` selected ${releaseNames.length} (${match.triplet} fallback)`,
);
}
}
}
let tsv = rows.join('\n');
console.info('');
console.info('#rows', rows.length);
await Fs.writeFile('builds.tsv', tsv, 'utf8');
console.info('');
console.info('Triplets Detected:');
let triplets = Object.keys(bc._triplets);
if (triplets.length) {
triplets.sort();
console.info(' ', triplets.join('\n '));
} else {
console.info(' (none)');
}
console.info('');
console.info('New / Unknown Terms:');
let unknowns = Object.keys(bc.unknownTerms);
if (unknowns.length) {
unknowns.sort();
console.warn(' ', unknowns.join('\n '));
} else {
console.info(' (none)');
}
console.info('');
console.info('Unused Terms:');
let unuseds = Object.keys(bc.orphanTerms);
if (unuseds.length) {
unuseds.sort();
console.warn(' ', unuseds.join('\n '));
} else {
console.info(' (none)');
}
console.info('');
console.info('Formats:');
if (bc.formats.length) {
let formats = bc.formats.slice();
formats.sort();
if (!formats[0]) {
formats[0] = '(bin)';
}
console.warn(' ', formats.join('\n '));
} else {
console.info(' (none)');
}
// sort -u -k1 builds.tsv | rg -v '^#|^https?:' | rg -i arm
// cut -f1 builds.tsv | sort -u -k1 | rg -v '^#|^https?:' | rg -i arm
}
if (module === require.main) {
let times = [];
let now = Date.now();
main()
.then(async function () {
let then = Date.now();
let delta = then - now;
times.push(delta);
now = then;
await main();
then = Date.now();
delta = then - now;
times.push(delta);
})
.then(function () {
console.info('');
console.info('Run times');
for (let delta of times) {
let s = delta / 1000;
console.info(` ${s}`);
}
function forceExit() {
console.warn(`warn: dangling event loop reference`);
process.exit(0);
}
let exitTimeout = setTimeout(forceExit, 250);
exitTimeout.unref();
})
.catch(function (err) {
console.error(err.stack || err);
process.exit(1);
});
}

View File

@@ -26,7 +26,6 @@ __bootstrap_webi() {
#PKG_ARCHES=
#PKG_LIBCS=
#PKG_FORMATS=
#PKG_LATEST=
WEBI_PKG_DOWNLOAD=""
WEBI_DOWNLOAD_DIR="${HOME}/Downloads"
if command -v xdg-user-dir > /dev/null; then
@@ -126,13 +125,8 @@ __bootstrap_webi() {
{
echo ""
echo " $(t_err "Error: no '${PKG_NAME:-"Unknown Package"}@${WEBI_TAG:-"Unknown Tag"}' release for '${WEBI_OS:-"Unknown OS"}' (${WEBI_LIBC:-"Unknown Libc"}) on '${WEBI_ARCH:-"Unknown CPU"}' as one of '${WEBI_FORMATS:-"Unknown File Type"}'")"
echo ""
echo " Latest Version: ${PKG_LATEST}"
echo " CPUs: $PKG_ARCHES"
echo " OSes: $PKG_OSES"
echo " libcs: $PKG_LIBCS"
echo " Package Formats: $PKG_FORMATS"
echo " (check that the package name and version are correct)"
echo " '$PKG_NAME' is available for '$PKG_OSES' ($PKG_LIBCS) on '$PKG_ARCHES' as one of '$PKG_FORMATS'"
echo " (check that the package name and version are correct)"
echo ""
my_release_url="$(echo "$WEBI_RELEASES" | sed 's:?.*::')"
@@ -264,8 +258,6 @@ __bootstrap_webi() {
)"
my_export="export PATH=\"$my_path_export:\$PATH\""
touch -a ~/.config/envman/PATH.env
if grep -q -F "${my_export}" ~/.config/envman/PATH.env; then
return 0
fi
@@ -284,6 +276,85 @@ __bootstrap_webi() {
fi
}
fn_envman_init() {
mkdir -p ~/.config/envman/
if ! test -e ~/.config/envman/PATH.env; then
touch ~/.config/envman/PATH.env
fi
if ! test -e ~/.config/envman/load.sh; then
# shellcheck disable=SC2016
{
echo '# Generated for envman. Do not edit.'
echo 'for x in ~/.config/envman/*.env; do'
echo ' my_basename="$(basename "${x}")"'
echo ' if [ "*.env" = "${my_basename}" ]; then'
echo ' continue'
echo ' fi'
echo ''
echo ' # shellcheck source=/dev/null'
echo ' . "${x}"'
echo 'done'
} > ~/.config/envman/load.sh
fi
if command -v sh > /dev/null; then
if test -e ~/.profile; then
if ! grep -q -F '/.config/envman/load.sh' ~/.profile; then
fn_echo_load_sh >> ~/.profile
fi
fi
fi
if command -v bash > /dev/null; then
if test -e ~/.bashrc; then
if ! grep -q -F '/.config/envman/load.sh' ~/.bashrc; then
fn_echo_load_sh >> ~/.bashrc
fi
fi
fi
if command -v zsh > /dev/null; then
if test -e ~/.zshrc; then
if ! grep -q -F '/.config/envman/load.sh' ~/.zshrc; then
fn_echo_load_sh >> ~/.zshrc
fi
fi
fi
if command -v fish > /dev/null; then
if test ! -e ~/.config/envman/load.fish; then
# shellcheck disable=SC2016
{
echo '# Generated for envman. Do not edit.'
echo 'for x in ~/.config/envman/*.env'
echo ' source "$x"'
echo 'end'
} > ~/.config/envman/load.fish
fi
mkdir -p ~/.config/fish
if test -e ~/.config/fish/config.fish; then
touch ~/.config/fish/config.fish
fi
if ! grep -q -F '/.config/envman/load.fish' ~/.config/fish/config.fish; then
fn_echo_load_fish >> ~/.config/fish/config.fish
fi
fi
}
fn_echo_load_fish() {
echo ''
echo '# Generated for envman. Do not edit.'
# shellcheck disable=SC2016
echo 'test -s "$HOME/.config/envman/load.fish"; and source "$HOME/.config/envman/load.fish"'
}
fn_echo_load_sh() {
echo ''
echo '# Generated for envman. Do not edit.'
# shellcheck disable=SC2016
echo '[ -s "$HOME/.config/envman/load.sh" ] && source "$HOME/.config/envman/load.sh"'
}
fn_is_defined_in_all_shells() {
my_path="${1}"
@@ -311,12 +382,10 @@ __bootstrap_webi() {
echo "${HOME}/.config/fish/config.fish"
)"
for my_conf in $my_confs; do
if ! test -e "${my_conf}"; then
continue
fi
if ! grep -q -F "${my_paths}" "${my_conf}"; then
return 1
if test -e "${my_conf}"; then
if ! grep -q -F "${my_paths}" "${my_conf}"; then
return 1
fi
fi
done
}
@@ -500,7 +569,7 @@ __bootstrap_webi() {
if [ -z "${_WEBI_CHILD-}" ] && [ -f "$_webi_tmp/.PATH.env" ]; then
if test -s "$_webi_tmp/.PATH.env"; then
# shellcheck disable=SC2088 # ~ should not expand here
echo " Edit $(t_path '~/.config/envman/PATH.env') to add:"
echo " Updated $(t_path '~/.config/envman/PATH.env') updated with:"
sort -u "$_webi_tmp/.PATH.env" | while read -r my_new_path; do
echo " $(t_path "${my_new_path}")"
done
@@ -508,11 +577,13 @@ __bootstrap_webi() {
rm -f "$_webi_tmp/.PATH.env"
echo ">>> $(t_info 'ACTION REQUIRED') <<<"
echo ">>> $(t_attn 'ACTION REQUIRED') <<<"
echo ""
echo " Copy, paste & run the following command:"
echo " $(t_attn 'source ~/.config/envman/PATH.env')"
echo " (newly opened terminal windows will update automatically)"
echo ""
echo "^^^ $(t_attn 'ACTION REQUIRED') ^^^"
fi
fi
@@ -539,42 +610,18 @@ fn_show_welcome_back() { (
echo ""
echo " $(t_attn 'Success')? Star it! $(t_url 'https://github.com/webinstall/webi-installers')"
echo " $(t_attn 'Problem')? Report it: $(t_url 'https://github.com/webinstall/webi-installers/issues')"
echo " $(t_dim "(your system is") $(t_host "$(fn_get_os)")/$(t_host "$(uname -m)") $(t_dim "with") $(t_host "$(fn_get_libc)") $(t_dim "&") $(t_host "$(fn_get_http_client_name)")$(t_dim ")")"
echo " $(t_dim "(your system is") $(t_host "$(uname -s)")/$(t_host "$(uname -m)") $(t_dim "with") $(t_host "$(fn_get_libc)") $(t_dim "&") $(t_host "$(fn_get_http_client_name)")$(t_dim ")")"
sleep 0.2
); }
fn_get_os() { (
# Ex:
# GNU/Linux
# Android
# Linux (often Alpine, musl)
# Darwin
b_os="$(uname -o 2> /dev/null || echo '')"
b_sys="$(uname -s)"
if test -z "${b_os}" || test "${b_os}" = "${b_sys}"; then
# ex: 'Darwin' (and plain, non-GNU 'Linux')
echo "${b_sys}"
return 0
fi
if echo "${b_os}" | grep -q "${b_sys}"; then
# ex: 'GNU/Linux'
echo "${b_os}"
return 0
fi
# ex: 'Android/Linux'
echo "${b_os}/${b_sys}"
); }
fn_get_libc() { (
# Ex:
# musl
# libc
if ldd /bin/ls 2> /dev/null | grep -q 'musl' 2> /dev/null; then
echo 'musl'
elif fn_get_os | grep -q 'GNU|Linux'; then
elif uname -o | grep -q 'GNU' || uname -s | grep -q 'Linux'; then
echo 'gnu'
else
echo 'libc'
@@ -662,14 +709,9 @@ fn_wget() { (
a_url="${1}"
a_path="${2}"
cmd_wget="wget -c -q --user-agent"
cmd_wget="wget -q --user-agent"
if fn_is_tty; then
cmd_wget="wget -c -q --show-progress --user-agent"
fi
# busybox wget doesn't support --show-progress
# See <https://github.com/webinstall/webi-installers/pull/772>
if readlink "$(command -v wget)" | grep -q busybox; then
cmd_wget="wget --user-agent"
cmd_wget="wget -q --show-progress --user-agent"
fi
b_triple_ua="$(fn_get_target_triple_user_agent)"
@@ -678,9 +720,9 @@ fn_wget() { (
b_agent="webi/wget+curl ${b_triple_ua}"
fi
if ! $cmd_wget "${b_agent}" "${a_url}" -O "${a_path}"; then
if ! $cmd_wget "${b_agent}" -c "${a_url}" -O "${a_path}"; then
echo >&2 " $(t_err "failed to download (wget)") '$(t_url "${a_url}")'"
echo >&2 " $cmd_wget '${b_agent}' '${a_url}' -O '${a_path}'"
echo >&2 " $cmd_wget '${b_agent}' -c '${a_url}' -O '${a_path}'"
echo >&2 " $(wget -V)"
return 1
fi
@@ -713,9 +755,9 @@ fn_curl() { (
fn_get_target_triple_user_agent() { (
# Ex:
# x86_64/unknown GNU/Linux/5.15.107-2-pve gnu
# x86_64/unknown Linux/5.15.107-2-pve gnu
# arm64/unknown Darwin/22.6.0 libc
echo "$(uname -m)/unknown $(fn_get_os)/$(uname -r) $(fn_get_libc)"
echo "$(uname -m)/unknown $(uname -s)/$(uname -r) $(fn_get_libc)"
); }
fn_download_to_path() { (
@@ -723,10 +765,10 @@ fn_download_to_path() { (
a_path="${2}"
mkdir -p "$(dirname "${a_path}")"
if command -v curl > /dev/null; then
fn_curl "${a_url}" "${a_path}.part"
elif command -v wget > /dev/null; then
if command -v wget > /dev/null; then
fn_wget "${a_url}" "${a_path}.part"
elif command -v curl > /dev/null; then
fn_curl "${a_url}" "${a_path}.part"
else
echo >&2 " $(t_err "failed to detect HTTP client (curl, wget)")"
return 1
@@ -763,6 +805,11 @@ webi_upgrade() { (
echo "$(t_task 'Updating') $(t_pkg 'Webi')"
fi
b_download_dir="$(dirname "${a_path}")"
if ! test -w "${b_download_dir}"; then
echo " Creating $(t_path "${b_download_dir}/")"
mkdir -p "${b_download_dir}"
fi
echo " Downloading $(t_url "${b_webi_file_url}")"
echo " to $(t_path "${b_path_rel}")"
fn_download_to_path "${b_webi_file_url}" "${a_path}"
@@ -845,141 +892,4 @@ main() { (
__bootstrap_webi
); }
##############################################
# #
# envman helper functions #
# #
##############################################
fn_envman_init() { (
if ! test -r ~/.config/envman/; then
echo " Initializing ~/.config/envman/"
mkdir -p ~/.config/envman/
fi
# Note: the variables $BASH, $ZSH_NAME, etc are always empty
# because the active shell is always sh when this script runs
fn_envman_init_load_sh
fn_envman_init_shell sh '.profile' '.ash_history'
fn_envman_init_shell bash '.bashrc' '.bash_history'
fn_envman_init_shell zsh '.zshrc' '.zsh_sessions'
if command -v fish > /dev/null; then
fn_envman_init_load_fish
fn_envman_init_fish
fi
); }
fn_envman_init_load_sh() { (
touch -a ~/.config/envman/load.sh
if grep -q -F 'ENVMAN_LOAD' ~/.config/envman/load.sh; then
return 0
fi
cat << LOAD_SH > ~/.config/envman/load.sh
# Generated for envman. Do not edit.
# shellcheck disable=SC1090
touch -a ~/.config/envman/PATH.env
touch -a ~/.config/envman/ENV.env
touch -a ~/.config/envman/alias.env
touch -a ~/.config/envman/function.sh
# ENV first because we may use it in PATH
test -z "\${ENVMAN_LOAD:-}" && . ~/.config/envman/ENV.env
test -z "\${ENVMAN_LOAD:-}" && . ~/.config/envman/PATH.env
export ENVMAN_LOAD='loaded'
# function first because we may use it in alias
test -z "\${g_envman_load_sh:-}" && . ~/.config/envman/function.sh
test -z "\${g_envman_load_sh:-}" && . ~/.config/envman/alias.env
g_envman_load_sh='loaded'
LOAD_SH
); }
fn_envman_init_shell() { (
a_shell="${1}"
a_rc="${2}"
a_history="${3:-_history_file_doesnt_exist}"
a_login_shell="$(basename "${SHELL:-}")"
if ! command -v "${a_shell}" > /dev/null; then
return 0
fi
# .bashrc and .zshrc no longer exist by default on macOS
if ! test -e ~/"${a_rc}" && ! test -e ~/"${a_history}"; then
if test "${a_login_shell}" != "${a_shell}"; then
return 0
fi
fi
touch -a ~/"${a_rc}"
if grep -q -F '/.config/envman/load.sh' ~/"${a_rc}"; then
return 0
fi
# shellcheck disable=SC2088 # ~ should not expand here
echo >&2 " Edit $(t_path "~/${a_rc}") to $(t_cmd "source ~/.config/envman/load.sh")"
{
echo ''
echo '# Generated for envman. Do not edit.'
#shellcheck disable=SC2016 # vars should not expand here
echo '[ -s "$HOME/.config/envman/load.sh" ] && source "$HOME/.config/envman/load.sh"'
} >> ~/"${a_rc}"
); }
fn_envman_init_load_fish() { (
mkdir -p ~/.config/envman/
touch -a ~/.config/envman/load.fish
if grep -q -F 'ENVMAN_LOAD' ~/.config/envman/load.fish; then
return 0
fi
echo >&2 " Create ~/.config/envman/load.fish"
cat << EOF > ~/.config/envman/load.fish
# Generated for envman. Do not edit.
touch -a ~/.config/envman/PATH.env
touch -a ~/.config/envman/ENV.env
touch -a ~/.config/envman/alias.env
touch -a ~/.config/envman/function.fish
not set -q ENVMAN_LOAD; and source ~/.config/envman/ENV.env
not set -q ENVMAN_LOAD; and source ~/.config/envman/PATH.env
set -x ENVMAN_LOAD 'loaded'
not set -q g_envman_load_fish; and source ~/.config/envman/function.fish
not set -q g_envman_load_fish; and source ~/.config/envman/alias.env
set -g g_envman_load_fish 'loaded'
EOF
); }
fn_envman_init_fish() {
mkdir -p ~/.config/fish/
touch -a ~/.config/fish/config.fish
if grep -q -F '/.config/envman/load.fish' ~/.config/fish/config.fish; then
return 0
fi
# shellcheck disable=SC2088 # ~ should not expand here
echo >&2 " Edit $(t_path "~/.config/fish/config.fish") to $(t_cmd "source ~/.config/envman/load.fish")"
cat << EOF >> ~/.config/fish/config.fish
# Generated for envman. Do not edit.
test -s ~/.config/envman/load.fish; and source ~/.config/envman/load.fish
EOF
}
main

View File

@@ -1 +0,0 @@
projects.js

96
_webi/packages.js Normal file
View File

@@ -0,0 +1,96 @@
'use strict';
var frontmarker = require('./frontmarker.js');
var fs = require('fs');
var path = require('path');
var pkgs = module.exports;
pkgs.create = function (Pkgs, basepath) {
if (!Pkgs) {
Pkgs = {};
}
if (!basepath) {
basepath = path.join(__dirname, '../');
}
Pkgs.all = function () {
return fs.promises.readdir(basepath).then(function (nodes) {
var items = [];
return nodes
.reduce(function (p, node) {
return p.then(function () {
return pkgs.get(node).then(function (meta) {
if (meta && '_' !== node[0]) {
meta.name = node;
items.push(meta);
}
});
});
}, Promise.resolve())
.then(function () {
return items;
});
});
};
Pkgs.get = function (node) {
return fs.promises.access(path.join(basepath, node)).then(function () {
return Pkgs._get(node);
});
};
Pkgs._get = function (node) {
var curlbash = path.join(basepath, node, 'install.sh');
var readme = path.join(basepath, node, 'README.md');
var winstall = path.join(basepath, node, 'install.ps1');
return Promise.all([
fs.promises
.readFile(readme, 'utf-8')
.then(function (txt) {
// TODO
return frontmarker.parse(txt);
})
.catch(function (e) {
if ('ENOENT' !== e.code && 'ENOTDIR' !== e.code) {
console.error("failed to read '" + node + "/README.md'");
console.error(e);
}
}),
fs.promises.access(curlbash).catch(function (e) {
// no *nix installer
curlbash = '';
if ('ENOENT' !== e.code && 'ENOTDIR' !== e.code) {
console.error("failed to parse '" + node + "/install.sh'");
console.error(e);
}
}),
fs.promises.access(winstall).catch(function (e) {
// no winstaller
winstall = '';
if ('ENOENT' !== e.code && 'ENOTDIR' !== e.code) {
console.error("failed to read '" + node + "/install.ps1'");
console.error(e);
}
}),
]).then(function (items) {
var meta = items[0] || items[1];
if (!meta) {
// doesn't exist
return;
}
meta.windows = !!winstall;
meta.bash = !!curlbash;
return meta;
});
};
return Pkgs;
};
pkgs.create(pkgs);
if (module === require.main) {
pkgs.all().then(function (data) {
console.info('package info:');
console.info(data);
});
}

View File

@@ -1,44 +0,0 @@
'use strict';
var Parallel = module.exports;
Parallel.run = async function (limit, arr, fn) {
let index = 0;
let actives = [];
let results = [];
limit = Math.min(limit, arr.length);
function launch() {
let _index = index;
let p = fn(arr[_index], _index, arr);
// some tasks may be synchronous
// so we must push before removing
actives.push(p);
p.then(function _resolve(result) {
let i = actives.indexOf(p);
actives.splice(i, 1);
results[_index] = result;
});
index += 1;
}
// start tasks in parallel, up to limit
for (; actives.length < limit; ) {
launch();
}
// keep the task queue full
for (; index < arr.length; ) {
// wait for one task to complete
await Promise.race(actives);
// add one task again
launch();
}
// wait for all remaining tasks
await Promise.all(actives);
return results;
};

View File

@@ -1,96 +0,0 @@
'use strict';
var frontmarker = require('./frontmarker.js');
var fs = require('fs');
var path = require('path');
var pkgs = module.exports;
pkgs.create = function (Pkgs, basepath) {
if (!Pkgs) {
Pkgs = {};
}
if (!basepath) {
basepath = path.join(__dirname, '../');
}
Pkgs.all = function () {
return fs.promises.readdir(basepath).then(function (nodes) {
var items = [];
return nodes
.reduce(function (p, node) {
return p.then(function () {
return pkgs.get(node).then(function (meta) {
if (meta && '_' !== node[0]) {
meta.name = node;
items.push(meta);
}
});
});
}, Promise.resolve())
.then(function () {
return items;
});
});
};
Pkgs.get = function (node) {
return fs.promises.access(path.join(basepath, node)).then(function () {
return Pkgs._get(node);
});
};
Pkgs._get = function (node) {
var curlbash = path.join(basepath, node, 'install.sh');
var readme = path.join(basepath, node, 'README.md');
var winstall = path.join(basepath, node, 'install.ps1');
return Promise.all([
fs.promises
.readFile(readme, 'utf-8')
.then(function (txt) {
// TODO
return frontmarker.parse(txt);
})
.catch(function (e) {
if ('ENOENT' !== e.code && 'ENOTDIR' !== e.code) {
console.error("failed to read '" + node + "/README.md'");
console.error(e);
}
}),
fs.promises.access(curlbash).catch(function (e) {
// no *nix installer
curlbash = '';
if ('ENOENT' !== e.code && 'ENOTDIR' !== e.code) {
console.error("failed to parse '" + node + "/install.sh'");
console.error(e);
}
}),
fs.promises.access(winstall).catch(function (e) {
// no winstaller
winstall = '';
if ('ENOENT' !== e.code && 'ENOTDIR' !== e.code) {
console.error("failed to read '" + node + "/install.ps1'");
console.error(e);
}
}),
]).then(function (items) {
var meta = items[0] || items[1];
if (!meta) {
// doesn't exist
return;
}
meta.windows = !!winstall;
meta.bash = !!curlbash;
return meta;
});
};
return Pkgs;
};
pkgs.create(pkgs);
if (module === require.main) {
pkgs.all().then(function (data) {
console.info('package info:');
console.info(data);
});
}

View File

@@ -1,22 +1,39 @@
'use strict';
var Installers = module.exports;
var Crypto = require('crypto');
var Fs = require('node:fs/promises');
var path = require('node:path');
var request = require('@root/request');
var _normalize = require('../_webi/normalize.js');
var reInstallTpl = /\s*#?\s*{{ installer }}/;
var Releases = module.exports;
Releases.get = async function (pkgdir) {
let get;
try {
get = require(path.join(pkgdir, 'releases.js'));
} catch (e) {
let err = new Error('no releases.js for', pkgdir.split(/[\/\\]+/).pop());
err.code = 'E_NO_RELEASE';
throw err;
}
let all = await get(request);
return _normalize(all);
};
function padScript(txt) {
return txt.replace(/^/g, ' ');
}
var BAD_SH_RE = /[<>'"`$\\]/;
Installers.renderBash = async function (
Releases.renderBash = async function (
pkgdir,
rel,
{ baseurl, pkg, tag, ver, os = '', arch = '', libc = '', formats, latest },
{ baseurl, pkg, tag, ver, os = '', arch = '', libc = '', formats },
) {
if (!Array.isArray(formats)) {
formats = [];
@@ -73,7 +90,7 @@ Installers.renderBash = async function (
.join(',')
.replace(/'/g, '');
let webiChecksum = await Installers.getWebiShChecksum();
let webiChecksum = await Releases.getWebiShChecksum();
let envReplacements = [
['WEBI_CHECKSUM', webiChecksum],
['WEBI_PKG', webiPkg],
@@ -82,7 +99,7 @@ Installers.renderBash = async function (
['WEBI_ARCH', arch],
['WEBI_LIBC', libc],
['WEBI_TAG', tag],
['WEBI_RELEASES', `${baseurl}${releaseUrl}`],
['WEBI_RELEASES', `${baseurl}/${releaseUrl}`],
['WEBI_CSV', releaseCsv],
['WEBI_VERSION', rel.version],
['WEBI_MAJOR', v.major],
@@ -99,11 +116,10 @@ Installers.renderBash = async function (
['WEBI_PKG_PATHNAME', pkgFile],
['WEBI_PKG_FILE', pkgFile], // TODO replace with pathname
['PKG_NAME', pkg],
['PKG_OSES', (rel.oses || []).join(' ')],
['PKG_ARCHES', (rel.arches || []).join(' ')],
['PKG_LIBCS', (rel.libcs || []).join(' ')],
['PKG_FORMATS', (rel.formats || []).join(' ')],
['PKG_LATEST', latest],
['PKG_OSES', rel.oses],
['PKG_ARCHES', rel.arches],
['PKG_LIBCS', rel.libcs],
['PKG_FORMATS', (rel.formats || []).join(',')],
];
for (let env of envReplacements) {
@@ -131,7 +147,7 @@ Installers.renderBash = async function (
return tplTxt;
};
Installers.renderPowerShell = async function (
Releases.renderPowerShell = async function (
pkgdir,
rel,
{ baseurl, pkg, tag, ver, os, arch, libc = '', formats },
@@ -208,7 +224,7 @@ var _webiShMeta = {
checksum: '',
mtime: 0,
};
Installers.getWebiShChecksum = async function () {
Releases.getWebiShChecksum = async function () {
let now = Date.now();
let ago = now - _webiShMeta.updated_at;
if (ago <= _webiShMeta.stale) {

View File

@@ -1,16 +1,19 @@
'use strict';
var InstallerServer = module.exports;
var Installers = module.exports;
let Fs = require('fs/promises');
let Path = require('path');
var Fs = require('fs/promises');
var path = require('path');
let HostTargets = require('./build-classifier/host-targets.js');
let Builds = require('./builds.js');
let Installers = require('./installers.js');
var uaDetect = require('./ua-detect.js');
var packages = require('./packages.js');
var Releases = require('./releases.js');
InstallerServer.INSTALLERS_DIR = Path.join(__dirname, '..');
InstallerServer.serveInstaller = async function (
// handlers caching and transformation, probably should be broken down
var getReleases = require('./transform-releases.js');
Installers.INSTALLERS_DIR = path.join(__dirname, '..');
Installers.serveInstaller = async function (
baseurl,
ua,
pkg,
@@ -19,205 +22,120 @@ InstallerServer.serveInstaller = async function (
formats,
libc,
) {
let unameAgent = ua;
let projectName = pkg;
let [rel, tmplParams] = await InstallerServer.helper({
unameAgent,
projectName,
let [rel, opts] = await Installers.helper({
ua,
pkg,
tag,
formats,
libc,
});
Object.assign(tmplParams, {
Object.assign(opts, {
baseurl,
});
var pkgdir = Path.join(InstallerServer.INSTALLERS_DIR, projectName);
var pkgdir = path.join(Installers.INSTALLERS_DIR, pkg);
if ('ps1' === ext) {
return Installers.renderPowerShell(pkgdir, rel, tmplParams);
return Releases.renderPowerShell(pkgdir, rel, opts);
}
return Installers.renderBash(pkgdir, rel, tmplParams);
return Releases.renderBash(pkgdir, rel, opts);
};
Installers.helper = async function ({ ua, pkg, tag, formats, libc }) {
// TODO put some of this in a middleware? or common function?
// TODO put some of this in a middleware? or common function?
// TODO maybe move package/version/lts/channel detection into getReleases
InstallerServer.helper = async function ({
unameAgent,
projectName,
tag,
formats,
libc,
}) {
console.log(`dbg: Installer User-Agent: ${unameAgent}`);
// TODO maybe move package/version/lts/channel detection into getReleases
var ver = tag.replace(/^v/, '');
var lts;
var channel;
let releaseTarget = toReleaseTarget(tag);
let hostFormats = formats;
let terms = unameAgent.split(/[\s\/]+/g);
let hostTarget = {};
try {
void HostTargets.termsToTarget(hostTarget, terms);
} catch (e) {
// if we can't guarantee the results...
// "in the face of ambiguity, refuse the temptation to guess"
throw e;
}
console.log(`dbg: Installer Host Target:`);
console.log(hostTarget);
if (!hostTarget.os) {
throw new Error(`OS could not be identified by User-Agent '${unameAgent}'`);
switch (ver) {
case 'latest':
ver = '';
channel = 'stable';
break;
case 'lts':
lts = true;
channel = 'stable';
ver = '';
break;
case 'stable':
channel = 'stable';
ver = '';
break;
case 'beta':
channel = 'beta';
ver = '';
break;
case 'dev':
channel = 'dev';
ver = '';
break;
}
console.log(`dbg: Get Project Installer Type for '${projectName}':`);
let proj = await Builds.getProjectType(projectName);
console.log(proj);
let validTypes = ['alias', 'selfhosted', 'valid'];
if (!validTypes.includes(proj.type)) {
let msg = `'${projectName}' doesn't have an installer: '${proj.type}': '${proj.detail}'`;
let err = new Error(msg);
err.code = 'ENOENT';
throw err;
var myOs = uaDetect.os(ua);
var myArch = uaDetect.arch(ua);
var myLibc;
if (libc) {
myLibc = uaDetect.libc(libc);
}
if (proj.type === 'alias') {
projectName = proj.detail;
if (!myLibc) {
myLibc = uaDetect.libc(ua);
}
if (!myLibc) {
myLibc = 'libc';
}
let tmplParams = {
pkg: projectName,
tag: tag,
os: hostTarget.os,
arch: hostTarget.arch,
libc: hostTarget.libc,
formats: hostFormats,
let cfg = await packages.get(pkg);
let releaseQuery = {
pkg: cfg.alias || pkg,
ver,
os: myOs,
arch: myArch,
libc: myLibc,
lts,
channel,
// TODO use formats for sorting, not exclusion
// (it's better to install xz or report an error to install zip)
formats,
limit: 1,
};
Object.assign(tmplParams, releaseTarget);
console.log('tmplParams', tmplParams);
let errPackage = {
name: 'doesntexist.ext',
version: '0.0.0',
lts: '-',
channel: 'error',
date: '1970-01-01',
os: hostTarget.os || '-',
arch: hostTarget.arch || '-',
libc: hostTarget.libc || '-',
ext: 'err',
download: 'https://example.com/doesntexist.ext',
comment:
'No matches found. Could be bad or missing version info' +
',' +
"Check query parameters. Should be something like '/api/releases/{package}@{version}.tab?os={macos|linux|windows|-}&arch={amd64|x86|aarch64|arm64|armv7l|-}&libc={musl|gnu|msvc|libc|static}&limit=10'",
let rels = await getReleases(releaseQuery);
var rel = rels.releases[0];
var opts = {
pkg: cfg.alias || pkg,
ver,
tag,
os: myOs,
arch: myArch,
libc: myLibc,
lts,
channel,
formats,
limit: 1,
};
if (proj.type === 'selfhosted') {
return [errPackage, tmplParams];
}
let projInfo = await Builds.getPackage({
name: projectName,
date: new Date(),
});
let latest = projInfo.versions[0];
Object.assign(tmplParams, { latest });
//console.log('projInfo', projInfo);
let buildTargetInfo = {
triplets: projInfo.triplets,
oses: projInfo.oses,
arches: projInfo.arches,
libcs: projInfo.libcs,
formats: projInfo.formats,
};
let hasOs = projInfo.oses.includes(hostTarget.os);
if (!hasOs) {
hasOs = projInfo.oses.includes('ANYOS');
}
if (!hasOs) {
let pkg1 = Object.assign(buildTargetInfo, errPackage);
return [pkg1, tmplParams];
}
let targetRelease = Builds.findMatchingPackages(
projInfo,
hostTarget,
releaseTarget,
rel = Object.assign(
{
oses: rels.oses,
arches: rels.arches,
libcs: rels.libcs,
formats: rels.formats,
},
rel,
);
// { triplet: `${os}-${arch}-${libc}`, packages: targetPackages
// , latest: projInfo.versions[0], versions: matchInfo
// }
if (!targetRelease?.packages) {
let pkg1 = Object.assign(buildTargetInfo, errPackage);
return [pkg1, tmplParams];
}
let buildPkg = Builds.selectPackage(targetRelease.packages, hostFormats);
let ext = buildPkg.ext || '.exe';
if (ext.startsWith('.')) {
ext = ext.slice(1);
}
let version = targetRelease.version;
if (version.startsWith('v')) {
version = version.slice(1);
}
buildPkg = Object.assign(buildTargetInfo, buildPkg, { ext, version });
console.log('dbg: buildPkg', buildPkg);
console.log('dbg: tmplParams', tmplParams);
return [buildPkg, tmplParams];
return [rel, opts];
};
let channelNames = [
'stable',
// 'hotfix',
'latest',
'rc',
'preview',
'pre',
'dev',
'beta',
'alpha',
];
function toReleaseTarget(tag) {
tag = tag.replace(/^v/, '');
let releaseTarget = {
channel: '',
lts: false,
version: '',
};
if (tag === 'lts') {
releaseTarget.lts = true;
releaseTarget.channel = 'stable';
} else if (channelNames.includes(tag)) {
releaseTarget.channel = tag;
} else {
releaseTarget.version = tag;
}
return releaseTarget;
}
var CURL_PIPE_PS1_BOOT = Path.join(__dirname, 'curl-pipe-bootstrap.tpl.ps1');
var CURL_PIPE_SH_BOOT = Path.join(__dirname, 'curl-pipe-bootstrap.tpl.sh');
var CURL_PIPE_PS1_BOOT = path.join(__dirname, 'curl-pipe-bootstrap.tpl.ps1');
var CURL_PIPE_SH_BOOT = path.join(__dirname, 'curl-pipe-bootstrap.tpl.sh');
var BAD_SH_RE = /[<>'"`$\\]/;
InstallerServer.getPosixCurlPipeBootstrap = async function ({
baseurl,
pkg,
ver,
}) {
Installers.getPosixCurlPipeBootstrap = async function ({ baseurl, pkg, ver }) {
let bootTxt = await Fs.readFile(CURL_PIPE_SH_BOOT, 'utf8');
var webiPkg = [pkg, ver].filter(Boolean).join('@');
var webiChecksum = await Installers.getWebiShChecksum();
var webiChecksum = await Releases.getWebiShChecksum();
var envReplacements = [
['WEBI_PKG', webiPkg],
['WEBI_HOST', baseurl],
@@ -228,6 +146,7 @@ InstallerServer.getPosixCurlPipeBootstrap = async function ({
let name = env[0];
let value = env[1];
// TODO create REs once, in higher scope
let envRe = new RegExp(
`^[ \\t]*#?[ \\t]*(export[ \\t])?[ \\t]*(${name})=.*`,
'm',
@@ -245,7 +164,7 @@ InstallerServer.getPosixCurlPipeBootstrap = async function ({
return bootTxt;
};
InstallerServer.getPwshCurlPipeBootstrap = async function ({
Installers.getPwshCurlPipeBootstrap = async function ({
baseurl,
pkg,
ver,
@@ -254,7 +173,7 @@ InstallerServer.getPwshCurlPipeBootstrap = async function ({
let bootTxt = await Fs.readFile(CURL_PIPE_PS1_BOOT, 'utf8');
var webiPkg = [pkg, ver].filter(Boolean).join('@');
//var webiChecksum = await InstallerServer.getWebiPs1Checksum();
//var webiChecksum = await Releases.getWebiPs1Checksum();
var envReplacements = [
['Env:WEBI_PKG', webiPkg],
['Env:WEBI_HOST', baseurl],
@@ -275,6 +194,9 @@ InstallerServer.getPwshCurlPipeBootstrap = async function ({
let tplRe = new RegExp(`{{ (${name}) }}`, 'g');
bootTxt = bootTxt.replace(tplRe, `${value}`);
// let envRe = new RegExp(`^[ \\t]*#?[ \\t]*($$${name})[ \\t]*=.*`, 'im');
// bootTxt = bootTxt.replace(envRe, `$$${name} = '${value}'`);
let setRe = new RegExp(
`(#[ \\t]*)?(\\$${name})[ \\t]*=[ \\t]['"].*['"][ \\t]`,
'im',

View File

@@ -32,8 +32,7 @@ if (/\b-?-h(elp)?\b/.test(process.argv.join(' '))) {
var os = require('os');
var fs = require('fs');
var path = require('path');
var Builds = require('./builds.js');
var Installers = require('./installers.js');
var Releases = require('./releases.js');
var ServeInstaller = require('./serve-installer.js');
var pkg = process.argv[2].split('@');
@@ -65,8 +64,7 @@ console.info('Has the necessary files?');
});
console.info('');
let projName = pkgdir.split('/').filter(Boolean).pop();
Builds.getPackage({ name: projName }).then(async function (/*projInfo*/) {
Releases.get(path.join(process.cwd(), pkgdir)).then(async function (all) {
var pkgname = path.basename(pkgdir.replace(/\/$/, ''));
var nodeOs = os.platform();
var nodeOsRelease = os.release();
@@ -83,8 +81,8 @@ Builds.getPackage({ name: projName }).then(async function (/*projInfo*/) {
var formats = ['exe', 'xz', 'tar', 'zip', 'git'];
let [rel, opts] = await ServeInstaller.helper({
unameAgent: `${nodeOs}/${nodeOsRelease} ${nodeArch}/unknown ${nodeLibc}`,
projectName: pkgname,
ua: `${nodeOs}/${nodeOsRelease} ${nodeArch}/unknown ${nodeLibc}`,
pkg: pkgname,
tag: pkgtag || '',
formats: formats,
libc: nodeLibc,
@@ -118,8 +116,8 @@ Builds.getPackage({ name: projName }).then(async function (/*projInfo*/) {
console.info('');
return Promise.all([
Installers.renderBash(pkgdir, rel, opts).catch(function () {}),
Installers.renderPowerShell(pkgdir, rel, opts).catch(function () {}),
Releases.renderBash(pkgdir, rel, opts).catch(function () {}),
Releases.renderPowerShell(pkgdir, rel, opts).catch(function () {}),
]).then(function (scripts) {
var bashTxt = scripts[0];
var ps1Txt = scripts[1];

View File

@@ -1,10 +1,7 @@
'use strict';
var Releases = module.exports;
var path = require('path');
var request = require('@root/request');
var _normalize = require('./normalize.js');
var Releases = require('./releases.js');
var cache = {};
//var staleAge = 5 * 1000;
@@ -14,21 +11,6 @@ var expiredAge = 15 * 60 * 1000;
let installerDir = path.join(__dirname, '..');
Releases.get = async function (pkgdir) {
let get;
try {
get = require(path.join(pkgdir, 'releases.js'));
} catch (e) {
let err = new Error('no releases.js for', pkgdir.split(/[\/\\]+/).pop());
err.code = 'E_NO_RELEASE';
throw err;
}
let all = await get(request);
return _normalize(all);
};
// TODO needs a proper test, and more accurate (though perhaps far less simple) code
function createFormatsSorter(formats) {
return function sortByVerExt(a, b) {
@@ -140,7 +122,7 @@ async function getCachedReleases(pkg) {
cache[pkg].all = all;
complete = true;
}),
sleep(15000).then(function () {
sleep(5000).then(function () {
if (complete) {
return;
}
@@ -260,7 +242,7 @@ async function filterReleases(
return sortedRels.slice(0, limit || 1000);
}
Releases.getReleases = function ({
module.exports = function getReleases({
_count,
pkg,
ver,
@@ -308,7 +290,7 @@ Releases.getReleases = function ({
if (_count < 1) {
// Apple Silicon M1 hacky-do workaround fix
if ('macos' === os && 'arm64' === arch) {
return Releases.getReleases({
return getReleases({
pkg,
ver,
os,
@@ -322,7 +304,7 @@ Releases.getReleases = function ({
}
// Windows ARM hacky-do workaround fix
if ('windows' === os && 'arm64' === arch) {
return Releases.getReleases({
return getReleases({
pkg,
ver,
os,
@@ -336,7 +318,7 @@ Releases.getReleases = function ({
}
// Raspberry Pi 3+ on Ubuntu arm64 (via Bionic?)
if ('linux' === os && 'arm64' === arch) {
return Releases.getReleases({
return getReleases({
_count: _count + 1,
pkg,
ver,
@@ -351,7 +333,7 @@ Releases.getReleases = function ({
}
// armv7 can run armv6
if ('linux' === os && 'armv7l' === arch) {
return Releases.getReleases({
return getReleases({
_count: _count + 1,
pkg,
ver,
@@ -369,7 +351,7 @@ Releases.getReleases = function ({
// Raspberry Pi 3+ on Raspbian arm7 (not Ubuntu arm64)
// hail mary
if ('linux' === os && 'armv7l' === arch) {
return Releases.getReleases({
return getReleases({
_count: _count + 1,
pkg,
ver,

View File

@@ -66,10 +66,10 @@ module.exports = async function (request) {
json: true,
});
let osBaseUrl = resp.body.base_url;
let osReleases = resp.body.releases;
let body = resp.body;
all.download = `${body.base_url}/{{ download }}`;
for (let asset of osReleases) {
for (let asset of body.releases) {
if (!channelMap[asset.channel]) {
channelMap[asset.channel] = true;
}
@@ -81,8 +81,7 @@ module.exports = async function (request) {
channel: asset.channel,
date: asset.release_date.replace(/T.*/, ''),
//sha256: asset.sha256,
download: `${osBaseUrl}/${asset.archive}`,
_filename: asset.archive,
download: asset.archive,
});
}
}

View File

@@ -44,7 +44,7 @@ function getAllReleases(request) {
var goReleases = resp.body;
var all = {
releases: [],
download: '',
download: 'https://dl.google.com/go/{{ download }}',
};
goReleases.forEach((release) => {
@@ -77,7 +77,7 @@ function getAllReleases(request) {
arch: arch,
ext: '', // let normalize run the split/test/join
hash: '-', // not ready to standardize this yet
download: `https://dl.google.com/go/${filename}`,
download: filename,
});
});
});

View File

@@ -6,7 +6,6 @@ var repo = 'goreleaser';
module.exports = function (request) {
return github(request, owner, repo).then(function (all) {
all._names = ['goreleaser', '1'];
return all;
});
};

View File

@@ -7,25 +7,19 @@ var repo = 'ollama';
module.exports = async function (request) {
let all = await github(request, owner, repo);
// TODO why are the 0.0.x releases sorting so high?
let releases = [];
for (let rel of all.releases) {
// this is a janky, sudo-wantin' .app
let isJank = rel.name.startsWith('Ollama-darwin');
if (isJank) {
let isLow = rel.version.startsWith('v0.0.');
if (isLow) {
continue;
}
let isUniversal = rel.name === 'ollama-darwin';
if (isUniversal) {
let x64 = Object.assign({ arch: 'x86_64' }, rel);
releases.push(x64);
rel.arch = 'aarch64';
}
releases.push(rel);
}
all.releases = releases;
all._names = ['Ollama', 'ollama'];
return all;
};

4
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "@webinstall/webi-installers",
"version": "1.1.1",
"version": "1.0.1",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@webinstall/webi-installers",
"version": "1.1.1",
"version": "1.0.1",
"license": "MPL-2.0",
"dependencies": {
"@root/request": "^1.9.2",

View File

@@ -1,11 +1,10 @@
{
"name": "@webinstall/webi-installers",
"private": true,
"version": "1.1.1",
"version": "1.0.1",
"description": "webinstall script repository",
"main": "_webi/",
"scripts": {
"bump": "npm version -m \"chore(release): bump to v%s\"",
"fmt": "npm run prettier && npm run shfmt && npm run pwsh-fmt",
"lint": "npm run shellcheck && npm run jshint && npm run pwsh-lint",
"prepare": "npm run tooling-init && npm run git-hooks-init",

View File

@@ -4,38 +4,16 @@ var github = require('../_common/github.js');
var owner = 'powershell';
var repo = 'powershell';
let ODDITIES = ['-fxdependent'];
function isOdd(build) {
for (let oddity of ODDITIES) {
let isOddity = build.name.includes(oddity);
if (isOddity) {
return true;
}
}
}
module.exports = function (request) {
return github(request, owner, repo).then(function (all) {
// remove checksums and .deb
all.releases = all.releases.filter(function (rel) {
let odd = isOdd(rel);
if (odd) {
return false;
}
let isPreview = rel.name.includes('-preview.');
if (isPreview) {
rel.channel = 'beta';
}
let isMusl = rel.download.match(/(\b|_)(musl|alpine)(\b|_)/i);
if (isMusl) {
// not a fully static build, not gnu-compatible
rel.libc = 'musl';
}
return true;
return !/(alpine)|(fxdependent)|(\.deb)|(\.pkg)|(\.rpm)$/i.test(rel.name);
});
all._names = ['PowerShell', 'powershell'];

View File

@@ -20,22 +20,10 @@ __init_sd() {
# pkg_install must be defined by every package
pkg_install() {
# ~/.local/opt/sd-v0.99.9/bin
mkdir -p "$(dirname "$pkg_src_cmd")"
# mv ./sd-*/sd "$pkg_src_cmd"
if test -f sd-*; then
# ~/.local/opt/sd-v0.99.9/bin
mkdir -p "$(dirname "$pkg_src_cmd")"
mv sd-* "$pkg_src_cmd"
elif test -f sd-*/sd; then
# ~/.local/opt/sd-v0.99.9/bin
mkdir -p "$(dirname "$pkg_src_cmd")"
mv sd-*/sd "$pkg_src_cmd"
if test -f sd-*/sd.1; then
mkdir -p "$pkg_src_dir/share/man/man1"
mv sd-*/sd.1 "$pkg_src_dir/share/man/man1"
fi
elif test -d sd-*/bin; then
mv sd-* "$pkg_src_dir"
fi
mv sd-* "$pkg_src_cmd"
}
# pkg_get_current_version is recommended, but (soon) not required

View File

@@ -6,6 +6,27 @@ var repo = 'shellcheck';
module.exports = function (request) {
return github(request, owner, repo).then(function (all) {
all.releases = all.releases.filter(function (rel) {
// don't include meta versions as actual versions
if (
['latest', 'stable'].includes(rel.version) ||
'v' !== rel.version[0]
) {
return false;
}
return true;
});
all.releases.forEach(function (rel) {
// if there is no os or arch or source designation, and it's a .zip, it's Windows amd64
if (
!/(darwin|mac|linux|x86_64|arm|src|source)/i.test(rel.name) &&
/\.zip$/.test(rel.name)
) {
rel.os = 'windows';
rel.arch = 'amd64';
}
});
return all;
});
};

View File

@@ -6,13 +6,6 @@ var repo = 'watchexec';
module.exports = function (request) {
return github(request, owner, repo).then(function (all) {
let builds = [];
for (let build of all.releases) {
build.version = build.version.replace(/^cli-/, '');
builds.push(build);
}
all.releases = builds;
return all;
});
};

View File

@@ -1,117 +0,0 @@
---
title: Webi Essentials
homepage: https://webinstall.dev/webi-essentials
tagline: |
The Webi Essentials are curl, git, tar, wget, xz, zip, & webi shell integrations
---
## Cheat Sheet
> Even Webi needs a place to start.
This installs the tools that are commonly needed to download and unpack various
packages, using the native package manager.
This requires **root or `sudo`** for `apt` or `apk` on Linux.
## Table of Contents
- Files
- Manual Install
- macOS
- Linux
- Alpine
- Windows
### Files
These are the files / directories that are created and/or modified with this
install:
```sh
# macOS
/Library/Developer/CommandLineTools/
/Library/Apple/usr/libexec/oah/libRosettaRuntime
# macOS & Windows
~/.local/bin/xz
~/.local/bin/unxz
# Windows
~/.local/opt/git/
# Linux
/usr/bin/git
```
### How to Install Manually
Most of these utilities are either built-in or provided by the native package
manager.
Webi makes it easy to automate across a wide variety of development systems, but
if you wanted to automate for a specific system, you could do so in a much
simpler way - because you don't have to worry about autodetection or fallbacks
😉.
So here's how to do that for each:
#### macOS
- Rosetta
```sh
softwareupdate --install-rosetta --agree-to-license
```
- Command Line Developer Tools (incl. `git`)
```sh
xcode-select --install
```
- `xz`
```sh
curl https://webi.sh/xz | sh
```
See <https://github.com/therootcompany/xz-static>.
#### Linux (Debian, Ubuntu)
```sh
sudo apt update
sudo apt install -y \
curl \
git \
tar \
wget \
xz \
zip
```
#### Alpine (Docker)
```sh
sudo apk add --no-cache \
curl \
git \
tar \
wget \
xz \
zip
```
#### Windows
`curl.exe` and BSD `tar.exe` are included as part of Windows 10+ - so there's no
needfor `wget` or `zip`.
- `git`
```sh
curl.exe -A MS https://webi.ms/git | powershell
```
See <https://github.com/git-for-windows/git/wiki/MinGit>
- `xz`
```sh
curl https://webi.sh/xz | sh
```
See <https://github.com/therootcompany/xz-static>.

View File

@@ -1,38 +0,0 @@
#!/usr/bin/env pwsh
function Install-WebiEssential() {
# wget: we only use curl.exe on Windows (Invoke-WebRequest is too slow)
# zip: BSD tar.exe handles .zip as well
# Fetch curl
Write-Output " Checking for Windows 10+ Built-in curl.exe ..."
IF (-Not (Get-Command -Name "curl.exe" -ErrorAction Silent)) {
Write-Error"error: curl.exe not found: something is very wrong"
Exit 1
}
# Fetch git
Write-Output " Checking for git ..."
IF (-Not (Get-Command -Name "git" -ErrorAction Silent)) {
& "$Env:USERPROFILE\.local\bin\webi-pwsh.ps1" git
$null = Sync-EnvPath
}
# Fetch tar
Write-Output " Checking for Windows 10+ Built-in BSD tar ..."
IF (-Not (Get-Command -Name "tar.exe" -ErrorAction Silent)) {
Write-Error"error: tar.exe not found: something is very wrong"
Exit 1
}
# Fetch xz
Write-Output " Checking for xz ..."
IF (-Not (Get-Command -Name "xz" -ErrorAction Silent)) {
& "$Env:USERPROFILE\.local\bin\webi-pwsh.ps1" xz
$null = Sync-EnvPath
}
Write-Output " OK"
}
Install-WebiEssential

View File

@@ -1,260 +0,0 @@
#!/bin/sh
set -e
set -u
#set -x
fn_install_webi_essentials_macos() { (
# streamline the output to be pretty
b_pkgs="$(
fn_check_pkg 'curl'
fn_check_pkg '/Library/Developer/CommandLineTools/usr/bin/git' 'git'
if test "$(uname -m)" = 'arm64'; then
fn_check_pkg '/Library/Apple/usr/libexec/oah/libRosettaRuntime' 'rosetta'
fi
fn_check_pkg 'tar'
# no wget because it requires brew
#fn_check_pkg 'wget'
fn_check_pkg 'xz'
fn_check_pkg 'zip'
)"
echo >&2 ""
#curl - built-in
# git
if ! xcode-select -p > /dev/null 2> /dev/null; then
cmd_xcode_cli_tools_install="xcode-select --install"
echo " Running $(t_cmd "${cmd_xcode_cli_tools_install}")"
$cmd_xcode_cli_tools_install 2> /dev/null
echo ""
echo ">>> $(t_attn 'ACTION REQUIRED') <<<"
echo ""
echo " $(t_attn "Click") '$(t_bold 'Install')' $(t_attn "in the pop-up")"
echo " (it may appear $(t_em 'under') this window)"
echo ""
echo "^^^ $(t_attn 'ACTION REQUIRED') ^^^"
echo ""
echo " waiting $(t_em 'for you') to finish installing Command Line Developer Tools ..."
while ! test -x /Library/Developer/CommandLineTools/usr/bin/git; do
sleep 0.25
done
echo " $(t_info 'OK')"
echo ""
sleep 1
fi
# rosetta
if test "$(uname -m)" = 'arm64'; then
# Also pkgutil --pkg-info com.apple.pkg.RosettaUpdateAuto
# See <https://apple.stackexchange.com/q/427970/27465>
cmd_install_rosetta="softwareupdate --install-rosetta --agree-to-license"
if ! arch -arch x86_64 uname -m > /dev/null 2>&1; then
echo " Running $(t_cmd "${cmd_install_rosetta}")"
$cmd_install_rosetta
fi
fi
#tar - built-in
#wget - skip
#xz
if ! command -v xz > /dev/null; then
echo " Running $(t_cmd "webi xz")"
~/.local/bin/webi xz
fi
#zip - built-in
# This should NEVER fail, but... Sanity check :)
{
b_pkgs_builtin="$(
fn_check_pkg 'curl'
fn_check_pkg 'tar'
fn_check_pkg 'zip'
)"
} 2> /dev/null
if test -n "${b_pkgs_builtin}"; then
echo ""
echo "error: expected these to be macOS built-ins:"
echo " ${b_pkgs}"
echo ""
exit 1
fi
echo " $(t_dim 'OK')"
); }
fn_check_pkg() { (
a_pkg="${1}"
a_pkgname="${2:-$a_pkg}"
printf >&2 ' %s %s %s' \
"$(t_dim "Checking for")" \
"$(t_pkg "${a_pkgname}")" \
"$(t_dim "...")"
if command -v "${a_pkg}" > /dev/null; then
echo >&2 " $(t_dim 'OK')"
return 0
fi
echo >&2 ' missing'
echo "${a_pkg}"
); }
###########
# mainish #
###########
fn_polite_sudo() { (
a_sudo="${1}"
a_cmds="${2}"
# no sudo needed, so don't ask
if test -z "${a_sudo}"; then
return 0
fi
# this is scripted, not user-interactive, continue
if test -z "${WEBI_TTY}"; then
return 0
fi
# this is user interactive, ask the user,defaulting to yes
echo ""
#shellcheck disable=SC2005 # echo for newline
echo "$(t_attn 'Use sudo for the following? [Y/n]')"
echo "${a_cmds}"
read -r b_yes < /dev/tty
b_yes="$(
echo "${b_yes}" |
tr '[:upper:]' '[:lower:]' |
tr -d '[:space:]'
)"
if test -z "${b_yes}" || test "${b_yes}" = "y" || test "${b_yes}" = "yes"; then
return 0
fi
echo " aborted"
return 1
); }
_install_webi_essentials() { (
if test "$(uname -s)" = 'Darwin'; then
fn_install_webi_essentials_macos
return 0
fi
b_pkgs="$(
fn_check_pkg 'curl'
fn_check_pkg 'git'
fn_check_pkg 'tar'
fn_check_pkg 'wget'
fn_check_pkg 'xz'
fn_check_pkg 'zip'
)"
b_pkgs="$(echo "${b_pkgs}" | tr '\n' ' ' | tr -s ' ')"
if test "${b_pkgs}" = " "; then
b_pkgs=''
fi
if test -z "${b_pkgs}"; then
echo " $(t_dim 'OK')"
return 0
fi
cmd_sudo=''
if test "$(id -u)" != "0"; then
echo " $(t_dim 'Checking for sudo ...')"
if command -v sudo > /dev/null; then
cmd_sudo='sudo '
else
if test "${b_pkgs}" = "xz"; then
_install_webi_essentials_webi "${cmd_sudo}" "${b_pkgs}"
echo " $(t_dim 'OK')"
return 0
fi
echo ""
echo ">>> $(t_warn 'WARNING') <<<"
echo ""
echo " You are not the 'root' user and there is no 'sudo'."
echo " (We'll try anyway but... this isn't going to work.)"
echo ""
echo "^^^ $(t_warn 'WARNING') ^^^"
echo ""
fi
fi
printf '%s' " $(t_dim 'Checking for apt/apk ...')"
if command -v apt > /dev/null; then
echo " $(t_pkg 'apt')"
_install_webi_essentials_apt "${cmd_sudo}" "${b_pkgs}"
echo " $(t_dim 'OK')"
return 0
fi
if command -v apk > /dev/null; then
echo " $(t_pkg 'apk')"
_install_webi_essentials_apk "${cmd_sudo}" "${b_pkgs}"
echo " $(t_dim 'OK')"
return 0
fi
echo " $(t_dim 'none')"
_install_webi_essentials_webi "${cmd_sudo}" "${b_pkgs}"
echo " $(t_dim 'OK')"
); }
_install_webi_essentials_apt() { (
cmd_sudo="${1}"
b_pkgs="${2}"
b_pkgs="$(echo "${b_pkgs}" | sed 's/xz/xz-utils/g')"
b_cmds="$(
printf ' %s\n %s' \
"$(t_cmd 'apt update')" \
"$(t_cmd "apt install -y ${b_pkgs}")"
)"
fn_polite_sudo "${cmd_sudo}" "${b_cmds}"
echo " $(t_dim 'Running') $(t_cmd "${cmd_sudo}apt update")"
${cmd_sudo} apt update
echo " $(t_dim 'Running') $(t_cmd "${cmd_sudo}apt install -y ${b_pkgs}")"
# shellcheck disable=SC2086
${cmd_sudo} apt install -y ${b_pkgs}
); }
_install_webi_essentials_apk() { (
cmd_sudo="${1}"
b_pkgs="${2}"
echo " $(t_dim 'Running') $(t_cmd "${cmd_sudo}apk add --no-cache")"
fn_polite_sudo "${cmd_sudo}" " $(t_cmd "apk add --no-cache ${b_pkgs}")"
# shellcheck disable=SC2086
${cmd_sudo} apk add --no-cache ${b_pkgs}
); }
_install_webi_essentials_webi() { (
cmd_sudo="${1}"
b_pkgs="${2}"
if test "${b_pkgs}" == 'xz'; then
~/.local/bin/webi xz
return 0
fi
if echo "${b_pkgs}" | grep -q xz; then
~/.local/bin/webi xz
b_pkgs="$(echo "${b_pkgs}" | sed 's/\sxz\s/ /')"
fi
echo ""
echo "error: unknown operating system:"
echo " $(uname -srm)"
if test -r /etc/issue; then
echo " $(cat /etc/issue)"
fi
echo ""
echo "$(t_err 'could not install'): $(t_warn "${b_pkgs}")"
echo ""
return 1
); }
_install_webi_essentials

43
webi/releases.js Normal file
View File

@@ -0,0 +1,43 @@
'use strict';
module.exports = async function () {
return {
releases: [
{
name: 'webi.tar',
version: '0.0.0',
lts: false,
channel: 'stable',
date: '',
os: 'linux',
arch: 'amd64',
ext: 'tar',
download: '',
},
{
name: 'webi.tar',
version: '0.0.0',
lts: false,
channel: 'stable',
date: '',
os: 'macos',
arch: 'amd64',
ext: 'tar',
download: '',
},
{
name: 'webi.tar',
version: '0.0.0',
lts: false,
channel: 'stable',
date: '',
os: 'windows',
arch: 'amd64',
ext: 'tar',
download: '',
},
],
formats: [],
downloads: '',
};
};

View File

@@ -1,9 +1,219 @@
#!/bin/sh
set -e
set -u
#set -x
#########################################
# #
# Display Debug Info in Case of Failure #
# #
#########################################
fn_show_welcome_back() { (
if test -n "${WEBI_WELCOME:-}"; then
return 0
fi
echo ""
# invert t_task and t_pkg for top-level welcome message
printf -- ">>> %s %s <<<\n" \
"$(t_pkg 'Welcome to') $(t_task 'Webi')$(t_pkg '!')" \
"$(t_dim "- modern tools, instant installs.")"
echo " We expect your experience to be $(t_em 'absolutely perfect')!"
echo ""
echo " $(t_attn 'Success')? Star it! $(t_url 'https://github.com/webinstall/webi-installers')"
echo " $(t_attn 'Problem')? Report it: $(t_url 'https://github.com/webinstall/webi-installers/issues')"
echo " $(t_dim "(your system is") $(t_host "$(uname -s)")/$(t_host "$(uname -m)") $(t_dim "with") $(t_host "$(fn_get_libc)") $(t_dim "&") $(t_host "$(fn_get_http_client_name)")$(t_dim ")")"
sleep 0.2
); }
fn_get_libc() { (
# Ex:
# musl
# libc
if ldd /bin/ls 2> /dev/null | grep -q 'musl' 2> /dev/null; then
echo 'musl'
elif uname -o | grep -q 'GNU' || uname -s | grep -q 'Linux'; then
echo 'gnu'
else
echo 'libc'
fi
); }
fn_get_http_client_name() { (
# Ex:
# curl
# curl+wget
b_client=""
if command -v curl > /dev/null; then
b_client="curl"
fi
if command -v wget > /dev/null; then
if test -z "${b_client}"; then
b_client="wget"
else
b_client="curl+wget"
fi
fi
echo "${b_client}"
); }
#########################################
# #
# For Making the Display Nice #
# #
#########################################
# Term Types
t_cmd() { (fn_printf '\e[2m\e[35m%s\e[39m\e[22m' "${1}"); }
t_host() { (fn_printf '\e[2m\e[33m%s\e[39m\e[22m' "${1}"); }
t_link() { (fn_printf '\e[1m\e[36m%s\e[39m\e[22m' "${1}"); }
t_path() { (fn_printf '\e[2m\e[32m%s\e[39m\e[22m' "${1}"); }
t_pkg() { (fn_printf '\e[1m\e[32m%s\e[39m\e[22m' "${1}"); }
t_task() { (fn_printf '\e[36m%s\e[39m' "${1}"); }
t_url() { (fn_printf '\e[2m%s\e[22m' "${1}"); }
# Levels
t_info() { (fn_printf '\e[1m\e[36m%s\e[39m\e[22m' "${1}"); }
t_attn() { (fn_printf '\e[1m\e[33m%s\e[39m\e[22m' "${1}"); }
t_warn() { (fn_printf '\e[1m\e[33m%s\e[39m\e[22m' "${1}"); }
t_err() { (fn_printf '\e[31m%s\e[39m' "${1}"); }
# Styles
t_bold() { (fn_printf '\e[1m%s\e[22m' "${1}"); }
t_dim() { (fn_printf '\e[2m%s\e[22m' "${1}"); }
t_em() { (fn_printf '\e[3m%s\e[23m' "${1}"); }
t_under() { (fn_printf '\e[4m%s\e[24m' "${1}"); }
# FG Colors
t_cyan() { (fn_printf '\e[36m%s\e[39m' "${1}"); }
t_green() { (fn_printf '\e[32m%s\e[39m' "${1}"); }
t_magenta() { (fn_printf '\e[35m%s\e[39m' "${1}"); }
t_yellow() { (fn_printf '\e[33m%s\e[39m' "${1}"); }
fn_printf() { (
a_style="${1}"
a_text="${2}"
if fn_is_tty; then
#shellcheck disable=SC2059
printf -- "${a_style}" "${a_text}"
else
printf -- '%s' "${a_text}"
fi
); }
fn_sub_home() { (
my_rel=${HOME}
my_abs=${1}
echo "${my_abs}" | sed "s:^${my_rel}:~:"
); }
###################################
# #
# Detect HTTP Client #
# #
###################################
fn_wget() { (
# Doc:
# Downloads the file at the given url to the given path
a_url="${1}"
a_path="${2}"
cmd_wget="wget -q --user-agent"
if fn_is_tty; then
cmd_wget="wget -q --show-progress --user-agent"
fi
b_triple_ua="$(fn_get_target_triple_user_agent)"
b_agent="webi/wget ${b_triple_ua}"
if command -v curl > /dev/null; then
b_agent="webi/wget+curl ${b_triple_ua}"
fi
if ! $cmd_wget "${b_agent}" -c "${a_url}" -O "${a_path}"; then
echo >&2 " $(t_err "failed to download (wget)") '$(t_url "${a_url}")'"
echo >&2 " $cmd_wget '${b_agent}' -c '${a_url}' -O '${a_path}'"
echo >&2 " $(wget -V)"
return 1
fi
); }
fn_curl() { (
# Doc:
# Downloads the file at the given url to the given path
a_url="${1}"
a_path="${2}"
cmd_curl="curl -f -sSL -#"
if fn_is_tty; then
cmd_curl="curl -f -sSL"
fi
b_triple_ua="$(fn_get_target_triple_user_agent)"
b_agent="webi/curl ${b_triple_ua}"
if command -v wget > /dev/null; then
b_agent="webi/curl+wget ${b_triple_ua}"
fi
if ! $cmd_curl -A "${b_agent}" "${a_url}" -o "${a_path}"; then
echo >&2 " $(t_err "failed to download (curl)") '$(t_url "${a_url}")'"
echo >&2 " $cmd_curl -A '${b_agent}' '${a_url}' -o '${a_path}'"
echo >&2 " $(curl -V)"
return 1
fi
); }
fn_get_target_triple_user_agent() { (
# Ex:
# x86_64/unknown Linux/5.15.107-2-pve gnu
# arm64/unknown Darwin/22.6.0 libc
echo "$(uname -m)/unknown $(uname -s)/$(uname -r) $(fn_get_libc)"
); }
fn_download_to_path() { (
a_url="${1}"
a_path="${2}"
mkdir -p "$(dirname "${a_path}")"
if command -v wget > /dev/null; then
fn_wget "${a_url}" "${a_path}.part"
elif command -v curl > /dev/null; then
fn_curl "${a_url}" "${a_path}.part"
else
echo >&2 " $(t_err "failed to detect HTTP client (curl, wget)")"
return 1
fi
mv "${a_path}.part" "${a_path}"
); }
##############################################
# #
# Detect TTY and run main #
# #
##############################################
fn_is_tty() {
if test "${WEBI_TTY}" = 'tty'; then
return 0
fi
return 1
}
fn_detect_tty() { (
# stdin will NOT be a tty if it's being piped
# stdout & stderr WILL be a tty even when piped
# they are not a tty if being captured or redirected
# 'set -i' is NOT available in sh
if test -t 1 && test -t 2; then
return 0
fi
return 1
); }
__webi_main() {
my_date="$(date +%F_%H-%M-%S)"
@@ -61,6 +271,8 @@ __webi_main() {
set +e
WEBI_CURL="$(command -v curl)"
export WEBI_URL
WEBI_WGET="$(command -v wget)"
export WEBI_WGET
set -e
my_libc=''
@@ -152,7 +364,7 @@ __webi_main() {
my_checksum="$(
fn_checksum
)"
my_version=v1.2.8
my_version=v1.2.0
printf "\e[35mwebi\e[32m %s\e[0m Copyright 2020+ AJ ONeal\n" "${my_version} (${my_checksum})"
printf " \e[36mhttps://webinstall.dev/webi\e[0m\n"
}
@@ -188,8 +400,6 @@ __webi_main() {
echo " Output the version number of webi and exit."
echo ""
echo " Helper Utilities"
echo " --init Register command line completions with shell"
echo ""
echo " --list Show everything webi has to offer."
echo ""
echo " --info <package>"
@@ -214,18 +424,71 @@ __webi_main() {
exit 0
fi
if echo "$1" | grep -q -E '^(--list|list)$'; then
webi_list
if echo "$1" | grep -q -E '^(-l|--list|list)$'; then
echo >&2 "[warn] the format of --list output may change"
# because we don't have sitemap.xml for dev sites yet
my_host="https://webinstall.dev"
my_len="${#my_host}"
# 6 because the field will looks like "loc>WEBI_HOST/PKG_NAME"
# and the count is 1-indexed
my_count="$((my_len + 6))"
curl -fsS "${my_host}/sitemap.xml" |
grep -F "${my_host}" |
cut -d'<' -f2 |
cut -c "${my_count}"-
exit 0
fi
if echo "${1}" | grep -q -E '^(--info|info)$'; then
webi_info "$@"
exit 0
fi
if test -z "${2}"; then
echo >&2 "Usage: webi --info <package>"
exit 1
fi
echo >&2 "[warn] the output of --info is completely half-baked and will change"
my_pkg="${2}"
# TODO need a way to check that it exists at all (readme, win, lin)
echo ""
echo " Cheat Sheet: ${WEBI_HOST}/${my_pkg}"
echo " POSIX: curl -sS ${WEBI_HOST}/${my_pkg} | sh"
echo " Windows: curl.exe -A MS ${WEBI_HOST}/${my_pkg} | powershell"
echo "Releases (JSON): ${WEBI_HOST}/api/releases/${my_pkg}.json"
echo " Releases (tsv): ${WEBI_HOST}/api/releases/${my_pkg}.tab"
echo " (query params): ?channel=stable&limit=10"
echo " &os=${my_os}&arch=${my_arch}"
echo " Install Script: ${WEBI_HOST}/api/installers/${my_pkg}.sh?formats=tar,zip,xz,git,dmg,pkg"
echo " Static Assets: ${WEBI_HOST}/packages/${my_pkg}/README.md"
echo ""
# TODO os=linux,macos,windows (limit to tagged releases)
my_releases="$(
curl -fsS "${WEBI_HOST}/api/releases/${my_pkg}.json?channel=stable&limit=1&pretty=true"
)"
if printf '%s\n' "${my_releases}" | grep -q "error"; then
my_releases_beta="$(
curl -fsS "${WEBI_HOST}/api/releases/${my_pkg}.json?&limit=1&pretty=true"
)"
if printf '%s\n' "${my_releases_beta}" | grep -q "error"; then
echo >&2 "'${my_pkg}' is a special case that does not have releases"
else
echo >&2 "ERROR no stable releases for '${my_pkg}'!"
fi
exit 0
fi
echo >&2 "Stable '${my_pkg}' releases:"
if command -v jq > /dev/null; then
printf '%s\n' "${my_releases}" |
jq
else
printf '%s\n' "${my_releases}"
fi
if echo "$1" | grep -q -E '^(--init|init)$'; then
webi_shell_init "$@"
exit 0
fi
@@ -238,271 +501,4 @@ __webi_main() {
}
webi_shell_init() { (
a_shell="${2:-}"
fn_shell_integrate_bash ""
fn_shell_integrate_zsh ""
fn_shell_integrate_fish ""
# update completions now
webi_list > /dev/null
if [ $# -eq 1 ]; then
exit 0
fi
case "${a_shell}" in
bash)
fn_shell_integrate_bash "force"
fn_shell_init_bash
;;
zsh)
fn_shell_integrate_zsh "force"
fn_shell_init_zsh
;;
fish)
fn_shell_integrate_fish "force"
fn_shell_init_fish
;;
*)
echo >&2 "Unsupported shell: $2"
exit 1
;;
esac
) }
fn_shell_integrate_bash() { (
a_force="${1}"
if test -z "${a_force}"; then
if ! command -v bash > /dev/null; then
return 0
fi
if ! test -e ~/.bashrc && ! test -e ~/.bash_history; then
return 0
fi
fi
touch -a ~/.bashrc
if grep -q 'webi --init' ~/.bashrc; then
return 0
fi
echo >&2 " Edit ~/.bashrc to add 'eval \"\$(webi --init bash)\"'"
# shellcheck disable=SC2016
{
echo ''
echo '# Generated by Webi. Do not edit.'
echo 'eval "$(webi --init bash)"'
} >> ~/.bashrc
); }
# shellcheck disable=SC2016
fn_shell_init_bash() { (
echo '_webi() {'
echo ' COMPREPLY=()'
echo ' local cur="${COMP_WORDS[COMP_CWORD]}"'
echo ' if [ "$COMP_CWORD" -eq 1 ]; then'
echo ' local completions=$(webi --list | cut -d" " -f1)'
echo ' COMPREPLY=( $(compgen -W "$completions" -- "$cur") )'
echo ' fi'
echo '}'
echo ''
echo 'complete -F _webi webi'
); }
fn_shell_integrate_zsh() { (
a_force="${1}"
if test -z "${a_force}"; then
if ! command -v zsh > /dev/null; then
return 0
fi
if ! test -e ~/.zshrc &&
! test -e ~/.zsh_sessions &&
! test -e ~/.zsh_history; then
return 0
fi
fi
touch -a ~/.zshrc
if grep -q 'webi --init' ~/.zshrc; then
return 0
fi
echo >&2 " Edit ~/.zshrc to add 'eval \"\$(webi --init zsh)\"'"
# shellcheck disable=SC2016
{
echo ''
echo '# Generated by Webi. Do not edit.'
echo 'eval "$(webi --init zsh)"'
} >> ~/.zshrc
); }
# shellcheck disable=SC2016
fn_shell_init_zsh() { (
echo '_webi() {'
echo ' local -a list completions'
echo ' list=$(webi --list | cut -d" " -f1)'
echo ' completions=(${(f)list})'
echo ' _describe -t commands "command" completions && ret=0'
echo '}'
echo ''
echo 'autoload -Uz compinit && compinit'
echo 'compdef _webi webi'
); }
fn_shell_integrate_fish() { (
a_force="${1}"
if test -z "${a_force}"; then
if ! command -v fish > /dev/null; then
return 0
fi
fi
mkdir -p ~/.config/fish
touch -a ~/.config/fish/config.fish
if grep -q 'webi --init' ~/.config/fish/config.fish; then
return 0
fi
echo >&2 " Edit ~/.config/fish/config.fish to add 'webi --init fish | source'"
# shellcheck disable=SC2016
{
echo ''
echo '# Generated by Webi. Do not edit.'
echo 'webi --init fish | source'
} >> ~/.config/fish/config.fish
); }
# shellcheck disable=SC2016
fn_shell_init_fish() { (
echo 'function __fish_webi_needs_command'
echo ' set cmd (commandline -opc)'
echo ' if [ (count $cmd) -eq 1 -a $cmd[1] = "webi" ]'
echo ' return 0'
echo ' end'
echo ' return 1'
echo 'end'
echo ''
echo 'set completions (webi --list | cut -d" " -f1)'
echo 'complete -f -c webi -n __fish_webi_needs_command -a "$completions"'
); }
webi_list() { (
# make sure there's always a cache dir and timestamp file
mkdir -p ~/.local/share/webi/var/
if ! test -r ~/.local/share/webi/var/list.txt; then
echo '0' > ~/.local/share/webi/var/last_update
elif ! test -r ~/.local/share/webi/var/last_update; then
echo '0' > ~/.local/share/webi/var/last_update
fi
# compare the timestamp in the timestamp file to now
# (in seconds since unix epoch)
my_stale_age=600
my_expire_age=900
my_now="$(date -u '+%s')"
my_then="$(cat ~/.local/share/webi/var/last_update)"
my_diff=$((my_now - my_then))
# show when the cache will update
my_stales_in=$((my_stale_age - my_diff))
my_expires_in=$((my_expire_age - my_diff))
# update if it's been longer than the staletime
if test "${my_stales_in}" -lt "0"; then
if test "${my_expires_in}" -lt "0"; then
fn_list_uncached
else
fn_list_uncached &
fi
fi
# give back the list
cat ~/.local/share/webi/var/list.txt
); }
fn_list_uncached() { (
# because we don't have sitemap.xml for dev sites yet
my_host="https://webinstall.dev"
my_len="${#my_host}"
# 6 because the field will looks like "loc>WEBI_HOST/PKG_NAME"
# and the count is 1-indexed
my_count="$((my_len + 6))"
my_now="$(date -u '+%s')"
echo "${my_now}" > ~/.local/share/webi/var/last_update
my_tmp="$(mktemp)"
{
echo "help"
echo "--help"
echo "version"
echo "-V"
echo "--version"
echo "--init" # <shell>
echo "--list"
echo "--info" # <package>
} > "${my_tmp}"
curl -fsS "${my_host}/sitemap.xml" |
grep -F "${my_host}" |
cut -d'<' -f2 |
cut -c "${my_count}"- >> "${my_tmp}"
mv "${my_tmp}" ~/.local/share/webi/var/list.txt
my_now="$(date -u '+%s')"
echo "${my_now}" > ~/.local/share/webi/var/last_update
); }
webi_info() { (
if test -z "${2}"; then
echo >&2 "Usage: webi --info <package>"
exit 1
fi
echo >&2 "[warn] the output of --info is completely half-baked and will change"
my_pkg="${2}"
# TODO need a way to check that it exists at all (readme, win, lin)
echo ""
echo " Cheat Sheet: ${WEBI_HOST}/${my_pkg}"
echo " POSIX: curl -sS ${WEBI_HOST}/${my_pkg} | sh"
echo " Windows: curl.exe -A MS ${WEBI_HOST}/${my_pkg} | powershell"
echo "Releases (JSON): ${WEBI_HOST}/api/releases/${my_pkg}.json"
echo " Releases (tsv): ${WEBI_HOST}/api/releases/${my_pkg}.tab"
echo " (query params): ?channel=stable&limit=10"
echo " &os=${my_os}&arch=${my_arch}"
echo " Install Script: ${WEBI_HOST}/api/installers/${my_pkg}.sh?formats=tar,zip,xz,git,dmg,pkg"
echo " Static Assets: ${WEBI_HOST}/packages/${my_pkg}/README.md"
echo ""
# TODO os=linux,macos,windows (limit to tagged releases)
my_releases="$(
curl -fsS "${WEBI_HOST}/api/releases/${my_pkg}.json?channel=stable&limit=1&pretty=true"
)"
if printf '%s\n' "${my_releases}" | grep -q "error"; then
my_releases_beta="$(
curl -fsS "${WEBI_HOST}/api/releases/${my_pkg}.json?&limit=1&pretty=true"
)"
if printf '%s\n' "${my_releases_beta}" | grep -q "error"; then
echo >&2 "'${my_pkg}' is a special case that does not have releases"
else
echo >&2 "ERROR no stable releases for '${my_pkg}'!"
fi
exit 0
fi
echo >&2 "Stable '${my_pkg}' releases:"
if command -v jq > /dev/null; then
printf '%s\n' "${my_releases}" |
jq
else
printf '%s\n' "${my_releases}"
fi
); }
__webi_main "$@"

View File

@@ -1,7 +1,6 @@
'use strict';
var NON_BUILDS = ['bootstrap', 'src'];
var ODDITIES = NON_BUILDS.concat(['armv6kz-linux']);
module.exports = function (request) {
return request({
@@ -28,8 +27,8 @@ module.exports = function (request) {
return;
}
let isOdd = ODDITIES.includes(platform);
if (isOdd) {
let isNonBuild = NON_BUILDS.includes(platform);
if (isNonBuild) {
return;
}

View File

@@ -6,6 +6,11 @@ var repo = 'zoxide';
module.exports = function (request) {
return github(request, owner, repo).then(function (all) {
all.releases.forEach(function (rel) {
if (/-arm-/.test(rel.download)) {
rel.arch = 'armv6l';
}
});
return all;
});
};