github/vim-ale
1
0
Fork 0
mirror of https://github.com/webinstall/webi-installers.git synced 2026-04-19 00:16:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(installer): replace getReleases with new builds classifier

Browse Source
This commit is contained in:
AJ ONeal
2023-12-12 03:09:18 -07:00
parent 72c0cd5985
commit 22ba86dbed
5 changed files with 300 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
_webi/builds-cacher.js
View File

@@ -54,6 +54,54 @@ var TERMS_META = [
'stable',
];
/** @typedef {String} TripletString - {arch}-{vendor}-{os}-{libc} */
/** @typedef {String} VersionString */
/** @typedef {Object.<VersionString, Array<BuildAsset>>} PackagesByRelease */
/**
* @typedef ProjectInfo
* @prop {Array<BuildAsset>} releases
* @prop {Array<BuildAsset>} packages
* @prop {Object.<TripletString, PackagesByRelease>} releasesByTriplet
* @prop {Array<import('./build-classifier/types.js').ArchString>} arches
* @prop {Array<import('./build-classifier/types.js').OsString>} oses
* @prop {Array<import('./build-classifier/types.js').LibcString>} libcs
* @prop {Array<String>} channels
* @prop {Array<String>} formats
* @prop {Array<String>} triplets
* @prop {Array<String>} versions
* @prop {Array<String>} lexvers
* @prop {Object.<String, String>} lexversMap
*/
/**
* @typedef BuildAsset
* @prop {String} name
* @prop {String} version
* @prop {Boolean} lts
* @prop {String} date
* @prop {String} arch
* @prop {String} os
* @prop {String} libc
* @prop {String} ext
* @prop {String} download
*/
/**
* @typedef VersionTarget
* @prop {String} version
* @prop {Boolean} lts
* @prop {String} channel
*/
/** @typedef {TargetTriplet & HostTargetPartial} HostTarget */
/** @typedef {import('./build-classifier/types.js').TargetTriplet} TargetTriplet */
/**
* @typedef HostTargetPartial
* @prop {String} target.triplet - os-vendor-arch-libc
* @prop {Error} target.error
*/
async function getPartialHeader(path) {
let readme = `${path}/README.md`;
let head = await readFirstBytes(readme).catch(function (err) {
@@ -72,7 +120,7 @@ async function readFirstBytes(path) {
let start = 0;
let n = 1024;
let fh = await Fs.open(path, 'r');
let buf = new Buffer.alloc(n);
let buf = Buffer.alloc(n);
let result = await fh.read(buf, start, n);
let str = result.buffer.toString('utf8');
await fh.close();
@@ -734,21 +782,35 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
};
/**
* @param {Object} target
* @param {String} target.os - linux, darwin, windows, *bsd
* @param {String} target.arch - arm64, x86_64, ppc64le
* @param {String} target.libc - none, libc, gnu, musl, bionic, msvc
* @param {String} target.triplet - os-vendor-arch-libc
* @param {Error} target.error
* @param {ProjectInfo} projInfo
* @param {HostTarget} hostTarget
*/
bc.findMatchingPackages = function (pkgInfo, hostTarget, verTarget) {
let matchInfo = bc._enumerateVersions(pkgInfo, verTarget.version);
bc.enumerateLatestVersions = function (projInfo, hostTarget) {
let lexPrefix = '';
let matchInfo = Lexver.matchSorted(projInfo.lexvers, lexPrefix);
let verInfo = {
default: projInfo.lexversMap[matchInfo.default],
previous: projInfo.lexversMap[matchInfo.previous],
stable: projInfo.lexversMap[matchInfo.stable],
latest: projInfo.lexversMap[matchInfo.latest],
};
return verInfo;
};
/**
* @param {ProjectInfo} projInfo
* @param {HostTarget} hostTarget
* @param {VersionTarget} verTarget
*/
bc.findMatchingPackages = function (projInfo, hostTarget, verTarget) {
let matchInfo = bc._enumerateVersions(projInfo, verTarget.version);
let triplets = bc._enumerateTriplets(hostTarget);
//console.log('dbg: matchInfo', matchInfo);
if (matchInfo) {
for (let _triplet of triplets) {
let targetReleases = pkgInfo.releasesByTriplet[_triplet];
let targetReleases = projInfo.releasesByTriplet[_triplet];
if (!targetReleases) {
continue;
}
@@ -756,7 +818,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
// Make sure that these releases are the expected version
// (ex: jq1.7 => darwin-arm64-libc, jq1.6 => darwin-x86_64-libc)
for (let matchver of matchInfo.matches) {
let ver = pkgInfo.lexversMap[matchver] || matchver;
let ver = projInfo.lexversMap[matchver] || matchver;
let packages = targetReleases[ver];
if (!packages) {
continue;
@@ -765,7 +827,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
let match = {
triplet: _triplet,
packages: packages,
latest: pkgInfo.versions[0],
latest: projInfo.versions[0],
version: ver,
versions: matchInfo,
};
@@ -777,7 +839,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
}
for (let _triplet of triplets) {
let targetReleases = pkgInfo.releasesByTriplet[_triplet];
let targetReleases = projInfo.releasesByTriplet[_triplet];
if (!targetReleases) {
continue;
}
@@ -796,7 +858,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
// Make sure that these releases are the expected version
// (ex: jq1.7 => darwin-arm64-libc, jq1.6 => darwin-x86_64-libc)
for (let matchver of lexvers) {
let ver = pkgInfo.lexversMap[matchver] || matchver;
let ver = projInfo.lexversMap[matchver] || matchver;
let packages = targetReleases[ver];
//console.log('dbg: packages', packages);
if (!packages) {
@@ -812,7 +874,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
let match = {
triplet: _triplet,
packages: packages,
latest: pkgInfo.versions[0],
latest: projInfo.versions[0],
version: ver,
versions: matchInfo,
};
@@ -831,7 +893,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
let match = {
triplet: _triplet,
packages: packages,
latest: pkgInfo.versions[0],
latest: projInfo.versions[0],
version: ver,
versions: matchInfo,
};
@@ -878,12 +940,12 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
return triplets;
};
bc._enumerateVersions = function (pkgInfo, ver) {
bc._enumerateVersions = function (projInfo, ver) {
if (!ver) {
return null;
}
let lexPrefix = Lexver.parsePrefix(ver);
let matchInfo = Lexver.matchSorted(pkgInfo.lexvers, lexPrefix);
let matchInfo = Lexver.matchSorted(projInfo.lexvers, lexPrefix);
return matchInfo;
};

41
_webi/builds.js Normal file
View File

@@ -0,0 +1,41 @@
'use strict';
let Builds = module.exports;
let Path = require('node:path');
let BuildsCacher = require('./builds-cacher.js');
// let HostTargets = require('./build-classifier/host-targets.js');
let Parallel = require('./parallel.js');
var INSTALLERS_DIR = Path.join(__dirname, '..');
var CACHE_DIR = Path.join(__dirname, '../_cache');
let bc = BuildsCacher.create({
caches: CACHE_DIR,
installers: INSTALLERS_DIR,
});
bc.freshenRandomPackage(600 * 1000);
Builds.init = async function () {
bc.freshenRandomPackage(600 * 1000);
let dirs = await bc.getProjects();
let projNames = Object.keys(dirs.valid);
let parallel = 25;
await Parallel.run(parallel, projNames, getAll);
async function getAll(name) {
void (await bc.getPackages({
//Releases: Releases,
name: name,
date: new Date(),
}));
}
};
Builds.enumerateLatestVersions = bc.enumerateLatestVersions;
Builds.findMatchingPackages = bc.findMatchingPackages;
Builds.getPackage = bc.getPackages;
Builds.getProjectType = bc.getProjectType;
Builds.selectPackage = bc.selectPackage;

8
_webi/lint-builds.js
View File

@@ -125,6 +125,7 @@ let bc = BuildsCacher.create({
});
async function main() {
/* jshint maxcomplexity: 25 */
// TODO
// node ./_webi/lint-builds.js caddy@beta 'x86_64/unknown Darwin libc'
//
@@ -184,6 +185,7 @@ async function main() {
// non-build file
continue;
}
if (target.error) {
let e = target.error;
if (e.code === 'E_BUILD_NO_PATTERN') {
@@ -195,6 +197,12 @@ async function main() {
throw e;
}
if (target.unknownTerms?.length) {
let msg = `${projInfo.name}: unrecognized term(s) '${target.unknownTerms}' in '${build.download}'`;
let err = new Error(msg);
throw err;
}
triples.push(target.triplet);
// if (!build.version) {
// throw new Error(`no version for ${pkg.name} ${build.name}`);

261
_webi/serve-installer.js
View File

@@ -2,17 +2,14 @@
var InstallerServer = module.exports;
var Fs = require('fs/promises');
var path = require('path');
let Fs = require('fs/promises');
let Path = require('path');
var uaDetect = require('./ua-detect.js');
var Projects = require('./projects.js');
var Installers = require('./installers.js');
let HostTargets = require('./build-classifier/host-targets.js');
let Builds = require('./builds.js');
let Installers = require('./installers.js');
// handlers caching and transformation, probably should be broken down
var Releases = require('./transform-releases.js');
InstallerServer.INSTALLERS_DIR = path.join(__dirname, '..');
InstallerServer.INSTALLERS_DIR = Path.join(__dirname, '..');
InstallerServer.serveInstaller = async function (
baseurl,
ua,
@@ -22,113 +19,191 @@ InstallerServer.serveInstaller = async function (
formats,
libc,
) {
let [rel, opts] = await InstallerServer.helper({
ua,
pkg,
let unameAgent = ua;
let projectName = pkg;
let [rel, tmplParams] = await InstallerServer.helper({
unameAgent,
projectName,
tag,
formats,
libc,
});
Object.assign(opts, {
Object.assign(tmplParams, {
baseurl,
});
var pkgdir = path.join(InstallerServer.INSTALLERS_DIR, pkg);
var pkgdir = Path.join(InstallerServer.INSTALLERS_DIR, projectName);
if ('ps1' === ext) {
return Installers.renderPowerShell(pkgdir, rel, opts);
return Installers.renderPowerShell(pkgdir, rel, tmplParams);
}
return Installers.renderBash(pkgdir, rel, opts);
return Installers.renderBash(pkgdir, rel, tmplParams);
};
InstallerServer.helper = async function ({ ua, pkg, tag, formats, libc }) {
// TODO put some of this in a middleware? or common function?
// TODO maybe move package/version/lts/channel detection into getReleases
var ver = tag.replace(/^v/, '');
var lts;
var channel;
// TODO put some of this in a middleware? or common function?
// TODO maybe move package/version/lts/channel detection into getReleases
InstallerServer.helper = async function ({
unameAgent,
projectName,
tag,
formats,
libc,
}) {
console.log(`dbg: Installer User-Agent: ${unameAgent}`);
switch (ver) {
case 'latest':
ver = '';
channel = 'stable';
break;
case 'lts':
lts = true;
channel = 'stable';
ver = '';
break;
case 'stable':
channel = 'stable';
ver = '';
break;
case 'beta':
channel = 'beta';
ver = '';
break;
case 'dev':
channel = 'dev';
ver = '';
break;
let releaseTarget = toReleaseTarget(tag);
let hostFormats = formats;
let terms = unameAgent.split(/[\s\/]+/g);
let hostTarget = {};
try {
void HostTargets.termsToTarget(hostTarget, terms);
} catch (e) {
// if we can't guarantee the results...
// "in the face of ambiguity, refuse the temptation to guess"
throw e;
}
console.log(`dbg: Installer Host Target:`);
console.log(hostTarget);
if (!hostTarget.os) {
throw new Error(`OS could not be identified by User-Agent '${unameAgent}'`);
}
var myOs = uaDetect.os(ua);
var myArch = uaDetect.arch(ua);
var myLibc;
if (libc) {
myLibc = uaDetect.libc(libc);
console.log(`dbg: Get Project Installer Type for '${projectName}':`);
let proj = await Builds.getProjectType(projectName);
console.log(proj);
let validTypes = ['alias', 'selfhosted', 'valid'];
if (!validTypes.includes(proj.type)) {
let msg = `'${projectName}' doesn't have an installer: '${proj.type}': '${proj.detail}'`;
let err = new Error(msg);
err.code = 'ENOENT';
throw err;
}
if (!myLibc) {
myLibc = uaDetect.libc(ua);
}
if (!myLibc) {
myLibc = 'libc';
if (proj.type === 'alias') {
projectName = proj.detail;
}
let cfg = await Projects.get(pkg);
let releaseQuery = {
pkg: cfg.alias || pkg,
ver,
os: myOs,
arch: myArch,
libc: myLibc,
lts,
channel,
// TODO use formats for sorting, not exclusion
// (it's better to install xz or report an error to install zip)
formats,
let tmplParams = {
pkg: projectName,
tag: tag,
os: hostTarget.os,
arch: hostTarget.arch,
libc: hostTarget.libc,
formats: hostFormats,
limit: 1,
};
Object.assign(tmplParams, releaseTarget);
console.log('tmplParams', tmplParams);
let rels = await Releases.getReleases(releaseQuery);
var rel = rels.releases[0];
var opts = {
pkg: cfg.alias || pkg,
ver,
tag,
os: myOs,
arch: myArch,
libc: myLibc,
lts,
channel,
formats,
limit: 1,
let errPackage = {
name: 'doesntexist.ext',
version: '0.0.0',
lts: '-',
channel: 'error',
date: '1970-01-01',
os: hostTarget.os || '-',
arch: hostTarget.arch || '-',
libc: hostTarget.libc || '-',
ext: 'err',
download: 'https://example.com/doesntexist.ext',
comment:
'No matches found. Could be bad or missing version info' +
',' +
"Check query parameters. Should be something like '/api/releases/{package}@{version}.tab?os={macos|linux|windows|-}&arch={amd64|x86|aarch64|arm64|armv7l|-}&libc={musl|gnu|msvc|libc|static}&limit=10'",
};
rel = Object.assign(
{
oses: rels.oses,
arches: rels.arches,
libcs: rels.libcs,
formats: rels.formats,
},
rel,
if (proj.type === 'selfhosted') {
return [errPackage, tmplParams];
}
let projInfo = await Builds.getPackage({
name: projectName,
date: new Date(),
});
let latest = projInfo.versions[0];
Object.assign(tmplParams, { latest });
//console.log('projInfo', projInfo);
let buildTargetInfo = {
triplets: projInfo.triplets,
oses: projInfo.oses,
arches: projInfo.arches,
libcs: projInfo.libcs,
formats: projInfo.formats,
};
let hasOs = projInfo.oses.includes(hostTarget.os);
if (!hasOs) {
let pkg1 = Object.assign(buildTargetInfo, errPackage);
return [pkg1, tmplParams];
}
let targetRelease = Builds.findMatchingPackages(
projInfo,
hostTarget,
releaseTarget,
);
// { triplet: `${os}-${arch}-${libc}`, packages: targetPackages
// , latest: projInfo.versions[0], versions: matchInfo
// }
return [rel, opts];
if (!targetRelease?.packages) {
let pkg1 = Object.assign(buildTargetInfo, errPackage);
return [pkg1, tmplParams];
}
let buildPkg = Builds.selectPackage(targetRelease.packages, hostFormats);
let ext = buildPkg.ext || '.exe';
if (ext.startsWith('.')) {
ext = ext.slice(1);
}
let version = targetRelease.version;
if (version.startsWith('v')) {
version = version.slice(1);
}
buildPkg = Object.assign(buildTargetInfo, buildPkg, { ext, version });
console.log('dbg: buildPkg', buildPkg);
console.log('dbg: tmplParams', tmplParams);
return [buildPkg, tmplParams];
};
var CURL_PIPE_PS1_BOOT = path.join(__dirname, 'curl-pipe-bootstrap.tpl.ps1');
var CURL_PIPE_SH_BOOT = path.join(__dirname, 'curl-pipe-bootstrap.tpl.sh');
let channelNames = [
'stable',
// 'hotfix',
'latest',
'rc',
'preview',
'pre',
'dev',
'beta',
'alpha',
];
function toReleaseTarget(tag) {
tag = tag.replace(/^v/, '');
let releaseTarget = {
channel: '',
lts: false,
version: '',
};
if (tag === 'lts') {
releaseTarget.lts = true;
releaseTarget.channel = 'stable';
} else if (channelNames.includes(tag)) {
releaseTarget.channel = tag;
} else {
releaseTarget.version = tag;
}
return releaseTarget;
}
var CURL_PIPE_PS1_BOOT = Path.join(__dirname, 'curl-pipe-bootstrap.tpl.ps1');
var CURL_PIPE_SH_BOOT = Path.join(__dirname, 'curl-pipe-bootstrap.tpl.sh');
var BAD_SH_RE = /[<>'"`$\\]/;
InstallerServer.getPosixCurlPipeBootstrap = async function ({
@@ -150,7 +225,6 @@ InstallerServer.getPosixCurlPipeBootstrap = async function ({
let name = env[0];
let value = env[1];
// TODO create REs once, in higher scope
let envRe = new RegExp(
`^[ \\t]*#?[ \\t]*(export[ \\t])?[ \\t]*(${name})=.*`,
'm',
@@ -198,9 +272,6 @@ InstallerServer.getPwshCurlPipeBootstrap = async function ({
let tplRe = new RegExp(`{{ (${name}) }}`, 'g');
bootTxt = bootTxt.replace(tplRe, `${value}`);
// let envRe = new RegExp(`^[ \\t]*#?[ \\t]*($$${name})[ \\t]*=.*`, 'im');
// bootTxt = bootTxt.replace(envRe, `$$${name} = '${value}'`);
let setRe = new RegExp(
`(#[ \\t]*)?(\\$${name})[ \\t]*=[ \\t]['"].*['"][ \\t]`,
'im',

9
_webi/test.js
View File

@@ -32,7 +32,7 @@ if (/\b-?-h(elp)?\b/.test(process.argv.join(' '))) {
var os = require('os');
var fs = require('fs');
var path = require('path');
var Releases = require('./transform-releases.js');
var Builds = require('./builds.js');
var Installers = require('./installers.js');
var ServeInstaller = require('./serve-installer.js');
@@ -65,7 +65,8 @@ console.info('Has the necessary files?');
});
console.info('');
Releases.get(path.join(process.cwd(), pkgdir)).then(async function (all) {
let projName = pkgdir.split('/').filter(Boolean).pop();
Builds.getPackage({ name: projName }).then(async function (/*projInfo*/) {
var pkgname = path.basename(pkgdir.replace(/\/$/, ''));
var nodeOs = os.platform();
var nodeOsRelease = os.release();
@@ -82,8 +83,8 @@ Releases.get(path.join(process.cwd(), pkgdir)).then(async function (all) {
var formats = ['exe', 'xz', 'tar', 'zip', 'git'];
let [rel, opts] = await ServeInstaller.helper({
ua: `${nodeOs}/${nodeOsRelease} ${nodeArch}/unknown ${nodeLibc}`,
pkg: pkgname,
unameAgent: `${nodeOs}/${nodeOsRelease} ${nodeArch}/unknown ${nodeLibc}`,
projectName: pkgname,
tag: pkgtag || '',
formats: formats,
libc: nodeLibc,