From 5909757a5bb4f940d33e21db611a2d986cbf3b48 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Tue, 12 Dec 2023 03:09:18 -0700 Subject: [PATCH] feat(installer): replace getReleases with new builds classifier --- _webi/builds.js | 40 ++++++ _webi/serve-installer.js | 261 +++++++++++++++++++++++++-------------- _webi/test.js | 4 +- 3 files changed, 208 insertions(+), 97 deletions(-) create mode 100644 _webi/builds.js diff --git a/_webi/builds.js b/_webi/builds.js new file mode 100644 index 0000000..94be73e --- /dev/null +++ b/_webi/builds.js @@ -0,0 +1,40 @@ +'use strict'; + +let Builds = module.exports; + +let Path = require('node:path'); + +let BuildsCacher = require('./builds-cacher.js'); +// let HostTargets = require('./build-classifier/host-targets.js'); +let Parallel = require('./parallel.js'); + +var INSTALLERS_DIR = Path.join(__dirname, '..'); +var CACHE_DIR = Path.join(__dirname, '../_cache'); + +let bc = BuildsCacher.create({ + caches: CACHE_DIR, + installers: INSTALLERS_DIR, +}); +bc.freshenRandomPackage(600 * 1000); + +Builds.init = async function () { + bc.freshenRandomPackage(600 * 1000); + + let dirs = await bc.getProjects(); + let projNames = Object.keys(dirs.valid); + + let parallel = 25; + await Parallel.run(parallel, projNames, getAll); + async function getAll(name) { + void (await bc.getPackages({ + //Releases: Releases, + name: name, + date: new Date(), + })); + } +}; + +Builds.getProjectType = bc.getProjectType; +Builds.getPackage = bc.getPackages; +Builds.findMatchingPackages = bc.findMatchingPackages; +Builds.selectPackage = bc.selectPackage; diff --git a/_webi/serve-installer.js b/_webi/serve-installer.js index 46c631b..fb6b8bc 100644 --- a/_webi/serve-installer.js +++ b/_webi/serve-installer.js @@ -2,17 +2,14 @@ var InstallerServer = module.exports; -var Fs = require('fs/promises'); -var path = require('path'); +let Fs = require('fs/promises'); +let Path = require('path'); -var uaDetect = require('./ua-detect.js'); -var Projects = require('./projects.js'); -var Installers = require('./installers.js'); +let HostTargets = require('./build-classifier/host-targets.js'); +let Builds = require('./builds.js'); +let Installers = require('./installers.js'); -// handlers caching and transformation, probably should be broken down -var Releases = require('./transform-releases.js'); - -InstallerServer.INSTALLERS_DIR = path.join(__dirname, '..'); +InstallerServer.INSTALLERS_DIR = Path.join(__dirname, '..'); InstallerServer.serveInstaller = async function ( baseurl, ua, @@ -22,113 +19,191 @@ InstallerServer.serveInstaller = async function ( formats, libc, ) { - let [rel, opts] = await InstallerServer.helper({ - ua, - pkg, + let unameAgent = ua; + let projectName = pkg; + let [rel, tmplParams] = await InstallerServer.helper({ + unameAgent, + projectName, tag, formats, libc, }); - Object.assign(opts, { + Object.assign(tmplParams, { baseurl, }); - var pkgdir = path.join(InstallerServer.INSTALLERS_DIR, pkg); + var pkgdir = Path.join(InstallerServer.INSTALLERS_DIR, projectName); if ('ps1' === ext) { - return Installers.renderPowerShell(pkgdir, rel, opts); + return Installers.renderPowerShell(pkgdir, rel, tmplParams); } - return Installers.renderBash(pkgdir, rel, opts); + return Installers.renderBash(pkgdir, rel, tmplParams); }; -InstallerServer.helper = async function ({ ua, pkg, tag, formats, libc }) { - // TODO put some of this in a middleware? or common function? - // TODO maybe move package/version/lts/channel detection into getReleases - var ver = tag.replace(/^v/, ''); - var lts; - var channel; +// TODO put some of this in a middleware? or common function? +// TODO maybe move package/version/lts/channel detection into getReleases +InstallerServer.helper = async function ({ + unameAgent, + projectName, + tag, + formats, + libc, +}) { + console.log(`dbg: Installer User-Agent: ${unameAgent}`); - switch (ver) { - case 'latest': - ver = ''; - channel = 'stable'; - break; - case 'lts': - lts = true; - channel = 'stable'; - ver = ''; - break; - case 'stable': - channel = 'stable'; - ver = ''; - break; - case 'beta': - channel = 'beta'; - ver = ''; - break; - case 'dev': - channel = 'dev'; - ver = ''; - break; + let releaseTarget = toReleaseTarget(tag); + let hostFormats = formats; + let terms = unameAgent.split(/[\s\/]+/g); + let hostTarget = {}; + try { + void HostTargets.termsToTarget(hostTarget, terms); + } catch (e) { + // if we can't guarantee the results... + // "in the face of ambiguity, refuse the temptation to guess" + throw e; + } + console.log(`dbg: Installer Host Target:`); + console.log(hostTarget); + + if (!hostTarget.os) { + throw new Error(`OS could not be identified by User-Agent '${unameAgent}'`); } - var myOs = uaDetect.os(ua); - var myArch = uaDetect.arch(ua); - var myLibc; - if (libc) { - myLibc = uaDetect.libc(libc); + console.log(`dbg: Get Project Installer Type for '${projectName}':`); + let proj = await Builds.getProjectType(projectName); + console.log(proj); + + let validTypes = ['alias', 'selfhosted', 'valid']; + if (!validTypes.includes(proj.type)) { + let msg = `'${projectName}' doesn't have an installer: '${proj.type}': '${proj.detail}'`; + let err = new Error(msg); + err.code = 'ENOENT'; + throw err; } - if (!myLibc) { - myLibc = uaDetect.libc(ua); - } - if (!myLibc) { - myLibc = 'libc'; + if (proj.type === 'alias') { + projectName = proj.detail; } - let cfg = await Projects.get(pkg); - let releaseQuery = { - pkg: cfg.alias || pkg, - ver, - os: myOs, - arch: myArch, - libc: myLibc, - lts, - channel, - // TODO use formats for sorting, not exclusion - // (it's better to install xz or report an error to install zip) - formats, + let tmplParams = { + pkg: projectName, + tag: tag, + os: hostTarget.os, + arch: hostTarget.arch, + libc: hostTarget.libc, + formats: hostFormats, limit: 1, }; + Object.assign(tmplParams, releaseTarget); + console.log('tmplParams', tmplParams); - let rels = await Releases.getReleases(releaseQuery); - - var rel = rels.releases[0]; - var opts = { - pkg: cfg.alias || pkg, - ver, - tag, - os: myOs, - arch: myArch, - libc: myLibc, - lts, - channel, - formats, - limit: 1, + let errPackage = { + name: 'doesntexist.ext', + version: '0.0.0', + lts: '-', + channel: 'error', + date: '1970-01-01', + os: hostTarget.os || '-', + arch: hostTarget.arch || '-', + libc: hostTarget.libc || '-', + ext: 'err', + download: 'https://example.com/doesntexist.ext', + comment: + 'No matches found. Could be bad or missing version info' + + ',' + + "Check query parameters. Should be something like '/api/releases/{package}@{version}.tab?os={macos|linux|windows|-}&arch={amd64|x86|aarch64|arm64|armv7l|-}&libc={musl|gnu|msvc|libc|static}&limit=10'", }; - rel = Object.assign( - { - oses: rels.oses, - arches: rels.arches, - libcs: rels.libcs, - formats: rels.formats, - }, - rel, + + if (proj.type === 'selfhosted') { + return [errPackage, tmplParams]; + } + + let projInfo = await Builds.getPackage({ + name: projectName, + date: new Date(), + }); + let latest = projInfo.versions[0]; + Object.assign(tmplParams, { latest }); + //console.log('projInfo', projInfo); + + let buildTargetInfo = { + triplets: projInfo.triplets, + oses: projInfo.oses, + arches: projInfo.arches, + libcs: projInfo.libcs, + formats: projInfo.formats, + }; + + let hasOs = projInfo.oses.includes(hostTarget.os); + if (!hasOs) { + let pkg1 = Object.assign(buildTargetInfo, errPackage); + return [pkg1, tmplParams]; + } + + let targetRelease = Builds.findMatchingPackages( + projInfo, + hostTarget, + releaseTarget, ); + // { triplet: `${os}-${arch}-${libc}`, packages: targetPackages + // , latest: projInfo.versions[0], versions: matchInfo + // } - return [rel, opts]; + if (!targetRelease?.packages) { + let pkg1 = Object.assign(buildTargetInfo, errPackage); + return [pkg1, tmplParams]; + } + + let buildPkg = Builds.selectPackage(targetRelease.packages, hostFormats); + let ext = buildPkg.ext || '.exe'; + if (ext.startsWith('.')) { + ext = ext.slice(1); + } + + let version = targetRelease.version; + if (version.startsWith('v')) { + version = version.slice(1); + } + + buildPkg = Object.assign(buildTargetInfo, buildPkg, { ext, version }); + console.log('dbg: buildPkg', buildPkg); + console.log('dbg: tmplParams', tmplParams); + return [buildPkg, tmplParams]; }; -var CURL_PIPE_PS1_BOOT = path.join(__dirname, 'curl-pipe-bootstrap.tpl.ps1'); -var CURL_PIPE_SH_BOOT = path.join(__dirname, 'curl-pipe-bootstrap.tpl.sh'); +let channelNames = [ + 'stable', + // 'hotfix', + 'latest', + 'rc', + 'preview', + 'pre', + 'dev', + 'beta', + 'alpha', +]; + +function toReleaseTarget(tag) { + tag = tag.replace(/^v/, ''); + + let releaseTarget = { + channel: '', + lts: false, + version: '', + }; + + if (tag === 'lts') { + releaseTarget.lts = true; + releaseTarget.channel = 'stable'; + } else if (channelNames.includes(tag)) { + releaseTarget.channel = tag; + } else { + releaseTarget.version = tag; + } + + return releaseTarget; +} + +var CURL_PIPE_PS1_BOOT = Path.join(__dirname, 'curl-pipe-bootstrap.tpl.ps1'); +var CURL_PIPE_SH_BOOT = Path.join(__dirname, 'curl-pipe-bootstrap.tpl.sh'); var BAD_SH_RE = /[<>'"`$\\]/; InstallerServer.getPosixCurlPipeBootstrap = async function ({ @@ -150,7 +225,6 @@ InstallerServer.getPosixCurlPipeBootstrap = async function ({ let name = env[0]; let value = env[1]; - // TODO create REs once, in higher scope let envRe = new RegExp( `^[ \\t]*#?[ \\t]*(export[ \\t])?[ \\t]*(${name})=.*`, 'm', @@ -198,9 +272,6 @@ InstallerServer.getPwshCurlPipeBootstrap = async function ({ let tplRe = new RegExp(`{{ (${name}) }}`, 'g'); bootTxt = bootTxt.replace(tplRe, `${value}`); - // let envRe = new RegExp(`^[ \\t]*#?[ \\t]*($$${name})[ \\t]*=.*`, 'im'); - // bootTxt = bootTxt.replace(envRe, `$$${name} = '${value}'`); - let setRe = new RegExp( `(#[ \\t]*)?(\\$${name})[ \\t]*=[ \\t]['"].*['"][ \\t]`, 'im', diff --git a/_webi/test.js b/_webi/test.js index 1b6f5dd..a20ef5e 100755 --- a/_webi/test.js +++ b/_webi/test.js @@ -82,8 +82,8 @@ Releases.get(path.join(process.cwd(), pkgdir)).then(async function (all) { var formats = ['exe', 'xz', 'tar', 'zip', 'git']; let [rel, opts] = await ServeInstaller.helper({ - ua: `${nodeOs}/${nodeOsRelease} ${nodeArch}/unknown ${nodeLibc}`, - pkg: pkgname, + unameAgent: `${nodeOs}/${nodeOsRelease} ${nodeArch}/unknown ${nodeLibc}`, + projectName: pkgname, tag: pkgtag || '', formats: formats, libc: nodeLibc,