From fc36f3e93e3137ec10eb9f0dcd9962b57599d62b Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Mon, 6 Nov 2023 01:59:11 +0000 Subject: [PATCH] ref(webi): add explicit libc detection --- _webi/bootstrap.sh | 17 +++++++- _webi/normalize.js | 78 +++++++++++++++++++++++++++++++------ _webi/releases.js | 15 ++++++- _webi/serve-installer.js | 37 ++++++++++++++---- _webi/template.sh | 21 +++++++--- _webi/test.js | 24 ++++++++++-- _webi/transform-releases.js | 43 ++++++++++---------- _webi/ua-detect.js | 52 ++++++++++++++++++++----- 8 files changed, 224 insertions(+), 63 deletions(-) diff --git a/_webi/bootstrap.sh b/_webi/bootstrap.sh index 7168b8c..e1a9a78 100644 --- a/_webi/bootstrap.sh +++ b/_webi/bootstrap.sh @@ -14,9 +14,18 @@ __install_webi() { #WEBI_HOST=https://webinstall.dev export WEBI_HOST + my_libc='' + if ldd /bin/ls 2> /dev/null | grep -q 'musl' 2> /dev/null; then + my_libc='musl' + elif uname -o | grep -q 'GNU' || uname -s | grep -q 'Linux'; then + my_libc='gnu' + else + my_libc='libc' + fi + if test -z "$WEBI_WELCOME"; then echo "" - printf "Thanks for using webi to install '\e[32m%s\e[0m' on '\e[33m%s/%s\e[0m'.\n" "${WEBI_PKG-}" "$(uname -s)/$(uname -r)" "$(uname -m)" + printf "Thanks for using webi to install '\e[32m%s\e[0m' on '\e[33m%s (%s) %s\e[0m'.\n" "${WEBI_PKG:-"Unknown Package"}" "$(uname -s)" "${my_libc:-"Unknown Libc"}" "$(uname -m)" echo "Have a problem? Experience a bug? Please let us know:" printf " \e[2m\e[36mhttps://github.com/webinstall/webi-installers/issues\e[0m\n" echo "" @@ -104,7 +113,11 @@ __webi_main() { my_libc='' if ldd /bin/ls 2> /dev/null | grep -q 'musl' 2> /dev/null; then - my_libc=' musl-native' + my_libc='musl' + elif uname -o | grep -q 'GNU' || uname -s | grep -q 'Linux'; then + my_libc='gnu' + else + my_libc='libc' fi export WEBI_HOST="\${WEBI_HOST:-https://webinstall.dev}" diff --git a/_webi/normalize.js b/_webi/normalize.js index 1f7e912..90e6cae 100644 --- a/_webi/normalize.js +++ b/_webi/normalize.js @@ -13,6 +13,7 @@ var osMap = { var maps = { oses: {}, arches: {}, + libcs: {}, formats: {}, }; @@ -65,11 +66,18 @@ arches.forEach(function (name) { maps.arches[name] = true; }); +var libcs = ['none', 'musl', 'gnu', 'msvc', 'libc']; +libcs.forEach(function (name) { + maps.libcs[name] = true; +}); + function normalize(all) { - /* jshint maxcomplexity:26 */ + /* jshint maxcomplexity:50 */ + /* jshint maxdepth:10 */ var supported = { oses: {}, arches: {}, + libcs: {}, formats: {}, }; @@ -94,16 +102,6 @@ function normalize(all) { } } } - - // Hacky-doo for musl - // TODO 'libc' some sort of glibc vs musl tag? - if (!rel._musl_native) { - if (!rel._musl) { - if (/(\b|\.|_|-)(musl)(\b|\.|_|-)/.test(rel.download)) { - rel._musl = true; - } - } - } supported.oses[rel.os] = true; if (!rel.arch) { @@ -123,6 +121,60 @@ function normalize(all) { } supported.arches[rel.arch] = true; + // note: depends on rel.os + if (!rel.libc) { + let isMusl; + let isMsvc; + let isStatic; + let isGnu; + + // extra blocks to prevent copy pasta errors + + { + let muslRe = /(\b|_)(musl)(\b|_)/i; + isMusl = muslRe.test(rel.download) || muslRe.test(rel.name); + } + + { + let msvcRe = /(\b|_)(msvc)(\b|_)/i; + isMsvc = msvcRe.test(rel.download) || msvcRe.test(rel.name); + } + + { + let staticRe = /(\b|_)(static)(\b|_)/i; + isStatic = staticRe.test(rel.download) || staticRe.test(rel.name); + } + + { + let gnuRe = /(\b|_)(gnu|glibc|libc)(\b|_)/i; + isGnu = gnuRe.test(rel.download) || gnuRe.test(rel.name); + } + + if (isMusl) { + // we specifically tag things that need musl++ in their own releases + rel.libc = 'none'; + } else if (isStatic) { + rel.libc = 'none'; + } else if (isGnu) { + rel.libc = 'gnu'; + if (rel.os === 'windows') { + // windows gnu is static + rel.libc = 'none'; + } else if (rel.os === 'darwin') { + // if glibc is required on macos, it'll be static + rel.libc = 'none'; + } + } else if (isMsvc) { + rel.libc = 'msvc'; + } else { + // The default is no requirement for any particular libc + // (Go, Zig, POSIX Shell, JS, etc) + // and hopefully we never have to worry about mingw and friends + rel.libc = 'none'; + } + } + supported.libcs[rel.libc] = true; + var tarExt; if (!rel.ext) { // pkg-v1.0.tar.gz => ['gz', 'tar', '0', 'pkg-v1'] @@ -179,6 +231,9 @@ function normalize(all) { all.arches = Object.keys(supported.arches).filter(function (name) { return maps.arches[name]; }); + all.libcs = Object.keys(supported.libcs).filter(function (name) { + return maps.libcs[name]; + }); all.formats = Object.keys(supported.formats).filter(function (name) { return maps.formats[name]; }); @@ -199,4 +254,5 @@ module.exports._debug = function (all) { // NOT in order of priority (which would be tar, xz, zip, ...) module.exports.formats = formats; module.exports.arches = arches; +module.exports.libcs = libcs; module.exports.formatsMap = maps.formats; diff --git a/_webi/releases.js b/_webi/releases.js index d881306..cd508e9 100644 --- a/_webi/releases.js +++ b/_webi/releases.js @@ -30,7 +30,7 @@ function padScript(txt) { Releases.renderBash = async function ( pkgdir, rel, - { baseurl, pkg, tag, ver, os = '', arch = '', formats }, + { baseurl, pkg, tag, ver, os = '', arch = '', libc = '', formats }, ) { if (!Array.isArray(formats)) { formats = []; @@ -70,6 +70,7 @@ Releases.renderBash = async function ( .replace(/^\s*#?WEBI_HOST=.*/m, `WEBI_HOST='${baseurl}'`) .replace(/^\s*#?WEBI_OS=.*/m, `WEBI_OS='${os}'`) .replace(/^\s*#?WEBI_ARCH=.*/m, `WEBI_ARCH='${arch}'`) + .replace(/^\s*#?WEBI_LIBC=.*/m, `WEBI_LIBC='${libc}'`) .replace(/^\s*#?WEBI_TAG=.*/m, `WEBI_TAG='${tag}'`) .replace( /^\s*#?WEBI_RELEASES=.*/m, @@ -83,6 +84,8 @@ Releases.renderBash = async function ( rel.os + '&arch=' + rel.arch + + '&libc=' + + rel.libc + '&formats=' + formats.join(',') + '&pretty=true' + @@ -142,6 +145,10 @@ Releases.renderBash = async function ( /^\s*#?PKG_ARCHES=.*/m, "PKG_ARCHES='" + ((rel && rel.arches) || []).join(',') + "'", ) + .replace( + /^\s*#?PKG_LIBCS=.*/m, + "PKG_LIBCS='" + ((rel && rel.libcs) || []).join(',') + "'", + ) .replace( /^\s*#?PKG_FORMATS=.*/m, "PKG_FORMATS='" + ((rel && rel.formats) || []).join(',') + "'", @@ -157,7 +164,7 @@ Releases.renderBash = async function ( Releases.renderPowerShell = async function ( pkgdir, rel, - { baseurl, pkg, tag, ver, os, arch, formats }, + { baseurl, pkg, tag, ver, os, arch, libc = '', formats }, ) { if (!Array.isArray(formats)) { formats = []; @@ -189,6 +196,10 @@ Releases.renderPowerShell = async function ( var pkgver = pkg + '@' + ver; return ( tplTxt + .replace( + /^(#)?\$Env:WEBI_LIBC\s*=.*/im, + "$Env:WEBI_LIBC = '" + libc + "'", + ) .replace( /^(#)?\$Env:WEBI_HOST\s*=.*/im, "$Env:WEBI_HOST = '" + baseurl + "'", diff --git a/_webi/serve-installer.js b/_webi/serve-installer.js index ba72d27..9c9a24c 100644 --- a/_webi/serve-installer.js +++ b/_webi/serve-installer.js @@ -13,9 +13,10 @@ var installersDir = path.join(__dirname, '..'); serveInstaller.serveInstaller = serveInstaller; module.exports = serveInstaller; -async function serveInstaller(baseurl, ua, pkg, tag, ext, formats) { +async function serveInstaller(baseurl, ua, pkg, tag, ext, formats, libc) { // TODO put some of this in a middleware? or common function? + // TODO maybe move package/version/lts/channel detection into getReleases var ver = tag.replace(/^v/, ''); var lts; var channel; @@ -44,12 +45,21 @@ async function serveInstaller(baseurl, ua, pkg, tag, ext, formats) { break; } - // TODO maybe move package/version/lts/channel detection into getReleases var myOs = uaDetect.os(ua); var myArch = uaDetect.arch(ua); - var myLibc = uaDetect.libc(ua); + var myLibc; + if (libc) { + myLibc = uaDetect.libc(libc); + } + if (!myLibc) { + myLibc = uaDetect.libc(ua); + } + if (!myLibc) { + myLibc = 'libc'; + } + let cfg = await packages.get(pkg); - let rels = await getReleases({ + let releaseQuery = { pkg: cfg.alias || pkg, ver, os: myOs, @@ -57,9 +67,13 @@ async function serveInstaller(baseurl, ua, pkg, tag, ext, formats) { libc: myLibc, lts, channel, + // TODO use formats for sorting, not exclusion + // (it's better to install xz or report an error to install zip) formats, limit: 1, - }); + }; + + let rels = await getReleases(releaseQuery); var rel = rels.releases[0]; var pkgdir = path.join(installersDir, pkg); @@ -70,14 +84,21 @@ async function serveInstaller(baseurl, ua, pkg, tag, ext, formats) { tag, os: myOs, arch: myArch, + libc: myLibc, lts, channel, formats, limit: 1, }; - rel.oses = rels.oses; - rel.arches = rels.arches; - rel.formats = rels.formats; + rel = Object.assign( + { + oses: rels.oses, + arches: rels.arches, + libcs: rels.libcs, + formats: rels.formats, + }, + rel, + ); if ('ps1' === ext) { return Releases.renderPowerShell(pkgdir, rel, opts); diff --git a/_webi/template.sh b/_webi/template.sh index b55cb76..2a7d9f3 100644 --- a/_webi/template.sh +++ b/_webi/template.sh @@ -8,13 +8,18 @@ __bootstrap_webi() { my_libc='' if ldd /bin/ls 2> /dev/null | grep -q 'musl' 2> /dev/null; then - my_libc=' musl-native' + my_libc='musl' + elif uname -o | grep -q 'GNU' || uname -s | grep -q 'Linux'; then + my_libc='gnu' + else + my_libc='libc' fi #WEBI_PKG= #PKG_NAME= #WEBI_OS= #WEBI_ARCH= + #WEBI_LIBC= #WEBI_HOST= #WEBI_RELEASES= #WEBI_CSV= @@ -35,8 +40,9 @@ __bootstrap_webi() { #WEBI_PKG_PATHNAME= #PKG_OSES= #PKG_ARCHES= + #PKG_LIBCS= #PKG_FORMATS= - WEBI_UA="$(uname -s)/$(uname -r) $(uname -m)/unknown${my_libc}" + WEBI_UA="$(uname -s)/$(uname -r) $(uname -m)/unknown ${my_libc}" WEBI_PKG_DOWNLOAD="" WEBI_DOWNLOAD_DIR="${HOME}/Downloads" if command -v xdg-user-dir > /dev/null; then @@ -164,15 +170,20 @@ __bootstrap_webi() { return 0 fi - echo >&2 "Error: no '$PKG_NAME' release for '${WEBI_OS-}' on '$WEBI_ARCH' as one of '$WEBI_FORMATS' by the tag '${WEBI_TAG-}'" - echo >&2 " '$PKG_NAME' is available for '$PKG_OSES' on '$PKG_ARCHES' as one of '$PKG_FORMATS'" + echo >&2 "Error: no '${PKG_NAME:-"Unknown Package"}@${WEBI_TAG:-"Unknown Tag"}' release for '${WEBI_OS:-"Unknown OS"}' (${WEBI_LIBC:-"Unknown Libc"}) on '${WEBI_ARCH:-"Unknown CPU"}' as one of '${WEBI_FORMATS:-"Unknown File Type"}'" + echo >&2 " '$PKG_NAME' is available for '$PKG_OSES' ($PKG_LIBCS) on '$PKG_ARCHES' as one of '$PKG_FORMATS'" echo >&2 " (check that the package name and version are correct)" echo >&2 "" my_release_url="$( echo "$WEBI_RELEASES" | sed 's:?.*::' )" + my_release_params="$( + echo "$WEBI_RELEASES" | + sed 's:.*?:?:' + )" echo >&2 " Double check at ${my_release_url}" + echo >&2 " ${my_release_params}" echo >&2 "" exit 1 @@ -548,7 +559,7 @@ __bootstrap_webi() { if [ -z "${WEBI_WELCOME-}" ]; then echo "" - printf "Thanks for using webi to install '\e[32m%s\e[0m' on '\e[33m%s/%s\e[0m'.\n" "${WEBI_PKG-}" "$(uname -s)" "$(uname -m)" + printf "Thanks for using webi to install '\e[32m%s\e[0m' on '\e[33m%s (%s) %s\e[0m'.\n" "${WEBI_PKG:-"Unknown Package"}" "$(uname -s)" "${WEBI_LIBC:-"Unknown Libc"}" "$(uname -m)" echo "Have a problem? Experience a bug? Please let us know:" printf " \e[2m\e[36mhttps://github.com/webinstall/webi-installers/issues\e[0m\n" echo "" diff --git a/_webi/test.js b/_webi/test.js index 3ff1da0..376c6bd 100755 --- a/_webi/test.js +++ b/_webi/test.js @@ -66,6 +66,7 @@ Releases.get(path.join(process.cwd(), pkgdir)).then(function (all) { var pkgname = path.basename(pkgdir.replace(/\/$/, '')); var osrel = os.platform() + '-' + os.release(); var arch = os.arch(); + var libc = 'libc'; var formats = ['exe', 'xz', 'tar', 'zip', 'git']; var rel; @@ -89,6 +90,13 @@ Releases.get(path.join(process.cwd(), pkgdir)).then(function (all) { } } + if (_rel.libc !== 'none') { + let curLibc = uaDetect.libc(libc); + if (_rel.libc !== curLibc) { + continue; + } + } + if (_rel.arch !== '*') { let curArch = uaDetect.arch(arch); if (_rel.arch !== curArch) { @@ -118,9 +126,15 @@ Releases.get(path.join(process.cwd(), pkgdir)).then(function (all) { return; } - rel.oses = all.oses; - rel.arches = all.arches; - rel.formats = all.formats; + rel = Object.assign( + { + oses: all.oses, + arches: all.arches, + libcs: all.libcs, + formats: all.formats, + }, + rel, + ); console.info(''); console.info('Found release matching current os, arch, and tag:'); @@ -169,5 +183,9 @@ Releases.get(path.join(process.cwd(), pkgdir)).then(function (all) { console.info('\tNEEDS MANUAL TEST: powershell.exe %s', ps1File); } console.info(''); + setTimeout(function () { + console.warn(`[warn] dangling event loop handle`); + process.exit(0); + }, 300).unref(); }); }); diff --git a/_webi/transform-releases.js b/_webi/transform-releases.js index 2870a15..d9dbafe 100644 --- a/_webi/transform-releases.js +++ b/_webi/transform-releases.js @@ -2,7 +2,6 @@ var path = require('path'); var Releases = require('./releases.js'); -var uaDetect = require('./ua-detect.js'); var cache = {}; //var staleAge = 5 * 1000; @@ -54,21 +53,14 @@ function createFormatsSorter(formats) { return 1; } - // Hacky-doo for musl-native: prefer non-musl - if (a._musl_native && !b._musl_native) { - return 1; - } - if (!a._musl_native && b._musl_native) { + // rank builds that don't depend on any form of libc first + if (a.libc === 'none' && b.libc !== 'none') { return -1; } + if (a.libc !== 'none' && b.libc === 'none') { + return 1; + } - // Hacky-doo for linux: prefer musl - if (a._musl && !b._musl) { - return -1; - } - if (!a._musl && b._musl) { - return 1; - } return 0; }; } @@ -197,13 +189,20 @@ async function filterReleases( } } - // Hacky-doo for linux musl - if (libc === uaDetect.MUSL_NATIVE) { - if (!rel._musl && !rel._musl_native) { + if (rel.libc !== 'none') { + let releaseRequiresMusl = rel.libc === 'musl'; + // goal: handle non-glibc (Alpine / Docker / musl) + let osHasMusl = libc === 'musl'; + if (osHasMusl) { + // goal: fail if dependent on libc + let releaseRequiresLibc = rel.libc === 'gnu'; + if (releaseRequiresLibc) { + return false; + } + } else if (releaseRequiresMusl) { + // goal: don't use musl++ on glibc (Ubuntu, GNU, etc) return false; } - } else if (rel._musl_native) { - return false; } if (lts) { @@ -363,20 +362,20 @@ module.exports = function getReleases({ date: '1970-01-01', os: os || '-', arch: arch || '-', - _musl: undefined, - _musl_native: undefined, + libc: libc || '-', ext: 'err', download: 'https://example.com/doesntexist.ext', comment: 'No matches found. Could be bad or missing version info' + ',' + - "Check query parameters. Should be something like '/api/releases/{package}@{version}.tab?os={macos|linux|windows|-}&arch={amd64|x86|aarch64|arm64|armv7l|-}&limit=100'", + "Check query parameters. Should be something like '/api/releases/{package}@{version}.tab?os={macos|linux|windows|-}&arch={amd64|x86|aarch64|arm64|armv7l|-}&libc={musl|gnu|msvc|libc|static}&limit=10'", }, ]; } return { oses: all.oses, arches: all.arches, + libcs: all.libcs, formats: all.formats, releases: releases, }; @@ -392,7 +391,7 @@ if (require.main === module) { os: 'macos', arch: 'amd64', lts: true, - libc: '', + libc: 'libc', channel: 'stable', formats: ['tar', 'exe', 'zip', 'xz', 'dmg', 'pkg'], limit: 10, diff --git a/_webi/ua-detect.js b/_webi/ua-detect.js index 6b4168e..8b97f58 100644 --- a/_webi/ua-detect.js +++ b/_webi/ua-detect.js @@ -2,10 +2,6 @@ var uaDetect = module.exports; -const MUSL_NATIVE = 'musl-native'; - -uaDetect.MUSL_NATIVE = MUSL_NATIVE; - function getRequest(req) { var ua = req.headers['user-agent'] || ''; var os = req.query.os; @@ -100,19 +96,55 @@ function getLibc(ua) { return '-'; } - // Use native 'libc' information, if provided - // + // How to see a bunch of target host quadruples: + // go tool dist list + // rustup target list + // zig targets | jq -r '.libc[]' | sort - + // Generally, we prefer 'musl' builds because they DO work on glibc systems (Ubuntu), // but 'glibc' builds will NOT work on musl systems (Alpine / Docker). // // However, there are a few instances (ex: Node.js), where the 'musl' builds // DO NOT work on glibc systems. - if (ua.match(MUSL_NATIVE)) { - return MUSL_NATIVE; + + { + let muslRe = /(\b|_)(musl)(\b|_)/i; + if (muslRe.test(ua)) { + return 'musl'; + } } - // TODO handle explicit invalid different - return ''; + { + let msvcRe = /(\b|_)(msvc|windows|microsoft)(\b|_)/i; + if (msvcRe.test(ua)) { + return 'msvc'; + } + } + + { + let gnuRe = /(\b|_)(gnu|glibc|linux)(\b|_)/i; + if (gnuRe.test(ua)) { + return 'gnu'; + } + } + + { + let libcRe = /(\b|_)(libc)(\b|_)/i; + if (libcRe.test(ua)) { + return 'libc'; + } + } + + { + let darwinRe = /(\b|_)(darwin)(\b|_)/i; + if (darwinRe.test(ua)) { + // not sure whether this should be "darwin" or "libc" or "none" + // https://opensource.apple.com/source/Libc/ + return 'libc'; + } + } + + return 'libc'; } uaDetect.os = getOs;