github/vim-ale
1
0
Fork 0
mirror of https://github.com/webinstall/webi-installers.git synced 2026-08-23 22:16:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

ref(webi): add explicit libc detection

Browse Source
This commit is contained in:
AJ ONeal
2023-11-06 18:50:02 -07:00
parent ffa49401aa
commit fc36f3e93e
8 changed files with 224 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
_webi/bootstrap.sh
+15 -2
View File
@@ -14,9 +14,18 @@ __install_webi() {
#WEBI_HOST=https://webinstall.dev
export WEBI_HOST
my_libc=''
if ldd /bin/ls 2> /dev/null | grep -q 'musl' 2> /dev/null; then
my_libc='musl'
elif uname -o | grep -q 'GNU' || uname -s | grep -q 'Linux'; then
my_libc='gnu'
else
my_libc='libc'
fi
if test -z "$WEBI_WELCOME"; then
echo ""
printf "Thanks for using webi to install '\e[32m%s\e[0m' on '\e[33m%s/%s\e[0m'.\n" "${WEBI_PKG-}" "$(uname -s)/$(uname -r)" "$(uname -m)"
printf "Thanks for using webi to install '\e[32m%s\e[0m' on '\e[33m%s (%s) %s\e[0m'.\n" "${WEBI_PKG:-"Unknown Package"}" "$(uname -s)" "${my_libc:-"Unknown Libc"}" "$(uname -m)"
echo "Have a problem? Experience a bug? Please let us know:"
printf " \e[2m\e[36mhttps://github.com/webinstall/webi-installers/issues\e[0m\n"
echo ""
@@ -104,7 +113,11 @@ __webi_main() {
my_libc=''
if ldd /bin/ls 2> /dev/null | grep -q 'musl' 2> /dev/null; then
my_libc=' musl-native'
my_libc='musl'
elif uname -o | grep -q 'GNU' || uname -s | grep -q 'Linux'; then
my_libc='gnu'
else
my_libc='libc'
fi
export WEBI_HOST="\${WEBI_HOST:-https://webinstall.dev}"
_webi/normalize.js
+67 -11
View File
@@ -13,6 +13,7 @@ var osMap = {
var maps = {
oses: {},
arches: {},
libcs: {},
formats: {},
};
@@ -65,11 +66,18 @@ arches.forEach(function (name) {
maps.arches[name] = true;
});
var libcs = ['none', 'musl', 'gnu', 'msvc', 'libc'];
libcs.forEach(function (name) {
maps.libcs[name] = true;
});
function normalize(all) {
/* jshint maxcomplexity:26 */
/* jshint maxcomplexity:50 */
/* jshint maxdepth:10 */
var supported = {
oses: {},
arches: {},
libcs: {},
formats: {},
};
@@ -94,16 +102,6 @@ function normalize(all) {
}
}
}
// Hacky-doo for musl
// TODO 'libc' some sort of glibc vs musl tag?
if (!rel._musl_native) {
if (!rel._musl) {
if (/(\b|\.|_|-)(musl)(\b|\.|_|-)/.test(rel.download)) {
rel._musl = true;
}
}
}
supported.oses[rel.os] = true;
if (!rel.arch) {
@@ -123,6 +121,60 @@ function normalize(all) {
}
supported.arches[rel.arch] = true;
// note: depends on rel.os
if (!rel.libc) {
let isMusl;
let isMsvc;
let isStatic;
let isGnu;
// extra blocks to prevent copy pasta errors
{
let muslRe = /(\b|_)(musl)(\b|_)/i;
isMusl = muslRe.test(rel.download) || muslRe.test(rel.name);
}
{
let msvcRe = /(\b|_)(msvc)(\b|_)/i;
isMsvc = msvcRe.test(rel.download) || msvcRe.test(rel.name);
}
{
let staticRe = /(\b|_)(static)(\b|_)/i;
isStatic = staticRe.test(rel.download) || staticRe.test(rel.name);
}
{
let gnuRe = /(\b|_)(gnu|glibc|libc)(\b|_)/i;
isGnu = gnuRe.test(rel.download) || gnuRe.test(rel.name);
}
if (isMusl) {
// we specifically tag things that need musl++ in their own releases
rel.libc = 'none';
} else if (isStatic) {
rel.libc = 'none';
} else if (isGnu) {
rel.libc = 'gnu';
if (rel.os === 'windows') {
// windows gnu is static
rel.libc = 'none';
} else if (rel.os === 'darwin') {
// if glibc is required on macos, it'll be static
rel.libc = 'none';
}
} else if (isMsvc) {
rel.libc = 'msvc';
} else {
// The default is no requirement for any particular libc
// (Go, Zig, POSIX Shell, JS, etc)
// and hopefully we never have to worry about mingw and friends
rel.libc = 'none';
}
}
supported.libcs[rel.libc] = true;
var tarExt;
if (!rel.ext) {
// pkg-v1.0.tar.gz => ['gz', 'tar', '0', 'pkg-v1']
@@ -179,6 +231,9 @@ function normalize(all) {
all.arches = Object.keys(supported.arches).filter(function (name) {
return maps.arches[name];
});
all.libcs = Object.keys(supported.libcs).filter(function (name) {
return maps.libcs[name];
});
all.formats = Object.keys(supported.formats).filter(function (name) {
return maps.formats[name];
});
@@ -199,4 +254,5 @@ module.exports._debug = function (all) {
// NOT in order of priority (which would be tar, xz, zip, ...)
module.exports.formats = formats;
module.exports.arches = arches;
module.exports.libcs = libcs;
module.exports.formatsMap = maps.formats;
_webi/releases.js
+13 -2
View File
@@ -30,7 +30,7 @@ function padScript(txt) {
Releases.renderBash = async function (
pkgdir,
rel,
{ baseurl, pkg, tag, ver, os = '', arch = '', formats },
{ baseurl, pkg, tag, ver, os = '', arch = '', libc = '', formats },
) {
if (!Array.isArray(formats)) {
formats = [];
@@ -70,6 +70,7 @@ Releases.renderBash = async function (
.replace(/^\s*#?WEBI_HOST=.*/m, `WEBI_HOST='${baseurl}'`)
.replace(/^\s*#?WEBI_OS=.*/m, `WEBI_OS='${os}'`)
.replace(/^\s*#?WEBI_ARCH=.*/m, `WEBI_ARCH='${arch}'`)
.replace(/^\s*#?WEBI_LIBC=.*/m, `WEBI_LIBC='${libc}'`)
.replace(/^\s*#?WEBI_TAG=.*/m, `WEBI_TAG='${tag}'`)
.replace(
/^\s*#?WEBI_RELEASES=.*/m,
@@ -83,6 +84,8 @@ Releases.renderBash = async function (
rel.os +
'&arch=' +
rel.arch +
'&libc=' +
rel.libc +
'&formats=' +
formats.join(',') +
'&pretty=true' +
@@ -142,6 +145,10 @@ Releases.renderBash = async function (
/^\s*#?PKG_ARCHES=.*/m,
"PKG_ARCHES='" + ((rel && rel.arches) || []).join(',') + "'",
)
.replace(
/^\s*#?PKG_LIBCS=.*/m,
"PKG_LIBCS='" + ((rel && rel.libcs) || []).join(',') + "'",
)
.replace(
/^\s*#?PKG_FORMATS=.*/m,
"PKG_FORMATS='" + ((rel && rel.formats) || []).join(',') + "'",
@@ -157,7 +164,7 @@ Releases.renderBash = async function (
Releases.renderPowerShell = async function (
pkgdir,
rel,
{ baseurl, pkg, tag, ver, os, arch, formats },
{ baseurl, pkg, tag, ver, os, arch, libc = '', formats },
) {
if (!Array.isArray(formats)) {
formats = [];
@@ -189,6 +196,10 @@ Releases.renderPowerShell = async function (
var pkgver = pkg + '@' + ver;
return (
tplTxt
.replace(
/^(#)?\$Env:WEBI_LIBC\s*=.*/im,
"$Env:WEBI_LIBC = '" + libc + "'",
)
.replace(
/^(#)?\$Env:WEBI_HOST\s*=.*/im,
"$Env:WEBI_HOST = '" + baseurl + "'",
_webi/serve-installer.js
+29 -8
View File
@@ -13,9 +13,10 @@ var installersDir = path.join(__dirname, '..');
serveInstaller.serveInstaller = serveInstaller;
module.exports = serveInstaller;
async function serveInstaller(baseurl, ua, pkg, tag, ext, formats) {
async function serveInstaller(baseurl, ua, pkg, tag, ext, formats, libc) {
// TODO put some of this in a middleware? or common function?
// TODO maybe move package/version/lts/channel detection into getReleases
var ver = tag.replace(/^v/, '');
var lts;
var channel;
@@ -44,12 +45,21 @@ async function serveInstaller(baseurl, ua, pkg, tag, ext, formats) {
break;
}
// TODO maybe move package/version/lts/channel detection into getReleases
var myOs = uaDetect.os(ua);
var myArch = uaDetect.arch(ua);
var myLibc = uaDetect.libc(ua);
var myLibc;
if (libc) {
myLibc = uaDetect.libc(libc);
}
if (!myLibc) {
myLibc = uaDetect.libc(ua);
}
if (!myLibc) {
myLibc = 'libc';
}
let cfg = await packages.get(pkg);
let rels = await getReleases({
let releaseQuery = {
pkg: cfg.alias || pkg,
ver,
os: myOs,
@@ -57,9 +67,13 @@ async function serveInstaller(baseurl, ua, pkg, tag, ext, formats) {
libc: myLibc,
lts,
channel,
// TODO use formats for sorting, not exclusion
// (it's better to install xz or report an error to install zip)
formats,
limit: 1,
});
};
let rels = await getReleases(releaseQuery);
var rel = rels.releases[0];
var pkgdir = path.join(installersDir, pkg);
@@ -70,14 +84,21 @@ async function serveInstaller(baseurl, ua, pkg, tag, ext, formats) {
tag,
os: myOs,
arch: myArch,
libc: myLibc,
lts,
channel,
formats,
limit: 1,
};
rel.oses = rels.oses;
rel.arches = rels.arches;
rel.formats = rels.formats;
rel = Object.assign(
{
oses: rels.oses,
arches: rels.arches,
libcs: rels.libcs,
formats: rels.formats,
},
rel,
);
if ('ps1' === ext) {
return Releases.renderPowerShell(pkgdir, rel, opts);
_webi/template.sh
+16 -5
View File
@@ -8,13 +8,18 @@ __bootstrap_webi() {
my_libc=''
if ldd /bin/ls 2> /dev/null | grep -q 'musl' 2> /dev/null; then
my_libc=' musl-native'
my_libc='musl'
elif uname -o | grep -q 'GNU' || uname -s | grep -q 'Linux'; then
my_libc='gnu'
else
my_libc='libc'
fi
#WEBI_PKG=
#PKG_NAME=
#WEBI_OS=
#WEBI_ARCH=
#WEBI_LIBC=
#WEBI_HOST=
#WEBI_RELEASES=
#WEBI_CSV=
@@ -35,8 +40,9 @@ __bootstrap_webi() {
#WEBI_PKG_PATHNAME=
#PKG_OSES=
#PKG_ARCHES=
#PKG_LIBCS=
#PKG_FORMATS=
WEBI_UA="$(uname -s)/$(uname -r) $(uname -m)/unknown${my_libc}"
WEBI_UA="$(uname -s)/$(uname -r) $(uname -m)/unknown ${my_libc}"
WEBI_PKG_DOWNLOAD=""
WEBI_DOWNLOAD_DIR="${HOME}/Downloads"
if command -v xdg-user-dir > /dev/null; then
@@ -164,15 +170,20 @@ __bootstrap_webi() {
return 0
fi
echo >&2 "Error: no '$PKG_NAME' release for '${WEBI_OS-}' on '$WEBI_ARCH' as one of '$WEBI_FORMATS' by the tag '${WEBI_TAG-}'"
echo >&2 " '$PKG_NAME' is available for '$PKG_OSES' on '$PKG_ARCHES' as one of '$PKG_FORMATS'"
echo >&2 "Error: no '${PKG_NAME:-"Unknown Package"}@${WEBI_TAG:-"Unknown Tag"}' release for '${WEBI_OS:-"Unknown OS"}' (${WEBI_LIBC:-"Unknown Libc"}) on '${WEBI_ARCH:-"Unknown CPU"}' as one of '${WEBI_FORMATS:-"Unknown File Type"}'"
echo >&2 " '$PKG_NAME' is available for '$PKG_OSES' ($PKG_LIBCS) on '$PKG_ARCHES' as one of '$PKG_FORMATS'"
echo >&2 " (check that the package name and version are correct)"
echo >&2 ""
my_release_url="$(
echo "$WEBI_RELEASES" |
sed 's:?.*::'
)"
my_release_params="$(
echo "$WEBI_RELEASES" |
sed 's:.*?:?:'
)"
echo >&2 " Double check at ${my_release_url}"
echo >&2 " ${my_release_params}"
echo >&2 ""
exit 1
@@ -548,7 +559,7 @@ __bootstrap_webi() {
if [ -z "${WEBI_WELCOME-}" ]; then
echo ""
printf "Thanks for using webi to install '\e[32m%s\e[0m' on '\e[33m%s/%s\e[0m'.\n" "${WEBI_PKG-}" "$(uname -s)" "$(uname -m)"
printf "Thanks for using webi to install '\e[32m%s\e[0m' on '\e[33m%s (%s) %s\e[0m'.\n" "${WEBI_PKG:-"Unknown Package"}" "$(uname -s)" "${WEBI_LIBC:-"Unknown Libc"}" "$(uname -m)"
echo "Have a problem? Experience a bug? Please let us know:"
printf " \e[2m\e[36mhttps://github.com/webinstall/webi-installers/issues\e[0m\n"
echo ""
_webi/test.js
+21 -3
View File
@@ -66,6 +66,7 @@ Releases.get(path.join(process.cwd(), pkgdir)).then(function (all) {
var pkgname = path.basename(pkgdir.replace(/\/$/, ''));
var osrel = os.platform() + '-' + os.release();
var arch = os.arch();
var libc = 'libc';
var formats = ['exe', 'xz', 'tar', 'zip', 'git'];
var rel;
@@ -89,6 +90,13 @@ Releases.get(path.join(process.cwd(), pkgdir)).then(function (all) {
}
}
if (_rel.libc !== 'none') {
let curLibc = uaDetect.libc(libc);
if (_rel.libc !== curLibc) {
continue;
}
}
if (_rel.arch !== '*') {
let curArch = uaDetect.arch(arch);
if (_rel.arch !== curArch) {
@@ -118,9 +126,15 @@ Releases.get(path.join(process.cwd(), pkgdir)).then(function (all) {
return;
}
rel.oses = all.oses;
rel.arches = all.arches;
rel.formats = all.formats;
rel = Object.assign(
{
oses: all.oses,
arches: all.arches,
libcs: all.libcs,
formats: all.formats,
},
rel,
);
console.info('');
console.info('Found release matching current os, arch, and tag:');
@@ -169,5 +183,9 @@ Releases.get(path.join(process.cwd(), pkgdir)).then(function (all) {
console.info('\tNEEDS MANUAL TEST: powershell.exe %s', ps1File);
}
console.info('');
setTimeout(function () {
console.warn(`[warn] dangling event loop handle`);
process.exit(0);
}, 300).unref();
});
});
_webi/transform-releases.js
+21 -22
View File
@@ -2,7 +2,6 @@
var path = require('path');
var Releases = require('./releases.js');
var uaDetect = require('./ua-detect.js');
var cache = {};
//var staleAge = 5 * 1000;
@@ -54,21 +53,14 @@ function createFormatsSorter(formats) {
return 1;
}
// Hacky-doo for musl-native: prefer non-musl
if (a._musl_native && !b._musl_native) {
return 1;
}
if (!a._musl_native && b._musl_native) {
// rank builds that don't depend on any form of libc first
if (a.libc === 'none' && b.libc !== 'none') {
return -1;
}
if (a.libc !== 'none' && b.libc === 'none') {
return 1;
}
// Hacky-doo for linux: prefer musl
if (a._musl && !b._musl) {
return -1;
}
if (!a._musl && b._musl) {
return 1;
}
return 0;
};
}
@@ -197,13 +189,20 @@ async function filterReleases(
}
}
// Hacky-doo for linux musl
if (libc === uaDetect.MUSL_NATIVE) {
if (!rel._musl && !rel._musl_native) {
if (rel.libc !== 'none') {
let releaseRequiresMusl = rel.libc === 'musl';
// goal: handle non-glibc (Alpine / Docker / musl)
let osHasMusl = libc === 'musl';
if (osHasMusl) {
// goal: fail if dependent on libc
let releaseRequiresLibc = rel.libc === 'gnu';
if (releaseRequiresLibc) {
return false;
}
} else if (releaseRequiresMusl) {
// goal: don't use musl++ on glibc (Ubuntu, GNU, etc)
return false;
}
} else if (rel._musl_native) {
return false;
}
if (lts) {
@@ -363,20 +362,20 @@ module.exports = function getReleases({
date: '1970-01-01',
os: os || '-',
arch: arch || '-',
_musl: undefined,
_musl_native: undefined,
libc: libc || '-',
ext: 'err',
download: 'https://example.com/doesntexist.ext',
comment:
'No matches found. Could be bad or missing version info' +
',' +
"Check query parameters. Should be something like '/api/releases/{package}@{version}.tab?os={macos|linux|windows|-}&arch={amd64|x86|aarch64|arm64|armv7l|-}&limit=100'",
"Check query parameters. Should be something like '/api/releases/{package}@{version}.tab?os={macos|linux|windows|-}&arch={amd64|x86|aarch64|arm64|armv7l|-}&libc={musl|gnu|msvc|libc|static}&limit=10'",
},
];
}
return {
oses: all.oses,
arches: all.arches,
libcs: all.libcs,
formats: all.formats,
releases: releases,
};
@@ -392,7 +391,7 @@ if (require.main === module) {
os: 'macos',
arch: 'amd64',
lts: true,
libc: '',
libc: 'libc',
channel: 'stable',
formats: ['tar', 'exe', 'zip', 'xz', 'dmg', 'pkg'],
limit: 10,
_webi/ua-detect.js
+42 -10
View File
@@ -2,10 +2,6 @@
var uaDetect = module.exports;
const MUSL_NATIVE = 'musl-native';
uaDetect.MUSL_NATIVE = MUSL_NATIVE;
function getRequest(req) {
var ua = req.headers['user-agent'] || '';
var os = req.query.os;
@@ -100,19 +96,55 @@ function getLibc(ua) {
return '-';
}
// Use native 'libc' information, if provided
//
// How to see a bunch of target host quadruples:
// go tool dist list
// rustup target list
// zig targets | jq -r '.libc[]' | sort -
// Generally, we prefer 'musl' builds because they DO work on glibc systems (Ubuntu),
// but 'glibc' builds will NOT work on musl systems (Alpine / Docker).
//
// However, there are a few instances (ex: Node.js), where the 'musl' builds
// DO NOT work on glibc systems.
if (ua.match(MUSL_NATIVE)) {
return MUSL_NATIVE;
{
let muslRe = /(\b|_)(musl)(\b|_)/i;
if (muslRe.test(ua)) {
return 'musl';
}
}
// TODO handle explicit invalid different
return '';
{
let msvcRe = /(\b|_)(msvc|windows|microsoft)(\b|_)/i;
if (msvcRe.test(ua)) {
return 'msvc';
}
}
{
let gnuRe = /(\b|_)(gnu|glibc|linux)(\b|_)/i;
if (gnuRe.test(ua)) {
return 'gnu';
}
}
{
let libcRe = /(\b|_)(libc)(\b|_)/i;
if (libcRe.test(ua)) {
return 'libc';
}
}
{
let darwinRe = /(\b|_)(darwin)(\b|_)/i;
if (darwinRe.test(ua)) {
// not sure whether this should be "darwin" or "libc" or "none"
// https://opensource.apple.com/source/Libc/
return 'libc';
}
}
return 'libc';
}
uaDetect.os = getOs;