mirror of
https://github.com/webinstall/webi-installers.git
synced 2026-02-14 17:49:53 +00:00
339 lines
8.7 KiB
JavaScript
339 lines
8.7 KiB
JavaScript
#!/usr/bin/env node
|
|
'use strict';
|
|
|
|
let Fs = require('node:fs/promises');
|
|
let Path = require('node:path');
|
|
|
|
let BuildsCacher = require('./builds-cacher.js');
|
|
let HostTargets = require('./build-classifier/host-targets.js');
|
|
let Parallel = require('./parallel.js');
|
|
|
|
var INSTALLERS_DIR = Path.join(__dirname, '..');
|
|
var CACHE_DIR = Path.join(__dirname, '../_cache');
|
|
|
|
let UserAgentsMap = require('./build-classifier/uas.json');
|
|
let uas = Object.keys(UserAgentsMap);
|
|
let uaTargetsMap = {};
|
|
for (let ua of uas) {
|
|
let terms = ua.split(/[\s\/]+/g);
|
|
let target = {};
|
|
void HostTargets.termsToTarget(target, terms);
|
|
if (!target) {
|
|
continue;
|
|
}
|
|
if (target.errors.length) {
|
|
throw target.errors[0];
|
|
}
|
|
if (!target.os) {
|
|
// TODO make target null, or create error for this
|
|
console.warn(`no os for terms: ${terms}`);
|
|
//throw new Error(`terms: ${terms}`);
|
|
continue;
|
|
}
|
|
if (!target.arch) {
|
|
// TODO make target null, or create error for this
|
|
console.warn(`no arch for terms: ${terms}`);
|
|
//throw new Error(`terms: ${terms}`);
|
|
continue;
|
|
}
|
|
if (!target.libc) {
|
|
// TODO make target null, or create error for this
|
|
console.warn(`no libc for terms: ${terms}`);
|
|
//throw new Error(`terms: ${terms}`);
|
|
continue;
|
|
}
|
|
let triplet = `${target.os}-${target.arch}-${target.libc}`;
|
|
uaTargetsMap[triplet] = target;
|
|
}
|
|
let uaTargets = [];
|
|
let triplets = Object.keys(uaTargetsMap);
|
|
for (let triplet of triplets) {
|
|
let target = uaTargetsMap[triplet];
|
|
uaTargets.push(target);
|
|
}
|
|
|
|
function showDirs(dirs) {
|
|
{
|
|
let errors = Object.keys(dirs.errors);
|
|
console.error('');
|
|
console.error(`Errors: ${errors.length}`);
|
|
for (let name of errors) {
|
|
let err = dirs.errors[name];
|
|
console.error(`${name}/: ${err.message}`);
|
|
}
|
|
}
|
|
|
|
{
|
|
let hidden = Object.keys(dirs.hidden);
|
|
console.debug('');
|
|
console.debug(`Hidden: ${hidden.length}`);
|
|
for (let name of hidden) {
|
|
let kind = dirs.hidden[name];
|
|
if (kind === '!directory') {
|
|
console.debug(` ${name}`);
|
|
} else {
|
|
console.debug(` ${name}/`);
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
let alias = Object.keys(dirs.alias);
|
|
console.debug('');
|
|
console.debug(`Alias: ${alias.length}`);
|
|
for (let name of alias) {
|
|
let kind = dirs.alias[name];
|
|
if (kind === 'symlink') {
|
|
console.debug(` ${name} => ...`);
|
|
} else {
|
|
console.debug(` ${name}/`);
|
|
}
|
|
}
|
|
}
|
|
|
|
{
|
|
let invalids = Object.keys(dirs.invalid);
|
|
console.warn('');
|
|
console.warn(`Invalid: ${invalids.length}`);
|
|
for (let name of invalids) {
|
|
console.warn(` ${name}/`);
|
|
}
|
|
}
|
|
|
|
{
|
|
let selfhosted = Object.keys(dirs.selfhosted);
|
|
console.info('');
|
|
console.info(`Self-Hosted: ${selfhosted.length}`);
|
|
for (let name of selfhosted) {
|
|
console.info(` ${name}/`);
|
|
}
|
|
}
|
|
|
|
{
|
|
let valids = Object.keys(dirs.valid);
|
|
console.info('');
|
|
console.info(`Found: ${valids.length}`);
|
|
for (let name of valids) {
|
|
console.info(` ${name}/`);
|
|
}
|
|
}
|
|
}
|
|
|
|
let bc = BuildsCacher.create({
|
|
caches: CACHE_DIR,
|
|
installers: INSTALLERS_DIR,
|
|
});
|
|
|
|
async function main() {
|
|
/* jshint maxcomplexity: 25 */
|
|
// TODO
|
|
// node ./_webi/lint-builds.js caddy@beta 'x86_64/unknown Darwin libc'
|
|
//
|
|
//let [projName, userAgent] = process.argv.slice(2);
|
|
let projName = process.argv[2];
|
|
// create test case for zoxide, goreleaser, go, yq, caddy, rg
|
|
|
|
let dirs = await bc.getProjectsByType();
|
|
if (!projName) {
|
|
showDirs(dirs);
|
|
console.info('');
|
|
}
|
|
|
|
bc.freshenRandomPackage(600 * 1000);
|
|
|
|
let rows = [];
|
|
let triples = [];
|
|
let valids = Object.keys(dirs.valid);
|
|
|
|
if (projName) {
|
|
if (!valids.includes(projName)) {
|
|
throw new Error(`'${projName}' is not a valid installable project`);
|
|
}
|
|
valids = [projName];
|
|
}
|
|
//valids = ['atomicparsley', 'caddy', 'macos'];
|
|
//valids = ['atomicparsley'];
|
|
|
|
console.info('');
|
|
console.info(`Fetching project release assets`);
|
|
let parallel = 25;
|
|
let projects = [];
|
|
await Parallel.run(parallel, valids, getAll);
|
|
async function getAll(name, i) {
|
|
console.info(` ${name}`);
|
|
let projInfo = await bc.getPackages({
|
|
//Releases: Releases,
|
|
name: name,
|
|
date: new Date(),
|
|
});
|
|
projects[i] = projInfo;
|
|
}
|
|
|
|
console.info(`Classifying build assets for...`);
|
|
for (let projInfo of projects) {
|
|
console.info(` ${projInfo.name}`);
|
|
|
|
let nStr = projInfo.releases.length.toString();
|
|
let n = nStr.padStart(5, ' ');
|
|
let row = `##### ${n}\t${projInfo.name}\tv`;
|
|
rows.push(row);
|
|
|
|
// ignore known, non-package extensions
|
|
for (let build of projInfo.releases) {
|
|
let target = bc.classify(projInfo, build);
|
|
if (!target) {
|
|
// non-build file
|
|
continue;
|
|
}
|
|
|
|
if (target.error) {
|
|
let e = target.error;
|
|
if (e.code === 'E_BUILD_NO_PATTERN') {
|
|
console.warn(`>>> ${e.message} <<<`);
|
|
console.warn(projInfo);
|
|
console.warn(build);
|
|
console.warn(`^^^ ${e.message} ^^^`);
|
|
}
|
|
throw e;
|
|
}
|
|
|
|
if (target.unknownTerms?.length) {
|
|
let msg = `${projInfo.name}: unrecognized term(s) '${target.unknownTerms}' in '${build.download}'`;
|
|
let err = new Error(msg);
|
|
throw err;
|
|
}
|
|
|
|
triples.push(target.triplet);
|
|
// if (!build.version) {
|
|
// throw new Error(`no version for ${pkg.name} ${build.name}`);
|
|
// }
|
|
// // For debug printing versions
|
|
// console.error(build.version);
|
|
rows.push(`${target.triplet}\t${projInfo.name}\t${build.version}`);
|
|
}
|
|
}
|
|
|
|
console.info(`Fetching builds for`);
|
|
for (let projInfo of projects) {
|
|
console.info('');
|
|
console.info('');
|
|
console.info(` ${projInfo.name}`);
|
|
|
|
for (let target of uaTargets) {
|
|
let libc = target.libc || 'libc';
|
|
let hostTriplet = `${target.os}-${target.arch}-${libc}`;
|
|
console.info('');
|
|
console.info(` target: ${hostTriplet}`);
|
|
let match = bc.findMatchingPackages(projInfo, target, {
|
|
ver: '',
|
|
});
|
|
if (!match) {
|
|
console.info(
|
|
` project: ${projInfo.name}: missing build for os '${target.os}'`,
|
|
);
|
|
continue;
|
|
}
|
|
|
|
if (!match.releases) {
|
|
console.info(
|
|
` project: ${projInfo.name}: missing build for os '${target.os}-${target.arch}-${libc}'`,
|
|
);
|
|
} else if (match.triplet === hostTriplet) {
|
|
let releaseNames = Object.keys(match.releases);
|
|
console.info(` selected ${releaseNames.length}`);
|
|
} else {
|
|
let releaseNames = Object.keys(match.releases);
|
|
console.info(
|
|
` selected ${releaseNames.length} (${match.triplet} fallback)`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
let tsv = rows.join('\n');
|
|
console.info('');
|
|
console.info('#rows', rows.length);
|
|
await Fs.writeFile('builds.tsv', tsv, 'utf8');
|
|
|
|
console.info('');
|
|
console.info('Triplets Detected:');
|
|
let triplets = Object.keys(bc._triplets);
|
|
if (triplets.length) {
|
|
triplets.sort();
|
|
console.info(' ', triplets.join('\n '));
|
|
} else {
|
|
console.info(' (none)');
|
|
}
|
|
|
|
console.info('');
|
|
console.info('New / Unknown Terms:');
|
|
let unknowns = Object.keys(bc.unknownTerms);
|
|
if (unknowns.length) {
|
|
unknowns.sort();
|
|
console.warn(' ', unknowns.join('\n '));
|
|
} else {
|
|
console.info(' (none)');
|
|
}
|
|
|
|
console.info('');
|
|
console.info('Unused Terms:');
|
|
let unuseds = Object.keys(bc.orphanTerms);
|
|
if (unuseds.length) {
|
|
unuseds.sort();
|
|
console.warn(' ', unuseds.join('\n '));
|
|
} else {
|
|
console.info(' (none)');
|
|
}
|
|
|
|
console.info('');
|
|
console.info('Formats:');
|
|
if (bc.formats.length) {
|
|
let formats = bc.formats.slice();
|
|
formats.sort();
|
|
if (!formats[0]) {
|
|
formats[0] = '(bin)';
|
|
}
|
|
console.warn(' ', formats.join('\n '));
|
|
} else {
|
|
console.info(' (none)');
|
|
}
|
|
|
|
// sort -u -k1 builds.tsv | rg -v '^#|^https?:' | rg -i arm
|
|
// cut -f1 builds.tsv | sort -u -k1 | rg -v '^#|^https?:' | rg -i arm
|
|
}
|
|
|
|
if (module === require.main) {
|
|
let times = [];
|
|
let now = Date.now();
|
|
main()
|
|
.then(async function () {
|
|
let then = Date.now();
|
|
let delta = then - now;
|
|
times.push(delta);
|
|
now = then;
|
|
await main();
|
|
then = Date.now();
|
|
delta = then - now;
|
|
times.push(delta);
|
|
})
|
|
.then(function () {
|
|
console.info('');
|
|
console.info('Run times');
|
|
for (let delta of times) {
|
|
let s = delta / 1000;
|
|
console.info(` ${s}`);
|
|
}
|
|
|
|
function forceExit() {
|
|
console.warn(`warn: dangling event loop reference`);
|
|
process.exit(0);
|
|
}
|
|
let exitTimeout = setTimeout(forceExit, 250);
|
|
exitTimeout.unref();
|
|
})
|
|
.catch(function (err) {
|
|
console.error(err.stack || err);
|
|
process.exit(1);
|
|
});
|
|
}
|