diff --git a/_webi/fetch-releases-and-assign-build-targets.js b/_webi/builds-cacher.js similarity index 60% rename from _webi/fetch-releases-and-assign-build-targets.js rename to _webi/builds-cacher.js index 5f57b40..86ec8d0 100644 --- a/_webi/fetch-releases-and-assign-build-targets.js +++ b/_webi/builds-cacher.js @@ -1,6 +1,6 @@ 'use strict'; -var BuildsCache = module.exports; +var BuildsCacher = module.exports; let Fs = require('node:fs/promises'); let Path = require('node:path'); @@ -9,10 +9,8 @@ let request = require('@root/request'); let Triplet = require('./build-classifier/triplet.js'); var ALIAS_RE = /^alias: (\w+)$/m; -var INSTALLERS_DIR = Path.join(__dirname, '..'); -var CACHE_DIR = Path.join(__dirname, '../_cache'); -BuildsCache.getPackages = async function (dir) { +BuildsCacher.getPackages = async function (dir) { let dirs = { hidden: {}, errors: {}, @@ -77,7 +75,7 @@ BuildsCache.getPackages = async function (dir) { dirs.errors[entry.name] = err; continue; } - if (err.requireStack.length === 1) { + if (err.requireStack.length === 2) { dirs.selfhosted[entry.name] = true; continue; } @@ -100,73 +98,6 @@ BuildsCache.getPackages = async function (dir) { return dirs; }; -function showDirs(dirs) { - { - let errors = Object.keys(dirs.errors); - console.error(''); - console.error(`Errors: ${errors.length}`); - for (let name of errors) { - let err = dirs.errors[name]; - console.error(`${name}/: ${err.message}`); - } - } - - { - let hidden = Object.keys(dirs.hidden); - console.debug(''); - console.debug(`Hidden: ${hidden.length}`); - for (let name of hidden) { - let kind = dirs.hidden[name]; - if (kind === '!directory') { - console.debug(` ${name}`); - } else { - console.debug(` ${name}/`); - } - } - } - - { - let alias = Object.keys(dirs.alias); - console.debug(''); - console.debug(`Alias: ${alias.length}`); - for (let name of alias) { - let kind = dirs.alias[name]; - if (kind === 'symlink') { - console.debug(` ${name} => ...`); - } else { - console.debug(` ${name}/`); - } - } - } - - { - let invalids = Object.keys(dirs.invalid); - console.warn(''); - console.warn(`Invalid: ${invalids.length}`); - for (let name of invalids) { - console.warn(` ${name}/`); - } - } - - { - let selfhosted = Object.keys(dirs.selfhosted); - console.info(''); - console.info(`Self-Hosted: ${selfhosted.length}`); - for (let name of selfhosted) { - console.info(` ${name}/`); - } - } - - { - let valids = Object.keys(dirs.valid); - console.info(''); - console.info(`Found: ${valids.length}`); - for (let name of valids) { - console.info(` ${name}/`); - } - } -} - async function getPartialHeader(path) { let readme = `${path}/README.md`; let head = await readFirstBytes(readme).catch(function (err) { @@ -247,9 +178,12 @@ async function getLatestBuildsInner(Releases, cacheDir, name) { return data; } -BuildsCache.create = function ({ ALL_TERMS, installers, caches }) { - let bc = {}; +BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) { + if (!ALL_TERMS) { + ALL_TERMS = Triplet.TERMS_PRIMARY_MAP; + } + let bc = {}; bc.usedTerms = {}; bc.orphanTerms = Object.assign({}, ALL_TERMS); bc.unknownTerms = {}; @@ -349,7 +283,7 @@ BuildsCache.create = function ({ ALL_TERMS, installers, caches }) { } if (bc._staleNames.length === 0) { - let dirs = await BuildsCache.getPackages(INSTALLERS_DIR); + let dirs = await BuildsCacher.getPackages(installers); bc._staleNames = Object.keys(dirs.valid); bc._staleNames.sort(function () { return 0.5 - Math.random(); @@ -443,161 +377,3 @@ BuildsCache.create = function ({ ALL_TERMS, installers, caches }) { return bc; }; - -async function main() { - let bc = BuildsCache.create({ - ALL_TERMS: Triplet.TERMS_PRIMARY_MAP, - caches: CACHE_DIR, - installers: INSTALLERS_DIR, - }); - - // let names = ['{NAME}-win32.exe']; - // for (let name of names) { - // console.log(name); - // name = replaceTriples('TEST', {}, name); - // console.log(name); - // } - // process.exit(0); - - let dirs = await BuildsCache.getPackages(INSTALLERS_DIR); - showDirs(dirs); - console.info(''); - - bc.freshenRandomPackage(600 * 1000); - - // let pkg = await bc.getBuilds({ - // name: name, - // date: new Date(), - // installers: INSTALLERS_DIR, - // caches: CACHE_DIR, - // }); - // let triples = []; - // for (let build of pkg.releases) { - // let triplet = await pkgToTriples('git', pkg, build); - // triples.push(triplet); - // } - // return triples; - // // process.exit(1) - - let triples = []; - let rows = []; - let valids = Object.keys(dirs.valid); - console.info(`Fetching builds for`); - for (let name of valids) { - if (name === 'webi') { - // TODO fix the webi faux package - // (not sure why I even created it) - continue; - } - - console.info(` ${name}`); - let pkg = await bc.getBuilds({ - name: name, - date: new Date(), - }); - - let nStr = pkg.releases.length.toString(); - let n = nStr.padStart(5, ' '); - let row = `##### ${n}\t${name}\tv`; - rows.push(row); - - // ignore known, non-package extensions - for (let build of pkg.releases) { - let triplet; - try { - triplet = bc.classify(pkg, build); - } catch (e) { - if (e.code === 'E_BUILD_NO_PATTERN') { - console.warn(`>>> ${e.message} <<<`); - console.warn(pkg); - console.warn(build); - console.warn(`^^^ ${e.message} ^^^`); - continue; - } - throw e; - } - if (!triplet) { - continue; - } - - triples.push(triplet); - rows.push(`${triplet}\t${name}\t${build.version}`); - } - } - let tsv = rows.join('\n'); - console.info(''); - console.info('#rows', rows.length); - await Fs.writeFile('builds.tsv', tsv, 'utf8'); - - // let terms = Object.keys(allTermsMap); - // terms.sort(); - // console.log(terms.join('\n')); - // // console.log(terms); - // for (; terms.length; ) { - // let a = terms.shift() || ''; - // let b = terms.shift() || ''; - // let c = terms.shift() || ''; - // let d = terms.shift() || ''; - // let e = terms.shift() || ''; - // console.log( - // [ - // a.padEnd(15, ' ').padStart(16, ' '), - // b.padEnd(15, ' ').padStart(16, ' '), - // c.padEnd(15, ' ').padStart(16, ' '), - // d.padEnd(15, ' ').padStart(16, ' '), - // e.padEnd(15, ' ').padStart(16, ' '), - // ].join(' '), - // ); - // } - - console.info(''); - console.info('Triplets Detected:'); - let triplets = Object.keys(bc._triplets); - if (triplets.length) { - triplets.sort(); - console.info(' ', triplets.join('\n ')); - } else { - console.info(' (none)'); - } - - console.info(''); - console.info('New / Unknown Terms:'); - let unknowns = Object.keys(bc.unknownTerms); - if (unknowns.length) { - unknowns.sort(); - console.warn(' ', unknowns.join('\n ')); - } else { - console.info(' (none)'); - } - - console.info(''); - console.info('Unused Terms:'); - let unuseds = Object.keys(bc.orphanTerms); - if (unuseds.length) { - unuseds.sort(); - console.warn(' ', unuseds.join('\n ')); - } else { - console.info(' (none)'); - } - - console.info(''); - - // sort -u -k1 builds.tsv | rg -v '^#|^https?:' | rg -i arm - // cut -f1 builds.tsv | sort -u -k1 | rg -v '^#|^https?:' | rg -i arm -} - -if (module === require.main) { - main() - .then(function () { - function forceExit() { - console.warn(`warn: dangling event loop reference`); - process.exit(0); - } - let exitTimeout = setTimeout(forceExit, 250); - exitTimeout.unref(); - }) - .catch(function (err) { - console.error(err); - process.exit(1); - }); -} diff --git a/_webi/lint-builds.js b/_webi/lint-builds.js new file mode 100644 index 0000000..a3ade95 --- /dev/null +++ b/_webi/lint-builds.js @@ -0,0 +1,234 @@ +#!/usr/bin/env node +'use strict'; + +let Fs = require('node:fs/promises'); +let Path = require('node:path'); + +let BuildsCacher = require('./builds-cacher.js'); + +var INSTALLERS_DIR = Path.join(__dirname, '..'); +var CACHE_DIR = Path.join(__dirname, '../_cache'); + +function showDirs(dirs) { + { + let errors = Object.keys(dirs.errors); + console.error(''); + console.error(`Errors: ${errors.length}`); + for (let name of errors) { + let err = dirs.errors[name]; + console.error(`${name}/: ${err.message}`); + } + } + + { + let hidden = Object.keys(dirs.hidden); + console.debug(''); + console.debug(`Hidden: ${hidden.length}`); + for (let name of hidden) { + let kind = dirs.hidden[name]; + if (kind === '!directory') { + console.debug(` ${name}`); + } else { + console.debug(` ${name}/`); + } + } + } + + { + let alias = Object.keys(dirs.alias); + console.debug(''); + console.debug(`Alias: ${alias.length}`); + for (let name of alias) { + let kind = dirs.alias[name]; + if (kind === 'symlink') { + console.debug(` ${name} => ...`); + } else { + console.debug(` ${name}/`); + } + } + } + + { + let invalids = Object.keys(dirs.invalid); + console.warn(''); + console.warn(`Invalid: ${invalids.length}`); + for (let name of invalids) { + console.warn(` ${name}/`); + } + } + + { + let selfhosted = Object.keys(dirs.selfhosted); + console.info(''); + console.info(`Self-Hosted: ${selfhosted.length}`); + for (let name of selfhosted) { + console.info(` ${name}/`); + } + } + + { + let valids = Object.keys(dirs.valid); + console.info(''); + console.info(`Found: ${valids.length}`); + for (let name of valids) { + console.info(` ${name}/`); + } + } +} + +async function main() { + let bc = BuildsCacher.create({ + caches: CACHE_DIR, + installers: INSTALLERS_DIR, + }); + + // let names = ['{NAME}-win32.exe']; + // for (let name of names) { + // console.log(name); + // name = replaceTriples('TEST', {}, name); + // console.log(name); + // } + // process.exit(0); + + let dirs = await BuildsCacher.getPackages(INSTALLERS_DIR); + showDirs(dirs); + console.info(''); + + bc.freshenRandomPackage(600 * 1000); + + // let pkg = await bc.getBuilds({ + // name: name, + // date: new Date(), + // installers: INSTALLERS_DIR, + // caches: CACHE_DIR, + // }); + // let triples = []; + // for (let build of pkg.releases) { + // let triplet = await pkgToTriples('git', pkg, build); + // triples.push(triplet); + // } + // return triples; + // // process.exit(1) + + let triples = []; + let rows = []; + let valids = Object.keys(dirs.valid); + console.info(`Fetching builds for`); + for (let name of valids) { + if (name === 'webi') { + // TODO fix the webi faux package + // (not sure why I even created it) + continue; + } + + console.info(` ${name}`); + let pkg = await bc.getBuilds({ + name: name, + date: new Date(), + }); + + let nStr = pkg.releases.length.toString(); + let n = nStr.padStart(5, ' '); + let row = `##### ${n}\t${name}\tv`; + rows.push(row); + + // ignore known, non-package extensions + for (let build of pkg.releases) { + let triplet; + try { + triplet = bc.classify(pkg, build); + } catch (e) { + if (e.code === 'E_BUILD_NO_PATTERN') { + console.warn(`>>> ${e.message} <<<`); + console.warn(pkg); + console.warn(build); + console.warn(`^^^ ${e.message} ^^^`); + continue; + } + throw e; + } + if (!triplet) { + continue; + } + + triples.push(triplet); + rows.push(`${triplet}\t${name}\t${build.version}`); + } + } + let tsv = rows.join('\n'); + console.info(''); + console.info('#rows', rows.length); + await Fs.writeFile('builds.tsv', tsv, 'utf8'); + + // let terms = Object.keys(allTermsMap); + // terms.sort(); + // console.log(terms.join('\n')); + // // console.log(terms); + // for (; terms.length; ) { + // let a = terms.shift() || ''; + // let b = terms.shift() || ''; + // let c = terms.shift() || ''; + // let d = terms.shift() || ''; + // let e = terms.shift() || ''; + // console.log( + // [ + // a.padEnd(15, ' ').padStart(16, ' '), + // b.padEnd(15, ' ').padStart(16, ' '), + // c.padEnd(15, ' ').padStart(16, ' '), + // d.padEnd(15, ' ').padStart(16, ' '), + // e.padEnd(15, ' ').padStart(16, ' '), + // ].join(' '), + // ); + // } + + console.info(''); + console.info('Triplets Detected:'); + let triplets = Object.keys(bc._triplets); + if (triplets.length) { + triplets.sort(); + console.info(' ', triplets.join('\n ')); + } else { + console.info(' (none)'); + } + + console.info(''); + console.info('New / Unknown Terms:'); + let unknowns = Object.keys(bc.unknownTerms); + if (unknowns.length) { + unknowns.sort(); + console.warn(' ', unknowns.join('\n ')); + } else { + console.info(' (none)'); + } + + console.info(''); + console.info('Unused Terms:'); + let unuseds = Object.keys(bc.orphanTerms); + if (unuseds.length) { + unuseds.sort(); + console.warn(' ', unuseds.join('\n ')); + } else { + console.info(' (none)'); + } + + console.info(''); + + // sort -u -k1 builds.tsv | rg -v '^#|^https?:' | rg -i arm + // cut -f1 builds.tsv | sort -u -k1 | rg -v '^#|^https?:' | rg -i arm +} + +if (module === require.main) { + main() + .then(function () { + function forceExit() { + console.warn(`warn: dangling event loop reference`); + process.exit(0); + } + let exitTimeout = setTimeout(forceExit, 250); + exitTimeout.unref(); + }) + .catch(function (err) { + console.error(err); + process.exit(1); + }); +}