From fca66feb3ebfaaa3379c9068f3c77851854cbdcb Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Sun, 12 Nov 2023 16:49:28 -0700 Subject: [PATCH] feat: list all releases across all of Webi --- _webi/list-all-releases.js | 861 +++++++++++++++++++++++++++++++++++++ 1 file changed, 861 insertions(+) create mode 100644 _webi/list-all-releases.js diff --git a/_webi/list-all-releases.js b/_webi/list-all-releases.js new file mode 100644 index 0000000..8936412 --- /dev/null +++ b/_webi/list-all-releases.js @@ -0,0 +1,861 @@ +'use strict'; + +let Fs = require('node:fs/promises'); +let Path = require('node:path'); + +let request = require('@root/request'); + +var ALIAS_RE = /^alias: (\w+)$/m; +var INSTALLERS_DIR = Path.join(__dirname, '..'); +var CACHE_DIR = Path.join(__dirname, '../_cache'); + +let CHANNEL_NAMES = ['master', 'nightly']; +let CHECKSUM_NAMES = [ + 'MD5SUMS', + 'B3SUMS', + 'SHA1SUMS', + 'SHA256SUMS', + 'SHA512SUMS', + 'checksum', +]; +let NON_BUILD_NAMES = [ + /(\b|_)(source)(\b|_)/, + /(\b|_)(wasm|js)(\b|_)/, + /(\b|_)(vendor)(\b|_)/, // rclone go vendor + /(\b|_)(src)(\b|_)/, + /(\b|_)(setup)(\b|_)/, + /(\b|_)(symbols)(\b|_)/, + // one-offs + /-armv8/, // pathman + /-no-oniguruma/, // jq + /man_page_only/, // yq + // a build, but not one you'd use given the alternative + /(\b|_)(unsigned)(\b|_)/, // dashd +]; +let NON_BUILD_EXTS = [ + '.1', + '.b3', + '.sha256', + '.sha256sum', + '.sha512', + '.md5', + '.txt', + '.sig', + '.pem', + '.sbom', + '.json', + '.asc', + // TODO: we could include these, and just sort them much lower + // neither macos nor linux have zstd by default + // don't be ridiculous! + '.7z', + '.tar.zstd', + '.zst', + // no android + '.apk', + // not sure + '.msixbundle', + // we can't use these yet + '.deb', + '.rpm', +]; + +// many assets are bare - we'd have to check if any contain a '.' +// in the name before using these as a whitelist +// (ordered by OS for clarity, by length for matching) +let BUILD_EXTS = [ + // Windows + '.exe.zip', + '.exe.xz', + '.exe', + '.msi', + // macOS + '.app.zip', + '.dmg.zip', // because some systems only allow .zip + '.app', // for classification (it'll be in a zip or dmg) + '.dmg', + '.pkg', + // Nondescript + '.zip', + '.tar.gz', + '.tar.xz', + '.gz', + '.xz', + // POSIX + // (also, could be the runnable, installable thing, or an install script) + '.sh', + // Any (font, vim script) + '.git', +]; + +// these don't provide any clues as to which OS is used +// (counter-examples: .exe (windows) .dmg (mac) .deb (linux) +let NON_INFORMATIVE_EXTS = ['.gz', '.xz', '.zstd', '.zip', '.tar']; + +var wordMap = {}; +var unknownMap = {}; + +async function getPackages(dir) { + let dirs = { + hidden: {}, + errors: {}, + alias: {}, + invalid: {}, + selfhosted: {}, + valid: {}, + }; + + let entries = await Fs.readdir(dir, { withFileTypes: true }); + for (let entry of entries) { + // skip non-installer dirs + if (entry.isSymbolicLink()) { + dirs.alias[entry.name] = 'symlink'; + continue; + } + if (!entry.isDirectory()) { + dirs.hidden[entry.name] = '!directory'; + continue; + } + if (entry.name === 'node_modules') { + dirs.hidden[entry.name] = 'node_modules'; + continue; + } + if (entry.name.startsWith('_')) { + dirs.hidden[entry.name] = '_*'; + continue; + } + if (entry.name.startsWith('.')) { + dirs.hidden[entry.name] = '.*'; + continue; + } + if (entry.name.startsWith('~')) { + dirs.hidden[entry.name] = '~*'; + continue; + } + if (entry.name.endsWith('~')) { + dirs.hidden[entry.name] = '*~'; + continue; + } + + // skip invalid installers + let path = Path.join(dir, entry.name); + let head = await getPartialHeader(path); + if (!head) { + // console.info(`skip ./${entry.name}/ (no README.md)`); + dirs.invalid[entry.name] = '!README.md'; + continue; + } + + let alias = head.match(ALIAS_RE); + if (alias) { + // console.info(`skip ./${entry.name}/ (alias of ./${alias[1]}/)`); + dirs.alias[entry.name] = true; + continue; + } + + let releasesPath = Path.join(path, 'releases.js'); + let releases; + try { + releases = require(releasesPath); + } catch (err) { + if (err.code !== 'MODULE_NOT_FOUND') { + dirs.errors[entry.name] = err; + continue; + } + if (err.requireStack.length === 1) { + // console.info(`skip ./${entry.name}/ (no releases.js)`); + dirs.selfhosted[entry.name] = true; + continue; + } + // err.requireStack.length > 1 + console.error(''); + console.error('PROBLEM'); + console.error(` ${err.message}`); + console.error(''); + console.error('SOLUTION'); + console.error(' npm clean-install'); + console.error(''); + process.exit(1); + } + + dirs.valid[entry.name] = true; + } + + return dirs; +} + +function showDirs(dirs) { + { + let errors = Object.keys(dirs.errors); + console.error(''); + console.error(`Errors: ${errors.length}`); + for (let name of errors) { + let err = dirs.errors[name]; + console.error(`${name}/: ${err.message}`); + } + } + + { + let hidden = Object.keys(dirs.hidden); + console.debug(''); + console.debug(`Hidden: ${hidden.length}`); + for (let name of hidden) { + let kind = dirs.hidden[name]; + if (kind === '!directory') { + console.debug(` ${name}`); + } else { + console.debug(` ${name}/`); + } + } + } + + { + let alias = Object.keys(dirs.alias); + console.debug(''); + console.debug(`Alias: ${alias.length}`); + for (let name of alias) { + let kind = dirs.alias[name]; + if (kind === 'symlink') { + console.debug(` ${name} => ...`); + } else { + console.debug(` ${name}/`); + } + } + } + + { + let invalids = Object.keys(dirs.invalid); + console.warn(''); + console.warn(`Invalid: ${invalids.length}`); + for (let name of invalids) { + console.warn(` ${name}/`); + } + } + + { + let selfhosted = Object.keys(dirs.selfhosted); + console.info(''); + console.info(`Self-Hosted: ${selfhosted.length}`); + for (let name of selfhosted) { + console.info(` ${name}/`); + } + } + + { + let valids = Object.keys(dirs.valid); + console.info(''); + console.info(`Found: ${valids.length}`); + for (let name of valids) { + console.info(` ${name}/`); + } + } +} + +async function getPartialHeader(path) { + let readme = `${path}/README.md`; + let head = await readFirstBytes(readme).catch(function (err) { + if (err.code !== 'ENOENT') { + console.warn(`warn: ${path}: ${err.message}`); + } + return null; + }); + + return head; +} + +// let fsOpen = util.promisify(Fs.open); +// let fsRead = util.promisify(Fs.read); +async function readFirstBytes(path) { + let start = 0; + let n = 1024; + let fh = await Fs.open(path, 'r'); + let buf = new Buffer.alloc(n); + let result = await fh.read(buf, start, n); + let str = result.buffer.toString('utf8'); + await fh.close(); + + return str; +} + +async function getBuilds({ caches, installers, name, date }) { + let cacheDir = caches; + let installerDir = installers; + if (!date) { + date = new Date(); + } + let isoDate = date.toISOString(); + let yearMonth = isoDate.slice(0, 7); + let dataFile = `${cacheDir}/${yearMonth}/${name}.json`; + + // let secondsStr = await Fs.readFile(tsFile, 'ascii').catch(function (err) { + // if (err.code !== 'ENOENT') { + // throw err; + // } + // return '0'; + // }); + // secondsStr = secondsStr.trim(); + // let seconds = parseFloat(secondsStr) || 0; + + // let age = now - seconds; + + let json = await Fs.readFile(dataFile, 'ascii').catch(async function (err) { + if (err.code !== 'ENOENT') { + throw err; + } + + return null; + }); + + let data; + try { + data = JSON.parse(json); + } catch (e) { + console.error(`error: ${dataFile}:\n\t${e.message}`); + data = null; + } + + if (!data) { + data = await getLatestBuilds(installerDir, cacheDir, name); + } + + return data; +} + +let promises = {}; +async function getLatestBuilds(installerDir, cacheDir, name) { + let Releases = require(`${installerDir}/${name}/releases.js`); + if (!Releases.latest) { + Releases.latest = Releases; + } + + if (!promises[name]) { + promises[name] = Promise.resolve(); + } + + promises[name] = promises[name].then(async function () { + return await getLatestBuildsInner(Releases, cacheDir, name); + }); + + return await promises[name]; +} + +async function getLatestBuildsInner(Releases, cacheDir, name) { + let data = await Releases.latest(request); + + let date = new Date(); + let isoDate = date.toISOString(); + let yearMonth = isoDate.slice(0, 7); + + let dataFile = `${cacheDir}/${yearMonth}/${name}.json`; + // TODO fsstat releases.js vs require-ing time as well + let tsFile = `${cacheDir}/${yearMonth}/${name}.updated.txt`; + + let dirPath = Path.dirname(dataFile); + await Fs.mkdir(dirPath, { recursive: true }); + + let json = JSON.stringify(data); + await Fs.writeFile(dataFile, json, 'utf8'); + + let seconds = date.valueOf(); + let ms = seconds / 1000; + let msStr = ms.toFixed(3); + await Fs.writeFile(tsFile, msStr, 'utf8'); + + return data; +} + +// Makes sure that packages are updated once an hour, on average +let staleNames = []; +let freshenTimeout; +async function freshenRandomPackage(minDelay) { + if (!minDelay) { + minDelay = 15 * 1000; + } + + if (staleNames.length === 0) { + let dirs = await getPackages(INSTALLERS_DIR); + staleNames = Object.keys(dirs.valid); + staleNames.sort(function () { + return 0.5 - Math.random(); + }); + } + + let name = staleNames.pop(); + await getBuilds({ + caches: CACHE_DIR, + installers: INSTALLERS_DIR, + name: name, + date: new Date(), + }); + + let hour = 60 * 60 * 1000; + let delay = minDelay; + let spread = hour / staleNames.length; + let seed = Math.random(); + delay += seed * spread; + + clearTimeout(freshenTimeout); + freshenTimeout = setTimeout(freshenRandomPackage, delay); + freshenTimeout.unref(); +} + +// TODO packages have many releases which have many builds +// go has 1.20 and 1.21 which have go1.20-darwin-arm64.tar.gz, etc +async function pkgToTriples(name) { + let triples = []; + + let pkg = await getBuilds({ + caches: CACHE_DIR, + installers: INSTALLERS_DIR, + name, + date: new Date(), + }); + + for (let build of pkg.releases) { + let maybeInstallable = looksInstallable(name, build, pkg); + if (!maybeInstallable) { + continue; + } + + let triplish = trimNameAndVersion(name, build, pkg); + if (!triplish) { + continue; + } + + triplish = replaceTriples(triplish); + + triples.push(triplish); + console.log(triplish); + } + + return triples; +} + +function looksInstallable(name, build, pkg) { + for (let sumname of CHECKSUM_NAMES) { + if (build.download.includes(sumname)) { + return false; + } + } + + for (let ext of NON_BUILD_EXTS) { + if (build.download.endsWith(ext)) { + return false; + } + } + + for (let name of NON_BUILD_NAMES) { + if (name.test(build.download)) { + return false; + } + } + + // don't count tip commits as versions + // (though a ${latest}-${date} could possibly be used as a substitute) + let channels = CHANNEL_NAMES; + for (let ch of channels) { + if (build.version === ch) { + return false; + } + } + + // not that we can say for sure this is a good file, + // but we can't say it's a bad one + return true; +} + +function trimNameAndVersion(name, build, pkg) { + let { download, _filename, version, _version } = build; + let { _names } = pkg; + + // for when the download URL is a uuid, for example + if (_filename) { + download = _filename; + } + + // use the raw, non-semver version for file names + if (_version) { + version = _version; + } + if (!_names) { + _names = [name]; + } + + // for watchexec tags like cli-v1.20.3 + version = version.replace(/^cli-/, ''); + version = version.replace(/^v?/i, 'v?'); + let verEsc = version.replace(/\./g, '\\.').replace(/\+/g, '\\+'); + // maybe just once before and once after + + // generic sources that benefit most from dynamic matching + { + let ghReleaseRe = + /https:..github.com.[^\/]+.[^\/]+.releases.download.[^\/]+.(.*)/; + let ghrMatches = download.match(ghReleaseRe); + if (ghrMatches) { + download = ghrMatches[1]; + } + } + + // sources that _don't_ benefit from matching + { + // ex: https://codeload.github.com/BeyondCodeBootcamp/DuckDNS.sh/legacy.zip/refs/tags/v1.0.2 + let ghSourceRe = + /https:..codeload.github.com\/([^\/]+)\/([^\/]+)\/([^\/]+)\/refs\/tags\/([^\/]+)/; + let ghsMatches = download.match(ghSourceRe); + if (ghsMatches) { + download = `${ghsMatches[2]}/${ghsMatches[4]}/${ghsMatches[3]}`; + } + + let gitSourceRe = /(https|http|git):.*\/(.*).git$/; + download = download.replace(gitSourceRe, '$2.git'); + } + + // many of these have fully custom logic + { + let sfRe = + /https:..sourceforge.net.projects.([^\/]+).files.([^\/]+).download/; + let sfMatches = download.match(sfRe); + if (sfMatches) { + download = `${sfMatches[1]}_${sfMatches[2]}`; + } + + let appleRe = /^http:..updates-http.cdn-apple.com.2019.cert.[^\/]+\//; + download = download.replace(appleRe, ''); + + let zigRe = /^https:..ziglang.org\/(download|builds)\//; + download = download.replace(zigRe, ''); + + let hcRe = /^https:..releases.hashicorp.com\/[^\/]+\/[^\/]+\//; + download = download.replace(hcRe, ''); + + // cuts off trailing query param as well + let pgRe = /^https:..get.enterprisedb.com\/postgresql\/([^?]+)\?.*/; + download = download.replace(pgRe, '$1'); + + let nodeRe = /^https:..(unofficial-builds.)?nodejs.org.download.release./; + download = download.replace(nodeRe, ''); + + // captures channel - stable, beta, etc + let it2Re = /^https:..iterm2.com.downloads.([^\/]+)./; + download = download.replace(it2Re, ''); + } + + // console.log('dbg', download); + // console.log('dbg', verEsc); + download = download.replace( + new RegExp(`(\\b|\\D)v?${verEsc}`, 'ig'), + '$1{VER}', + ); + // console.log('dbg', download); + for (let _name of _names) { + let nameEsc = _name.replace(/[\._\-\+]/g, '.'); + let nameRe = new RegExp(`(\\b|_)${nameEsc}(\\b|_|\\d|[A-Z])`, 'gi'); + download = download.replace(nameRe, '{NAME}$2'); + } + // console.log('dbg', download); + + // trim the start of any url + //download = download.replace(/https:\/\/[^\/]+\//, ''); + + // trim URLs up to the first {FOO} + download = download.replace(/^[^{]+\//, ''); + + for (let ext of NON_INFORMATIVE_EXTS) { + let hasNonInformativeExt = download.endsWith(ext); + if (hasNonInformativeExt) { + download = download.slice(0, -ext.length); + } + } + + let words = download.split(/[_\{\}\/\.\-]+/g); + for (let word of words) { + wordMap[word] = true; + } + + return download; +} + +let TERMS = [ + // just channels + { term: 'stable', channel: 'stable' }, + { term: 'preview', channel: 'preview' }, + { term: 'beta', channel: 'beta' }, + { term: 'dev', channel: 'dev' }, + { term: 'debug', channel: 'debug' }, + // mostly os + { term: 'windows', os: 'windows' }, + { term: 'windowsx86', os: 'windows', arch: 'x86' }, + { term: 'pc', vendor: 'pc' }, + { term: 'win64', os: 'windows', arch: 'amd64', bit: 64 }, + { term: 'win32', os: 'windows', arch: 'x86', bit: 32 }, + { term: 'linux64', os: 'linux', arch: 'amd64', bit: 64 }, + { term: 'linux32', os: 'linux', arch: 'x86', bit: 32 }, + { term: 'linux', os: 'linux' }, + { + term: /(\b|_)(apple|macos|osx|darwin|mac)([-_]?1\d\.\d+)?(\b|_)/gi, + os: 'darwin', + vendor: 'apple', + }, + { term: 'osx64', os: 'darwin', arch: 'amd64', bit: 64 }, + { term: 'mac64', os: 'darwin', arch: 'amd64', bit: 64 }, + { term: 'mac32', os: 'darwin', arch: 'x86', bit: 32 }, + { term: 'freebsd12', os: 'freebsd' }, + { term: 'freebsd', os: 'freebsd' }, + { term: 'openbsd', os: 'openbsd' }, + { term: 'netbsd', os: 'netbsd' }, + { term: 'dragonfly', os: 'dragonfly' }, + { term: 'plan9', os: 'plan9' }, + { term: 'illumos', os: 'illumos' }, + { term: 'aix', os: 'aix' }, + { term: 'sunos', os: 'sunos' }, + { term: 'solaris11', os: 'solaris' }, + { term: 'solaris', os: 'solaris' }, + // mostly arch + { + term: /(\b|_)(x64|amd64|x86[_-]64)([_\-]?v\d)?(\b|_)/gi, + arch: 'x86_64 $3', + }, + + // How to navigate the minefield of armv[567](e|l|a|hf|kz) + // See + // - "hf" seems to standard now, but carried over from the v5/v6 days + // - "kz" is a special architecture for security + // - "e" / "el" seems to be old v5 stuff + // - "a" seems to be the best of the v7 era + // - "l" ??? seems to be standard + // We could have some crazy fallback logic but... aarch64 is the future! + { term: /(\b|_)(arm64|aarch64)([_\-]?v\d)?(\b|_)/gi, arch: 'aarch64 $3' }, + { term: /(\b|_)(arm[_-]?v?7l?)(\b|_)/gi, arch: 'armv7' }, + //{ term: /(\b|_)(arm[_-]?v?7)(\b|_)/gi, arch: 'armv7' }, + { term: 'arm32', arch: 'armv7' }, + { term: 'armv7a', arch: 'armv7a' }, + // See + { term: 'armhf', arch: 'armv7' }, // are we sure about hf? + // armv6hf will always work on armv7, or armv6hf, but not armv6 or armv5e + { term: /(\b|_)(arm[_-]?v?6hf)(\b|_)/gi, arch: 'armv7' }, + // armv6, armv6l + { term: /(\b|_)(arm[_-]?v?6(l|hf)?)(\b|_)/gi, arch: 'armv6' }, + { term: 'armel', arch: 'armv5' }, + { term: /(\b|_)(arm[_-]?v?5l?)(\b|_)/gi, arch: 'armv5' }, + { term: /(\b|_)(arm[_-]?v?6kz)(\b|_)/gi, arch: 'armv6kz' }, + + // Enter the minefield of x86 + { term: 'i386', arch: 'x86' }, + { term: '386', arch: 'x86' }, + { term: 'i686', arch: 'x86', arch_ext: 'i686' }, + { term: '686', arch: 'x86', arch_ext: 'i686' }, + { term: 'x86', arch: 'x86' }, + { term: 'ia32', arch: 'x86' }, + // the weird ones + { term: 'loong64', arch: 'loong64' }, + { term: 'mips', arch: 'mips' }, + { term: 'mips64', arch: 'mips64' }, + { term: 'mips64el', arch: 'mips64el' }, + { term: 'mips64le', arch: 'mips64le' }, + { term: 'mips64r6', arch: 'mips64r6' }, + { term: 'mips64r6el', arch: 'mips64r6el' }, + { term: 'mipsel', arch: 'mipsel' }, + { term: 'mipsle', arch: 'mipsle' }, + { term: 'mipsr6', arch: 'mipsr6' }, + { term: 'mipsr6el', arch: 'mipsr6el' }, + { term: 'powerpc', arch: 'ppc' }, + { term: 'powerpc64le', arch: 'ppc64le' }, + { term: 'ppc64', arch: 'ppc64' }, + { term: 'ppc64el', arch: 'ppc64el' }, + { term: 'ppc64le', arch: 'ppc64le' }, + { term: 'riscv64', arch: 'riscv64' }, + { term: 's390x', arch: 's390x' }, + + // mostly libc + { term: 'static', libc: 'none' }, + // TODO how to determine when "musl" is NOT static (i.e. musl++) + { term: 'alpine', os: 'linux', libc: 'none' }, + { term: 'musl', libc: 'none' }, + + // saved for last due to ambiguity + { term: 'universal', os: 'darwin', arch: 'amd64', vendor: 'apple' }, + { term: 'arm', arch: 'armv7' }, + { term: 'gnueabihf', arch: 'armv7', os: 'linux' }, // are we sure about hf? + { term: 'musleabihf', arch: 'armv7', os: 'linux' }, // are we sure about hf? + { term: 'eabihf', arch: 'armv6' }, + { term: 'gnu', os: 'windows', libc: 'none' }, + { term: 'win', os: 'windows' }, + { term: 'msvc', os: 'windows', libc: 'msvc' }, + { term: /(\b|_)32[_-]?(bit)?(\b|_)/, arch: 'x86' }, + { term: /(\b|_)64[_-]?(bit)?(\b|_)/, arch: 'x86_64' }, + { term: 'all', os: 'darwin', arch: 'amd64', vendor: 'apple' }, + { term: 'm1', os: 'darwin', arch: 'aarch64', vendor: 'apple' }, + { term: 'unknown', os: 'linux', vendor: 'unknown' }, + { term: 'android', os: 'android', arch: 'aarch64' }, + { term: 'androideabi', os: 'android', arch: 'armv6' }, +]; +for (let term of TERMS) { + if ('string' === typeof term.term) { + term.term = new RegExp(`(\\b|_)(${term.term})(\\b|_)`, 'ig'); + } +} + +function replaceTriples(filename) { + let meta = { + os: '', + arch: '', + vendor: '', + libc: '', + ext: '', + }; + + for (let term of TERMS) { + let replacement = ''; + //replacement += '$1'; + if (term.os) { + if (!meta.os) { + replacement += '{OS}'; + } + meta.os = term.os; + } + if (term.vendor) { + if (!meta.vendor) { + replacement += '{VENDOR}'; + } + meta.vendor = term.vendor; + } + if (term.arch) { + if (!meta.arch) { + replacement += '{ARCH}'; + } + meta.arch = term.arch; + } + if (term.libc) { + if (!meta.libc) { + replacement += '{LIBC}'; + meta.libc = term.libc; + } + } + + //replacement += '$2'; + filename = filename.replace(term.term, replacement); + } + for (let ext of BUILD_EXTS) { + if (filename.endsWith(ext)) { + meta.ext = ext; + filename = filename.replace(ext, '.{EXT}'); + break; + } + } + + return filename; +} + +async function main() { + // let names = ['{NAME}-win32.exe']; + // for (let name of names) { + // console.log(name); + // name = replaceTriples(name); + // console.log(name); + // } + // process.exit(0); + + let dirs = await getPackages(INSTALLERS_DIR); + showDirs(dirs); + + freshenRandomPackage(600 * 1000); + + // await pkgToTriples('git'); + // process.exit(1) + + let triples = []; + let rows = []; + let valids = Object.keys(dirs.valid); + for (let name of valids) { + let pkg = await getBuilds({ + caches: CACHE_DIR, + installers: INSTALLERS_DIR, + name, + date: new Date(), + }); + let nStr = pkg.releases.length.toString(); + let n = nStr.padStart(5, ' '); + let row = `##### ${n}\t${name}\tv`; + // TODO get by next-most-recent version + let samples = pkg.releases; // .slice(0, 100); + //console.log(samples); + console.log(row); + rows.push(row); + + // ignore known, non-package extensions + for (let build of samples) { + let maybeInstallable = looksInstallable(name, build, pkg); + if (!maybeInstallable) { + continue; + } + + let triplish = trimNameAndVersion(name, build, pkg); + if (!triplish) { + continue; + } + + triplish = replaceTriples(triplish); + + //let row = `${build.download}\t${build.version}\t${build.name}`; + //console.info(`${build.name}\t${build.download}`); + //console.info(row); + //rows.push(row); + + triples.push(triplish); + rows.push(`${triplish}\t${name}\t${build.version}`); + } + } + let tsv = rows.join('\n'); + console.log('#rows', rows.length); + await Fs.writeFile('builds.tsv', tsv, 'utf8'); + + let terms = Object.keys(wordMap); + terms.sort(); + console.log(terms.join('\n')); + // console.log(terms); + for (; terms.length; ) { + let a = terms.shift() || ''; + let b = terms.shift() || ''; + let c = terms.shift() || ''; + let d = terms.shift() || ''; + let e = terms.shift() || ''; + console.log( + [ + a.padEnd(15, ' ').padStart(16, ' '), + b.padEnd(15, ' ').padStart(16, ' '), + c.padEnd(15, ' ').padStart(16, ' '), + d.padEnd(15, ' ').padStart(16, ' '), + e.padEnd(15, ' ').padStart(16, ' '), + ].join(' '), + ); + } + + for (let triple of triples) { + let unknowns = triple.split(/[_\{\}\/\.\-]+/g); + for (let unknown of unknowns) { + unknownMap[unknown] = true; + } + } + + let unknowns = Object.keys(unknownMap); + unknowns.sort(); + console.log(unknowns.join('\n')); + + // sort -u -k1 builds.tsv | rg -v '^#|^https?:' | rg -i arm + // cut -f1 builds.tsv | sort -u -k1 | rg -v '^#|^https?:' | rg -i arm +} + +main() + .then(function () { + function forceExit() { + console.warn(`warn: dangling event loop reference`); + process.exit(0); + } + let exitTimeout = setTimeout(forceExit, 250); + exitTimeout.unref(); + }) + .catch(function (err) { + console.error(err); + process.exit(1); + });