From ce9c111fa9be52cdd1e830e2620b75f76fca5a4c Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Thu, 7 Dec 2023 14:12:28 -0700 Subject: [PATCH] WIPPY WIP: feat: create builds tree --- _webi/builds-cacher.js | 25 ++++--- _webi/lint-builds.js | 155 +++++++++++++++++++++++++++-------------- 2 files changed, 118 insertions(+), 62 deletions(-) diff --git a/_webi/builds-cacher.js b/_webi/builds-cacher.js index 8b2868d..aafae96 100644 --- a/_webi/builds-cacher.js +++ b/_webi/builds-cacher.js @@ -129,7 +129,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) { bc.orphanTerms = Object.assign({}, ALL_TERMS); bc.unknownTerms = {}; bc._triplets = {}; - bc._downloadTriplets = {}; + bc._targetsByBuildIdCache = {}; bc._caches = {}; bc._staleAge = 15 * 60 * 1000; @@ -346,21 +346,23 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) { bc.classify = function (pkg, build) { let maybeInstallable = Triplet.maybeInstallable(pkg, build); if (!maybeInstallable) { - return; + return null; } let buildId = `${pkg.name}@${build.download}`; - let triplet = bc._downloadTriplets[buildId]; - if (triplet) { - Object.assign(build, { triplet }); - return triplet; + let target = bc._targetsByBuildIdCache[buildId]; + if (target) { + Object.assign(build, { target: target, triplet: target.triplet }); + return target; } let pattern = Triplet.toPattern(pkg, build); if (!pattern) { let err = new Error(`no pattern generated for ${name}`); err.code = 'E_BUILD_NO_PATTERN'; - throw err; + target = { error: err }; + bc._targetsByBuildIdCache[buildId] = target; + return target; } let rawTerms = pattern.split(/[_\{\}\/\.\-]+/g); @@ -373,7 +375,10 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) { // {NAME}.windows.x86_64v2.musl.exe let terms = Triplet.patternToTerms(pattern); if (!terms.length) { - throw new Error(`'${terms}' was trimmed to ''`); + let err = new Error(`'${terms}' was trimmed to ''`); + target = { error: err }; + bc._targetsByBuildIdCache[buildId] = target; + return target; } for (let term of terms) { @@ -392,12 +397,12 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) { // {NAME}.windows.x86_64v2.musl.exe // windows-x86_64_v2-musl - let target = { triplet: '' }; + target = { triplet: '' }; void Triplet.termsToTarget(target, pkg, build, terms); target.triplet = `${target.arch}-${target.vendor}-${target.os}-${target.libc}`; bc._triplets[target.triplet] = true; - bc._downloadTriplets[buildId] = target.triplet; + bc._targetsByBuildIdCache[buildId] = target; let triple = [target.arch, target.vendor, target.os, target.libc]; for (let term of triple) { diff --git a/_webi/lint-builds.js b/_webi/lint-builds.js index 6eea202..7b67382 100644 --- a/_webi/lint-builds.js +++ b/_webi/lint-builds.js @@ -82,9 +82,83 @@ let bc = BuildsCacher.create({ installers: INSTALLERS_DIR, }); -let packagesByName = {}; +async function getPackagesWithBuilds(installersDir, pkgNames, parallel = 25) { + let packages = []; + + await Parallel.run(parallel, pkgNames, getAll); + + async function getAll(name, i) { + let Releases = require(`${installersDir}/${name}/releases.js`); + let pkg = await bc.getBuilds({ + Releases: Releases, + name: name, + date: new Date(), + }); + packages[i] = pkg; + } + + return packages; +} + +function getBuildsByTarget(packages) { + let packagesByName = {}; + + for (let pkg of packages) { + let buildsByOs = getBuildsByOs(pkg); + packagesByName[pkg.name] = buildsByOs; + } + + return packagesByName; +} + +function getBuildsByOs(pkg) { + let buildsByOs = {}; + + for (let build of pkg.releases) { + // TODO check targets cache + let target = bc.classify(pkg, build); + if (!target) { + // ignore known, non-package extensions + continue; + } + + if (target.error) { + let err = target.error; + let code = err.code || ''; + console.error(`[ERROR]: ${code} ${pkg.name}: ${build.name}`); + console.error(`>>> ${err.message} <<<`); + console.error(pkg); + console.error(build); + console.error(`^^^ ${err.message} ^^^`); + console.error(err.stack); + continue; + } + + let buildsByRelease = getBuildsByRelease(buildsByOs, target); + buildsByRelease.push(build); + } + + return buildsByOs; +} + +function getBuildsByRelease(buildsByOs, target) { + if (!buildsByOs[target.os]) { + buildsByOs[target.os] = {}; + } + let buildsByArchLibc = buildsByOs[target.os]; + + let archLibc = `${target.arch}-${target.libc}`; + if (!buildsByArchLibc[archLibc]) { + buildsByArchLibc[archLibc] = []; + } + let buildsByRelease = buildsByArchLibc[archLibc]; + + return buildsByRelease; +} async function main() { + let packagesByName = {}; + // let names = ['{NAME}-win32.exe']; // for (let name of names) { // console.log(name); @@ -99,40 +173,25 @@ async function main() { bc.freshenRandomPackage(600 * 1000); - // let pkg = await bc.getBuilds({ - // name: name, - // date: new Date(), - // installers: INSTALLERS_DIR, - // caches: CACHE_DIR, - // }); - // let triples = []; - // for (let build of pkg.releases) { - // let triplet = await pkgToTriples('git', pkg, build); - // triples.push(triplet); - // } - // return triples; - // // process.exit(1) - let rows = []; let triples = []; let valids = Object.keys(dirs.valid); - console.info(`Fetching builds for`); - let limit = 25; - //let limit = 1; - await Parallel.run(limit, valids, async function (name, i) { - if (name === 'webi') { - // TODO fix the webi faux package - // (not sure why I even created it) - return; - } - console.info(` ${name}`); - let Releases = require(`${INSTALLERS_DIR}/${name}/releases.js`); - let pkg = await bc.getBuilds({ - Releases: Releases, - name: name, - date: new Date(), - }); + let index = valids.indexOf('webi'); + if (index >= 0) { + // TODO fix the webi faux package + // (not sure why I even created it) + void valids.splice(index, 1); + } + + let parallel = 25; + let packages = await getPackagesWithBuilds(INSTALLERS_DIR, valids, parallel); + let packagesTree = getBuildsByTarget(packages); + //console.log(`c`); + + console.info(`Fetching builds for`); + for (let pkg of packages) { + console.info(` ${pkg.name}`); if (!packagesByName[pkg.name]) { let buildsByOs = {}; packagesByName[pkg.name] = buildsByOs; @@ -142,21 +201,24 @@ async function main() { let nStr = pkg.releases.length.toString(); let n = nStr.padStart(5, ' '); - let row = `##### ${n}\t${name}\tv`; + let row = `##### ${n}\t${pkg.name}\tv`; rows.push(row); // ignore known, non-package extensions for (let build of pkg.releases) { let target; - try { - target = bc.classify(pkg, build); - } catch (e) { + target = bc.classify(pkg, build); + if (!target) { + // non-build file + continue; + } + if (target.error) { + let e = target.error; if (e.code === 'E_BUILD_NO_PATTERN') { console.warn(`>>> ${e.message} <<<`); console.warn(pkg); console.warn(build); console.warn(`^^^ ${e.message} ^^^`); - continue; } throw e; } @@ -164,25 +226,14 @@ async function main() { continue; } - let buildsByOs = packagesByName[pkg.name]; - let buildsByArchLibc = buildsByOs[target.os]; - if (!buildsByArchLibc) { - buildsByArchLibc = {}; - buildsByOs[target.os] = buildsByArchLibc; - } - - let archLibc = `${target.arch}-${target.libc}`; - let buildsByRelease = buildsByArchLibc[archLibc]; - if (!buildsByRelease) { - buildsByRelease = []; - buildsByArchLibc[archLibc] = buildsByRelease; - } - buildsByRelease.push(build); - triples.push(target.triplet); rows.push(`${target.triplet}\t${pkg.name}\t${build.version}`); } - }); + + let buildsByOs = getBuildsByOs(pkg); + packagesByName[pkg.name] = buildsByOs; + } + let tsv = rows.join('\n'); console.info(''); console.info('#rows', rows.length);