WIP: feat(webi): show supported channels

.github/workflows/node.js.yml

@@ -23,15 +23,10 @@ jobs:
           sh ./_scripts/install-ci-deps
           echo "${HOME}/.local/bin" >> $GITHUB_PATH
           echo "${HOME}/.local/opt/pwsh" >> $GITHUB_PATH
-      - run: |
-          git submodule init
-          git submodule update
       - run: shfmt --version
       - run: shellcheck -V
       - run: node --version
       - run: npm run fmt
       - run: npm ci
       - run: npm run lint
-      - env:
-          GITHUB_TOKEN: ${{ github.token }}
-        run: npm run test
+      - run: npm run test

_webi/builds-cacher.js

@@ -54,54 +54,6 @@ var TERMS_META = [
   'stable',
 ];
 
-/** @typedef {String} TripletString - {arch}-{vendor}-{os}-{libc} */
-/** @typedef {String} VersionString */
-/** @typedef {Object.<VersionString, Array<BuildAsset>>} PackagesByRelease */
-
-/**
- * @typedef ProjectInfo
- * @prop {Array<BuildAsset>} releases
- * @prop {Array<BuildAsset>} packages
- * @prop {Object.<TripletString, PackagesByRelease>} releasesByTriplet
- * @prop {Array<import('./build-classifier/types.js').ArchString>} arches
- * @prop {Array<import('./build-classifier/types.js').OsString>} oses
- * @prop {Array<import('./build-classifier/types.js').LibcString>} libcs
- * @prop {Array<String>} channels
- * @prop {Array<String>} formats
- * @prop {Array<String>} triplets
- * @prop {Array<String>} versions
- * @prop {Array<String>} lexvers
- * @prop {Object.<String, String>} lexversMap
- */
-
-/**
- * @typedef BuildAsset
- * @prop {String} name
- * @prop {String} version
- * @prop {Boolean} lts
- * @prop {String} date
- * @prop {String} arch
- * @prop {String} os
- * @prop {String} libc
- * @prop {String} ext
- * @prop {String} download
- */
-
-/**
- * @typedef VersionTarget
- * @prop {String} version
- * @prop {Boolean} lts
- * @prop {String} channel
- */
-
-/** @typedef {TargetTriplet & HostTargetPartial} HostTarget */
-/** @typedef {import('./build-classifier/types.js').TargetTriplet} TargetTriplet */
-/**
- * @typedef HostTargetPartial
- * @prop {String} target.triplet - os-vendor-arch-libc
- * @prop {Error} target.error
- */
-
 async function getPartialHeader(path) {
   let readme = `${path}/README.md`;
   let head = await readFirstBytes(readme).catch(function (err) {
@@ -120,7 +72,7 @@ async function readFirstBytes(path) {
   let start = 0;
   let n = 1024;
   let fh = await Fs.open(path, 'r');
-  let buf = Buffer.alloc(n);
+  let buf = new Buffer.alloc(n);
   let result = await fh.read(buf, start, n);
   let str = result.buffer.toString('utf8');
   await fh.close();
@@ -415,10 +367,6 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
         continue;
       }
 
-      if (!build.name) {
-        build.name = build.download.replace(/.*\//, '');
-      }
-
       build.target = buildTarget;
       meta.packages.push(build);
     }
@@ -542,7 +490,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
 
     let pattern = Triplet.toPattern(projInfo, build);
     if (!pattern) {
-      let err = new Error(`no pattern generated for ${projInfo.name}`);
+      let err = new Error(`no pattern generated for ${name}`);
       err.code = 'E_BUILD_NO_PATTERN';
       target = { error: err };
       bc._targetsByBuildIdCache[buildId] = target;
@@ -786,35 +734,21 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
   };
 
   /**
-   * @param {ProjectInfo} projInfo
-   * @param {HostTarget} hostTarget
+   * @param {Object} target
+   * @param {String} target.os - linux, darwin, windows, *bsd
+   * @param {String} target.arch - arm64, x86_64, ppc64le
+   * @param {String} target.libc - none, libc, gnu, musl, bionic, msvc
+   * @param {String} target.triplet - os-vendor-arch-libc
+   * @param {Error} target.error
    */
-  bc.enumerateLatestVersions = function (projInfo, hostTarget) {
-    let lexPrefix = '';
-    let matchInfo = Lexver.matchSorted(projInfo.lexvers, lexPrefix);
-    let verInfo = {
-      default: projInfo.lexversMap[matchInfo.default],
-      previous: projInfo.lexversMap[matchInfo.previous],
-      stable: projInfo.lexversMap[matchInfo.stable],
-      latest: projInfo.lexversMap[matchInfo.latest],
-    };
-
-    return verInfo;
-  };
-
-  /**
-   * @param {ProjectInfo} projInfo
-   * @param {HostTarget} hostTarget
-   * @param {VersionTarget} verTarget
-   */
-  bc.findMatchingPackages = function (projInfo, hostTarget, verTarget) {
-    let matchInfo = bc._enumerateVersions(projInfo, verTarget.version);
+  bc.findMatchingPackages = function (pkgInfo, hostTarget, verTarget) {
+    let matchInfo = bc._enumerateVersions(pkgInfo, verTarget.version);
     let triplets = bc._enumerateTriplets(hostTarget);
     //console.log('dbg: matchInfo', matchInfo);
 
     if (matchInfo) {
       for (let _triplet of triplets) {
-        let targetReleases = projInfo.releasesByTriplet[_triplet];
+        let targetReleases = pkgInfo.releasesByTriplet[_triplet];
         if (!targetReleases) {
           continue;
         }
@@ -822,7 +756,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
         // Make sure that these releases are the expected version
         // (ex: jq1.7 => darwin-arm64-libc, jq1.6 => darwin-x86_64-libc)
         for (let matchver of matchInfo.matches) {
-          let ver = projInfo.lexversMap[matchver] || matchver;
+          let ver = pkgInfo.lexversMap[matchver] || matchver;
           let packages = targetReleases[ver];
           if (!packages) {
             continue;
@@ -831,7 +765,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
           let match = {
             triplet: _triplet,
             packages: packages,
-            latest: projInfo.versions[0],
+            latest: pkgInfo.versions[0],
             version: ver,
             versions: matchInfo,
           };
@@ -843,7 +777,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     }
 
     for (let _triplet of triplets) {
-      let targetReleases = projInfo.releasesByTriplet[_triplet];
+      let targetReleases = pkgInfo.releasesByTriplet[_triplet];
       if (!targetReleases) {
         continue;
       }
@@ -862,7 +796,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
       // Make sure that these releases are the expected version
       // (ex: jq1.7 => darwin-arm64-libc, jq1.6 => darwin-x86_64-libc)
       for (let matchver of lexvers) {
-        let ver = projInfo.lexversMap[matchver] || matchver;
+        let ver = pkgInfo.lexversMap[matchver] || matchver;
         let packages = targetReleases[ver];
         //console.log('dbg: packages', packages);
         if (!packages) {
@@ -878,7 +812,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
           let match = {
             triplet: _triplet,
             packages: packages,
-            latest: projInfo.versions[0],
+            latest: pkgInfo.versions[0],
             version: ver,
             versions: matchInfo,
           };
@@ -897,7 +831,7 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
         let match = {
           triplet: _triplet,
           packages: packages,
-          latest: projInfo.versions[0],
+          latest: pkgInfo.versions[0],
           version: ver,
           versions: matchInfo,
         };
@@ -944,12 +878,12 @@ BuildsCacher.create = function ({ ALL_TERMS, installers, caches }) {
     return triplets;
   };
 
-  bc._enumerateVersions = function (projInfo, ver) {
+  bc._enumerateVersions = function (pkgInfo, ver) {
     if (!ver) {
       return null;
     }
     let lexPrefix = Lexver.parsePrefix(ver);
-    let matchInfo = Lexver.matchSorted(projInfo.lexvers, lexPrefix);
+    let matchInfo = Lexver.matchSorted(pkgInfo.lexvers, lexPrefix);
 
     return matchInfo;
   };

_webi/builds.js

@@ -34,8 +34,7 @@ Builds.init = async function () {
   }
 };
 
-Builds.enumerateLatestVersions = bc.enumerateLatestVersions;
-Builds.findMatchingPackages = bc.findMatchingPackages;
-Builds.getPackage = bc.getPackages;
 Builds.getProjectType = bc.getProjectType;
+Builds.getPackage = bc.getPackages;
+Builds.findMatchingPackages = bc.findMatchingPackages;
 Builds.selectPackage = bc.selectPackage;

_webi/installers.js

@@ -103,6 +103,7 @@ Installers.renderBash = async function (
     ['PKG_ARCHES', (rel.arches || []).join(' ')],
     ['PKG_LIBCS', (rel.libcs || []).join(' ')],
     ['PKG_FORMATS', (rel.formats || []).join(' ')],
+    ['PKG_CHANNELS', (rel.channels || []).join(' ')],
     ['PKG_LATEST', latest],
   ];
 

_webi/lint-builds.js

@@ -125,7 +125,6 @@ let bc = BuildsCacher.create({
 });
 
 async function main() {
-  /* jshint maxcomplexity: 25 */
   // TODO
   //     node ./_webi/lint-builds.js caddy@beta 'x86_64/unknown Darwin libc'
   //
@@ -145,6 +144,13 @@ async function main() {
   let triples = [];
   let valids = Object.keys(dirs.valid);
 
+  let index = valids.indexOf('webi');
+  if (index >= 0) {
+    // TODO fix the webi faux package
+    // (not sure why I even created it)
+    void valids.splice(index, 1);
+  }
+
   if (projName) {
     if (!valids.includes(projName)) {
       throw new Error(`'${projName}' is not a valid installable project`);
@@ -185,7 +191,6 @@ async function main() {
         // non-build file
         continue;
       }
-
       if (target.error) {
         let e = target.error;
         if (e.code === 'E_BUILD_NO_PATTERN') {
@@ -197,12 +202,6 @@ async function main() {
         throw e;
       }
 
-      if (target.unknownTerms?.length) {
-        let msg = `${projInfo.name}: unrecognized term(s) '${target.unknownTerms}' in '${build.download}'`;
-        let err = new Error(msg);
-        throw err;
-      }
-
       triples.push(target.triplet);
       // if (!build.version) {
       //   throw new Error(`no version for ${pkg.name} ${build.name}`);

_webi/serve-installer.js

@@ -74,15 +74,28 @@ InstallerServer.helper = async function ({
 
   let validTypes = ['alias', 'selfhosted', 'valid'];
   if (!validTypes.includes(proj.type)) {
-    let msg = `'${projectName}' doesn't have an installer: '${proj.type}': '${proj.detail}'`;
-    let err = new Error(msg);
-    err.code = 'ENOENT';
-    throw err;
+    throw new Error(
+      `'${projectName}' doesn't have an installer: '${proj.type}': '${proj.detail}'`,
+    );
   }
   if (proj.type === 'alias') {
     projectName = proj.detail;
   }
 
+  let projInfo = await Builds.getPackage({
+    name: projectName,
+    date: new Date(),
+  });
+  //console.log('projInfo', projInfo);
+
+  let buildTargetInfo = {
+    triplets: projInfo.triplets,
+    oses: projInfo.oses,
+    arches: projInfo.arches,
+    libcs: projInfo.libcs,
+    formats: projInfo.formats,
+  };
+
   let tmplParams = {
     pkg: projectName,
     tag: tag,
@@ -92,6 +105,8 @@ InstallerServer.helper = async function ({
     formats: hostFormats,
     limit: 1,
   };
+  let latest = projInfo.versions[0];
+  Object.assign(tmplParams, { latest });
   Object.assign(tmplParams, releaseTarget);
   console.log('tmplParams', tmplParams);
 
@@ -112,30 +127,7 @@ InstallerServer.helper = async function ({
       "Check query parameters. Should be something like '/api/releases/{package}@{version}.tab?os={macos|linux|windows|-}&arch={amd64|x86|aarch64|arm64|armv7l|-}&libc={musl|gnu|msvc|libc|static}&limit=10'",
   };
 
-  if (proj.type === 'selfhosted') {
-    return [errPackage, tmplParams];
-  }
-
-  let projInfo = await Builds.getPackage({
-    name: projectName,
-    date: new Date(),
-  });
-  let latest = projInfo.versions[0];
-  Object.assign(tmplParams, { latest });
-  //console.log('projInfo', projInfo);
-
-  let buildTargetInfo = {
-    triplets: projInfo.triplets,
-    oses: projInfo.oses,
-    arches: projInfo.arches,
-    libcs: projInfo.libcs,
-    formats: projInfo.formats,
-  };
-
   let hasOs = projInfo.oses.includes(hostTarget.os);
-  if (!hasOs) {
-    hasOs = projInfo.oses.includes('ANYOS');
-  }
   if (!hasOs) {
     let pkg1 = Object.assign(buildTargetInfo, errPackage);
     return [pkg1, tmplParams];
@@ -155,6 +147,11 @@ InstallerServer.helper = async function ({
     return [pkg1, tmplParams];
   }
 
+  if (!targetRelease.packages) {
+    let pkg1 = Object.assign(buildTargetInfo, errPackage);
+    return [pkg1, tmplParams];
+  }
+
   let buildPkg = Builds.selectPackage(targetRelease.packages, hostFormats);
   let ext = buildPkg.ext || '.exe';
   if (ext.startsWith('.')) {

_webi/test.js

@@ -32,7 +32,7 @@ if (/\b-?-h(elp)?\b/.test(process.argv.join(' '))) {
 var os = require('os');
 var fs = require('fs');
 var path = require('path');
-var Builds = require('./builds.js');
+var Releases = require('./transform-releases.js');
 var Installers = require('./installers.js');
 var ServeInstaller = require('./serve-installer.js');
 
@@ -65,8 +65,7 @@ console.info('Has the necessary files?');
   });
 
 console.info('');
-let projName = pkgdir.split('/').filter(Boolean).pop();
-Builds.getPackage({ name: projName }).then(async function (/*projInfo*/) {
+Releases.get(path.join(process.cwd(), pkgdir)).then(async function (all) {
   var pkgname = path.basename(pkgdir.replace(/\/$/, ''));
   var nodeOs = os.platform();
   var nodeOsRelease = os.release();
@@ -83,8 +82,8 @@ Builds.getPackage({ name: projName }).then(async function (/*projInfo*/) {
   var formats = ['exe', 'xz', 'tar', 'zip', 'git'];
 
   let [rel, opts] = await ServeInstaller.helper({
-    unameAgent: `${nodeOs}/${nodeOsRelease} ${nodeArch}/unknown ${nodeLibc}`,
-    projectName: pkgname,
+    ua: `${nodeOs}/${nodeOsRelease} ${nodeArch}/unknown ${nodeLibc}`,
+    pkg: pkgname,
     tag: pkgtag || '',
     formats: formats,
     libc: nodeLibc,

flutter/releases.js

@@ -66,10 +66,10 @@ module.exports = async function (request) {
       json: true,
     });
 
-    let osBaseUrl = resp.body.base_url;
-    let osReleases = resp.body.releases;
+    let body = resp.body;
+    all.download = `${body.base_url}/{{ download }}`;
 
-    for (let asset of osReleases) {
+    for (let asset of body.releases) {
       if (!channelMap[asset.channel]) {
         channelMap[asset.channel] = true;
       }
@@ -81,8 +81,7 @@ module.exports = async function (request) {
         channel: asset.channel,
         date: asset.release_date.replace(/T.*/, ''),
         //sha256: asset.sha256,
-        download: `${osBaseUrl}/${asset.archive}`,
-        _filename: asset.archive,
+        download: asset.archive,
       });
     }
   }

go/releases.js

@@ -44,7 +44,7 @@ function getAllReleases(request) {
     var goReleases = resp.body;
     var all = {
       releases: [],
-      download: '',
+      download: 'https://dl.google.com/go/{{ download }}',
     };
 
     goReleases.forEach((release) => {
@@ -77,7 +77,7 @@ function getAllReleases(request) {
           arch: arch,
           ext: '', // let normalize run the split/test/join
           hash: '-', // not ready to standardize this yet
-          download: `https://dl.google.com/go/${filename}`,
+          download: filename,
         });
       });
     });

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@webinstall/webi-installers",
-  "version": "1.1.1",
+  "version": "1.0.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@webinstall/webi-installers",
-      "version": "1.1.1",
+      "version": "1.0.1",
       "license": "MPL-2.0",
       "dependencies": {
         "@root/request": "^1.9.2",

package.json

@@ -1,11 +1,10 @@
 {
   "name": "@webinstall/webi-installers",
   "private": true,
-  "version": "1.1.1",
+  "version": "1.0.1",
   "description": "webinstall script repository",
   "main": "_webi/",
   "scripts": {
-    "bump": "npm version -m \"chore(release): bump to v%s\"",
     "fmt": "npm run prettier && npm run shfmt && npm run pwsh-fmt",
     "lint": "npm run shellcheck && npm run jshint && npm run pwsh-lint",
     "prepare": "npm run tooling-init && npm run git-hooks-init",

webi/releases.js

@@ -0,0 +1,43 @@
+'use strict';
+
+module.exports = async function () {
+  return {
+    releases: [
+      {
+        name: 'webi.tar',
+        version: '0.0.0',
+        lts: false,
+        channel: 'stable',
+        date: '',
+        os: 'linux',
+        arch: 'amd64',
+        ext: 'tar',
+        download: '',
+      },
+      {
+        name: 'webi.tar',
+        version: '0.0.0',
+        lts: false,
+        channel: 'stable',
+        date: '',
+        os: 'macos',
+        arch: 'amd64',
+        ext: 'tar',
+        download: '',
+      },
+      {
+        name: 'webi.tar',
+        version: '0.0.0',
+        lts: false,
+        channel: 'stable',
+        date: '',
+        os: 'windows',
+        arch: 'amd64',
+        ext: 'tar',
+        download: '',
+      },
+    ],
+    formats: [],
+    downloads: '',
+  };
+};