fix(yq): filter out special non-build files

chromedriver/releases.js

@@ -1,81 +1,77 @@
 'use strict';
 
-var matchers = {
-  key: /.*Key>(.*)<\/Key.*/,
-  generation: /.*Generation>(.*)<\/Generation.*/,
-  metaGeneration: /.*MetaGeneration>(.*)<\/MetaGeneration.*/,
-  lastModified: /.*LastModified>(.*)<\/LastModified.*/,
-  etag: /.*ETag>(.*)<\/ETag.*/,
-  size: /.*Size>(.*)<\/Size.*/,
-};
-var baseUrl = 'https://chromedriver.storage.googleapis.com';
+// See <https://googlechromelabs.github.io/chrome-for-testing/>
+var releaseApiUrl =
+  'https://googlechromelabs.github.io/chrome-for-testing/known-good-versions-with-downloads.json';
 
-module.exports = function (request) {
-  var all = {
+// {
+//   "timestamp": "2023-11-15T21:08:56.730Z",
+//   "versions": [
+//     {
+//       "version": "121.0.6120.0",
+//       "revision": "1222902",
+//       "downloads": {
+//         "chrome": [],
+//         "chromedriver": [
+//           {
+//             "platform": "linux64",
+//             "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/121.0.6120.0/linux64/chromedriver-linux64.zip"
+//           },
+//           {
+//             "platform": "mac-arm64",
+//             "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/121.0.6120.0/mac-arm64/chromedriver-mac-arm64.zip"
+//           },
+//           {
+//             "platform": "mac-x64",
+//             "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/121.0.6120.0/mac-x64/chromedriver-mac-x64.zip"
+//           },
+//           {
+//             "platform": "win32",
+//             "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/121.0.6120.0/win32/chromedriver-win32.zip"
+//           },
+//           {
+//             "platform": "win64",
+//             "url": "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/121.0.6120.0/win64/chromedriver-win64.zip"
+//           }
+//         ],
+//         "chrome-headless-shell": []
+//       }
+//     }
+//   ]
+// }
+
+module.exports = async function (request) {
+  let resp = await request({
+    url: releaseApiUrl,
+    json: true,
+  });
+
+  let builds = [];
+  for (let release of resp.body.versions) {
+    if (!release.downloads.chromedriver) {
+      continue;
+    }
+
+    let version = release.version;
+    for (let asset of release.downloads.chromedriver) {
+      let build = {
+        version: version,
+        download: asset.url,
+        // I' not sure that this is actually statically built but it
+        // seems to be and at worst we'll just get bug reports for Apline
+        libc: 'none',
+      };
+
+      builds.push(build);
+    }
+  }
+
+  let all = {
     download: '',
-    releases: [],
+    releases: builds,
   };
 
-  // XML
-  return request({
-    url: 'https://chromedriver.storage.googleapis.com/',
-    json: false,
-  })
-    .then(function (resp) {
-      var body = resp.body;
-      var groups = body.split(/<\/?Contents>/g);
-      // get rid of leading and trailing junk
-      groups.shift();
-      groups.pop();
-      var metas = groups.map(function (group) {
-        return {
-          key: group.replace(matchers.key, '$1'),
-          //generation: group.replace(matchers.generation, '$1'),
-          //metaGeneration: group.replace(matchers.metaGeneration, '$1'),
-          lastModified: group.replace(matchers.lastModified, '$1'),
-          //etag: group.replace(matchers.etag, '$1'),
-          //size: group.replace(matchers.size, '$1')
-        };
-      });
-      all.download = baseUrl + '/{{ download }}';
-      metas.forEach(function (asset) {
-        if (!asset.key.includes('chromedriver')) {
-          // skip the indexes, images, etc
-          return null;
-        }
-
-        var osname = asset.key.replace(/.*(win|mac|linux)/, '$1');
-        var arch;
-        if (asset.key.includes('linux')) {
-          osname = 'linux';
-        } else if (asset.key.includes('mac64')) {
-          osname = 'macos';
-          if (asset.key.includes('_m1.')) {
-            arch = 'arm64';
-          }
-        } else if (asset.key.includes('win')) {
-          osname = 'windows';
-          arch = 'amd64';
-        }
-        all.releases.push({
-          // 87.0.4280.88/chromedriver_win32.zip => 87.0.4280.88
-          version: asset.key.replace(/(.*)\/.*/, '$1'),
-          lts: false,
-          channel: 'stable',
-          date: asset.lastModified.replace(/T.*/, '$1'),
-          os: osname,
-          arch: arch,
-          hash: '-', // not sure about including etag as hash yet
-          download: asset.key,
-        });
-      });
-    })
-    .then(function () {
-      all.releases.sort(function (a, b) {
-        return new Date(b.date).valueOf() - new Date(a.date).valueOf();
-      });
-      return all;
-    });
+  return all;
 };
 
 if (module === require.main) {

cmake/releases.js

@@ -7,6 +7,10 @@ var repo = 'CMake';
 module.exports = function (request) {
   return github(request, owner, repo).then(function (all) {
     for (let rel of all.releases) {
+      if (rel.version.startsWith('v')) {
+        rel._version = rel.version.slice(1);
+      }
+
       {
         let linuxRe = /(\b|_)(linux|gnu)(\b|_)/i;
         let isLinux = linuxRe.test(rel.download) || linuxRe.test(rel.name);

dashcore/releases.js

@@ -6,14 +6,16 @@ var repo = 'dash';
 
 module.exports = function (request) {
   return github(request, owner, repo).then(function (all) {
-    all.releases = all.releases.filter(function (rel) {
-      return !rel.name.endsWith('.asc');
-    });
     all.releases.forEach(function (rel) {
       if (rel.name.includes('osx64')) {
         rel.os = 'macos';
       }
+
+      if (rel.version.startsWith('v')) {
+        rel._version = rel.version.slice(1);
+      }
     });
+
     all._names = ['dashd', 'dashcore'];
     return all;
   });

fd/releases.js

@@ -6,6 +6,17 @@ var repo = 'fd';
 
 module.exports = function (request) {
   return github(request, owner, repo).then(function (all) {
+    let builds = [];
+
+    for (let build of all.releases) {
+      if (build.name === 'fd') {
+        continue;
+      }
+
+      builds.push(build);
+    }
+
+    all.releases = builds;
     return all;
   });
 };

flutter/releases.js

@@ -76,12 +76,11 @@ module.exports = async function (request) {
 
       all.releases.push({
         version: asset.version,
+        _version: `${asset.version}-${asset.channel}`,
         lts: false,
         channel: asset.channel,
         date: asset.release_date.replace(/T.*/, ''),
-        os: osname,
-        arch: 'amd64',
-        hash: asset.hash,
+        //sha256: asset.sha256,
         download: asset.archive,
       });
     }

go/releases.js

@@ -7,6 +7,17 @@ var archMap = {
   386: 'x86',
 };
 
+let ODDITIES = ['bootstrap', '-arm6.'];
+
+function isOdd(filename) {
+  for (let oddity of ODDITIES) {
+    let isOddity = filename.includes(oddity);
+    if (isOddity) {
+      return true;
+    }
+  }
+}
+
 function getAllReleases(request) {
   /*
   {
@@ -47,8 +58,8 @@ function getAllReleases(request) {
       var fileversion = release.version.slice(2);
 
       release.files.forEach((asset) => {
-        let isArtifact = asset.filename.includes('bootstrap');
-        if (isArtifact) {
+        let odd = isOdd(asset.filename);
+        if (odd) {
           return;
         }
 

jq/releases.js

@@ -4,11 +4,32 @@ var github = require('../_common/github.js');
 var owner = 'stedolan';
 var repo = 'jq';
 
+let ODDITIES = ['-no-oniguruma'];
+
+function isOdd(build) {
+  for (let oddity of ODDITIES) {
+    let isOddity = build.name.includes(oddity);
+    if (isOddity) {
+      return true;
+    }
+  }
+}
+
 module.exports = function (request) {
   return github(request, owner, repo).then(function (all) {
-    all.releases.forEach(function (rel) {
-      rel.version = String(rel.version).replace(/^jq\-/, '');
-    });
+    let builds = [];
+
+    for (let build of all.releases) {
+      let odd = isOdd(build);
+      if (odd) {
+        continue;
+      }
+
+      build.version = build.version.replace(/^jq\-/, '');
+      builds.push(build);
+    }
+
+    all.releases = builds;
     return all;
   });
 };

kubectx/releases.js

@@ -6,7 +6,23 @@ var repo = 'kubectx';
 
 module.exports = function (request) {
   return github(request, owner, repo).then(function (all) {
-    all._names = ['kubectx', 'kubens'];
+    let builds = [];
+
+    for (let build of all.releases) {
+      // this installs separately
+      if (build.name.includes('kubens')) {
+        continue;
+      }
+
+      // this is the legacy bash script
+      if (build.name === 'kubectx') {
+        continue;
+      }
+
+      builds.push(build);
+    }
+
+    all.releases = builds;
     return all;
   });
 };

kubens/releases.js

@@ -6,10 +6,23 @@ var repo = 'kubectx';
 
 module.exports = function (request) {
   return github(request, owner, repo).then(function (all) {
-    // remove kubectx, etc. from kubens list
-    all.releases = all.releases.filter(function (rel) {
-      return !/(\.txt)|(kubectx)|(kubectx_)$/i.test(rel.name);
-    });
+    let builds = [];
+
+    for (let build of all.releases) {
+      // this installs separately
+      if (build.name.includes('kubectx')) {
+        continue;
+      }
+
+      // this is the legacy bash script
+      if (build.name === 'kubens') {
+        continue;
+      }
+
+      builds.push(build);
+    }
+
+    all.releases = builds;
     return all;
   });
 };

terraform/releases.js

@@ -1,15 +1,5 @@
 'use strict';
 
-let convert = {
-  freebsd: 'freebsd',
-  macos: 'darwin',
-  linux: 'linux',
-  windows: 'windows',
-  amd64: 'amd64',
-  arm: 'arm64',
-  386: 'x86',
-};
-
 function getAllReleases(request) {
   return request({
     url: 'https://releases.hashicorp.com/terraform/index.json',
@@ -26,9 +16,11 @@ function getAllReleases(request) {
         let r = {
           version: build.version,
           download: build.url,
-          os: convert[build.os],
-          arch: convert[build.arch],
-          channel: 'stable', // No other channels
+          // These are generic enough for the autodetect,
+          // and the per-file logic has proven to get outdated sooner
+          //os: convert[build.os],
+          //arch: convert[build.arch],
+          //channel: 'stable|-rc|-beta|-alpha',
         };
         all.releases.push(r);
       });

yq/releases.js

@@ -4,8 +4,31 @@ var github = require('../_common/github.js');
 var owner = 'mikefarah';
 var repo = 'yq';
 
+let ODDITIES = ['man_page_only'];
+
+function isOdd(build) {
+  for (let oddity of ODDITIES) {
+    let isOddity = build.name.includes(oddity);
+    if (isOddity) {
+      return true;
+    }
+  }
+}
+
 module.exports = function (request) {
   return github(request, owner, repo).then(function (all) {
+    let builds = [];
+
+    for (let build of all.releases) {
+      let odd = isOdd(build);
+      if (odd) {
+        continue;
+      }
+
+      builds.push(build);
+    }
+
+    all.releases = builds;
     return all;
   });
 };