added go releases and refactored some common code

_common/github.js

@@ -1,47 +1,5 @@
 'use strict';
 
-// this may need customizations between packages
-const osMap = {
-  macos: /\b(apple|mac|darwin|iPhone|iOS|iPad)/i,
-  linux: /\b(linux)/i,
-  windows: /\b(win|microsoft|msft)/i,
-  sunos: /\b(sun)/i,
-  aix: /\b(aix)/i
-};
-
-// evaluation order matters
-// (i.e. otherwise x86 and x64 can cross match)
-var archArr = [
-  'amd64', // first and most likely match
-  'arm64',
-  'x86',
-  'ppc64le',
-  'ppc64',
-  'armv7l',
-  'armv6l',
-  's390x'
-];
-var archMap = {
-  amd64: /(amd.?64|x64|[_\-]64)/i,
-  x86: /(86)\b/i,
-  ppc64le: /\b(ppc64le)/i,
-  ppc64: /\b(ppc64)\b/i,
-  arm64: /\b(arm64|arm)/i,
-  armv7l: /\b(armv?7l)/i,
-  armv6l: /\b(armv?6l)/i,
-  s390x: /\b(s390x)/i
-};
-
-var fileExtMap = {
-  deb: /\.deb$/i,
-  pkg: /\.pkg$/i,
-  exe: /\.exe$/i,
-  msi: /\.msi$/i,
-  zip: /\.zip$/i,
-  tar: /\.tar\..*$/i,
-  '7z': /\.7z$/i
-};
-
 /**
  * Gets the releases for 'ripgrep'. This function could be trimmed down and made
  * for use with any github release.
@@ -65,50 +23,22 @@ function getAllReleases(request, owner = 'BurntSushi', repo = 'ripgrep') {
     const gHubResp = resp.body;
     const all = {
       releases: [],
+      // todo make this ':baseurl' + ':releasename'
       download: ''
     };
 
     gHubResp.forEach((release) => {
       release['assets'].forEach((asset) => {
-        // set the primary download to the first of the releases
-        if (all.download === '') {
-          all.download = asset['browser_download_url'];
-        }
-
         const name = asset['name'];
-        const os =
-          Object.keys(osMap).find(function (regKey) {
-            //console.log('github release os:', name, regKey, osMap[regKey]);
-            return osMap[regKey].test(name);
-          }) || 'unknown';
-        var arch;
-        archArr.some(function (regKey) {
-          //console.log('github release arch:', name, regKey, archMap[regKey]);
-          arch = name.match(archMap[regKey]) && regKey;
-          if (arch) {
-            return true;
-          }
-        })[0];
-
-        let fileExt = '';
-        Object.keys(fileExtMap).find((regKey) => {
-          const match = name.match(fileExtMap[regKey]);
-          if (match) {
-            fileExt = match[0];
-            return true;
-          }
-          return false;
-        });
-
         all.releases.push({
-          download: asset['browser_download_url'],
-          date: (release['published_at'] || '').replace(/T.*/, ''),
           version: release['tag_name'], // TODO tags aren't always semver / sensical
           lts: /\b(lts)\b/.test(release['tag_name']),
           channel: !release['prerelease'] ? 'stable' : 'beta',
-          ext: fileExt.slice(1),
-          arch,
-          os
+          date: (release['published_at'] || '').replace(/T.*/, ''),
+          os: '', // will be guessed by download filename
+          arch: '', // will be guessed by download filename
+          ext: '', // will be normalized
+          download: asset['browser_download_url']
         });
       });
     });

_common/normalize.js

@@ -0,0 +1,74 @@
+'use strict';
+
+// this may need customizations between packages
+const osMap = {
+  macos: /\b(apple|mac|darwin|iPhone|iOS|iPad)/i,
+  linux: /\b(linux)/i,
+  windows: /\b(win|microsoft|msft)/i,
+  sunos: /\b(sun)/i,
+  aix: /\b(aix)/i
+};
+
+// evaluation order matters
+// (i.e. otherwise x86 and x64 can cross match)
+var archArr = [
+  'amd64', // first and most likely match
+  'arm64',
+  'x86',
+  'ppc64le',
+  'ppc64',
+  'armv7l',
+  'armv6l',
+  's390x'
+];
+var archMap = {
+  amd64: /(amd.?64|x64|[_\-]64)/i,
+  x86: /(86)\b/i,
+  ppc64le: /\b(ppc64le)/i,
+  ppc64: /\b(ppc64)\b/i,
+  arm64: /\b(arm64|arm)/i,
+  armv7l: /\b(armv?7l)/i,
+  armv6l: /\b(armv?6l)/i,
+  s390x: /\b(s390x)/i
+};
+
+function normalize(all) {
+  all.releases.forEach(function (rel) {
+    if (!rel.os) {
+      rel.os =
+        Object.keys(osMap).find(function (regKey) {
+          //console.log('release os:', rel.download, regKey, osMap[regKey]);
+          return osMap[regKey].test(rel.download);
+        }) || 'unknown';
+    }
+
+    if (!rel.arch) {
+      archArr.some(function (regKey) {
+        //console.log('release arch:', rel.download, regKey, archMap[regKey]);
+        var arch = rel.download.match(archMap[regKey]) && regKey;
+        if (arch) {
+          rel.arch = arch;
+          return true;
+        }
+      })[0];
+    }
+
+    if (!rel.ext) {
+      // pkg-v1.0.tar.gz => ['gz', 'tar', '0', 'pkg-v1']
+      // pkg-v1.0.tar => ['tar', '0' ,'pkg-v1']
+      // pkg-v1.0.zip => ['zip', '0', 'pkg-v1']
+      var exts = rel.download.split('.').reverse().slice(0, 2);
+      var ext;
+      if ('tar' === exts[1]) {
+        rel.ext = exts.reverse().join('.');
+      } else if ('tgz' == exts[0]) {
+        rel.ext = 'tar.gz';
+      } else {
+        rel.ext = exts[0];
+      }
+    }
+  });
+  return all;
+}
+
+module.exports = normalize;

golang/releases.js

@@ -0,0 +1,78 @@
+'use strict';
+
+var osMap = {
+  darwin: 'macos'
+};
+var archMap = {
+  '386': 'x86'
+};
+
+function getAllReleases(request) {
+  /*
+  {
+    version: 'go1.13.8',
+    stable: true,
+    files: [
+      {
+        filename: 'go1.13.8.src.tar.gz',
+        os: '',
+        arch: '',
+        version: 'go1.13.8',
+        sha256:
+          'b13bf04633d4d8cf53226ebeaace8d4d2fd07ae6fa676d0844a688339debec34',
+        size: 21631178,
+        kind: 'source'
+      }
+    ]
+  };
+  */
+  return request({
+    url: 'https://golang.org/dl/?mode=json&include=all',
+    json: true
+  }).then((resp) => {
+    var goReleases = resp.body;
+    var all = {
+      releases: [],
+      download: 'https://dl.google.com/go/{{ download }}'
+    };
+
+    goReleases.forEach((release) => {
+      // strip 'go' prefix, standardize version
+      var parts = release.version.slice(2).split('.');
+      while (parts.length < 3) {
+        parts.push('0');
+      }
+      var version = parts.join('.');
+
+      release.files.forEach((asset) => {
+        var filename = asset.filename;
+        var os = osMap[asset.os] || asset.os || '-';
+        var arch = archMap[asset.arch] || asset.arch || '-';
+        all.releases.push({
+          version: version,
+          // all go versions >= 1.0.0 are effectively LTS
+          lts: (parts[0] > 0 && asset.stable) || false,
+          channel: asset.stable || 'beta',
+          date: '1970-01-01', // the world may never know
+          os: os,
+          arch: arch,
+          ext: '', // let normalize run the split/test/join
+          hash: '-', // not ready to standardize this yet
+          download: filename
+        });
+      });
+    });
+
+    return all;
+  });
+}
+
+module.exports = getAllReleases;
+
+if (module === require.main) {
+  getAllReleases(require('@root/request')).then(function (all) {
+    all = require('../_common/normalize.js')(all);
+    all.releases = all.releases.slice(0, 10);
+    console.log(JSON.stringify(all, null, 2));
+  });
+}