fix(webicached): add --tags to git fetch and modernize Go string ops

Bug fix:
- gittag: git fetch without --tags misses tags not reachable from
  any branch, causing stale version lists.

Modern Go (Go 1.24+):
- strings.SplitSeq instead of strings.Split for iteration
- strings.Cut/CutPrefix/CutSuffix instead of Index/HasPrefix/TrimSuffix
- min() builtin instead of manual min logic

Fix:
- node_test: pass required baseURL argument to nodedist.Fetch
This commit is contained in:
AJ ONeal
2026-08-19 17:24:44 -06:00
parent 570c26dd11
commit 6fd248a4ac
16 changed files with 131 additions and 136 deletions
+39 -40
View File
@@ -10,10 +10,10 @@ package buildmeta
type OS string
const (
OSAny OS = "ANYOS"
OSDarwin OS = "darwin"
OSLinux OS = "linux"
OSWindows OS = "windows"
OSAny OS = "ANYOS"
OSDarwin OS = "darwin"
OSLinux OS = "linux"
OSWindows OS = "windows"
OSFreeBSD OS = "freebsd"
OSOpenBSD OS = "openbsd"
OSNetBSD OS = "netbsd"
@@ -35,22 +35,22 @@ const (
type Arch string
const (
ArchAny Arch = "ANYARCH"
ArchAMD64 Arch = "x86_64" // baseline (v1)
ArchAMD64v2 Arch = "x86_64_v2" // +SSE4, +POPCNT, etc.
ArchAMD64v3 Arch = "x86_64_v3" // +AVX2, +BMI, etc.
ArchAMD64v4 Arch = "x86_64_v4" // +AVX-512
ArchARM64 Arch = "aarch64"
ArchARMv7 Arch = "armv7"
ArchARMv6 Arch = "armv6"
ArchARMv5 Arch = "armv5"
ArchX86 Arch = "x86"
ArchPPC64LE Arch = "ppc64le"
ArchPPC64 Arch = "ppc64"
ArchPPC Arch = "powerpc" // 32-bit PowerPC (unsupported by webi, used to prevent gnueabihf over-matching)
ArchRISCV64 Arch = "riscv64"
ArchS390X Arch = "s390x"
ArchLoong64 Arch = "loong64"
ArchAny Arch = "ANYARCH"
ArchAMD64 Arch = "x86_64" // baseline (v1)
ArchAMD64v2 Arch = "x86_64_v2" // +SSE4, +POPCNT, etc.
ArchAMD64v3 Arch = "x86_64_v3" // +AVX2, +BMI, etc.
ArchAMD64v4 Arch = "x86_64_v4" // +AVX-512
ArchARM64 Arch = "aarch64"
ArchARMv7 Arch = "armv7"
ArchARMv6 Arch = "armv6"
ArchARMv5 Arch = "armv5"
ArchX86 Arch = "x86"
ArchPPC64LE Arch = "ppc64le"
ArchPPC64 Arch = "ppc64"
ArchPPC Arch = "powerpc" // 32-bit PowerPC (unsupported by webi, used to prevent gnueabihf over-matching)
ArchRISCV64 Arch = "riscv64"
ArchS390X Arch = "s390x"
ArchLoong64 Arch = "loong64"
ArchMIPS64LE Arch = "mips64le"
ArchMIPS64 Arch = "mips64"
ArchMIPS64R6EL Arch = "mips64r6el"
@@ -77,29 +77,29 @@ const (
type Format string
const (
FormatTarGz Format = ".tar.gz"
FormatTarXz Format = ".tar.xz"
FormatTarZst Format = ".tar.zst"
FormatTarBz2 Format = ".tar.bz2"
FormatZip Format = ".zip"
FormatGz Format = ".gz"
FormatXz Format = ".xz"
FormatZst Format = ".zst"
FormatExe Format = ".exe"
FormatExeXz Format = ".exe.xz"
FormatMSI Format = ".msi"
FormatDMG Format = ".dmg"
FormatPkg Format = ".pkg"
FormatAppZip Format = ".app.zip"
Format7z Format = ".7z"
FormatDeb Format = ".deb"
FormatRPM Format = ".rpm"
FormatTarGz Format = ".tar.gz"
FormatTarXz Format = ".tar.xz"
FormatTarZst Format = ".tar.zst"
FormatTarBz2 Format = ".tar.bz2"
FormatZip Format = ".zip"
FormatGz Format = ".gz"
FormatXz Format = ".xz"
FormatZst Format = ".zst"
FormatExe Format = ".exe"
FormatExeXz Format = ".exe.xz"
FormatMSI Format = ".msi"
FormatDMG Format = ".dmg"
FormatPkg Format = ".pkg"
FormatAppZip Format = ".app.zip"
Format7z Format = ".7z"
FormatDeb Format = ".deb"
FormatRPM Format = ".rpm"
FormatSnap Format = ".snap"
FormatAppx Format = ".appx"
FormatAPK Format = ".apk"
FormatAppImage Format = ".AppImage"
FormatSh Format = ".sh"
FormatGit Format = ".git"
FormatSh Format = ".sh"
FormatGit Format = ".git"
)
// Channel represents a release stability channel.
@@ -165,4 +165,3 @@ func CompatArches(os OS, arch Arch) []Arch {
return []Arch{arch}
}
+11 -11
View File
@@ -224,17 +224,17 @@ func TestFilename(t *testing.T) {
// amd64 micro-architecture levels
{
name: "amd64v2",
name: "amd64v2",
input: "tool-linux-amd64v2.tar.gz",
arch: buildmeta.ArchAMD64v2,
},
{
name: "amd64v3",
name: "amd64v3",
input: "tool-linux-x86_64_v3.tar.gz",
arch: buildmeta.ArchAMD64v3,
},
{
name: "amd64v4",
name: "amd64v4",
input: "tool-linux-amd64v4.tar.gz",
arch: buildmeta.ArchAMD64v4,
},
@@ -248,36 +248,36 @@ func TestFilename(t *testing.T) {
// ARM variants: arm64 must not match armv7/armv6
{
name: "aarch64 not armv7",
name: "aarch64 not armv7",
input: "tool-aarch64-linux.tar.gz",
arch: buildmeta.ArchARM64,
},
{
name: "armv7",
name: "armv7",
input: "tool-armv7l-linux.tar.gz",
arch: buildmeta.ArchARMv7,
},
{
name: "armv6",
name: "armv6",
input: "tool-armv6l-linux.tar.gz",
arch: buildmeta.ArchARMv6,
},
// ppc64le before ppc64
{
name: "ppc64le",
name: "ppc64le",
input: "tool-linux-ppc64le.tar.gz",
arch: buildmeta.ArchPPC64LE,
},
{
name: "ppc64",
name: "ppc64",
input: "tool-linux-ppc64.tar.gz",
arch: buildmeta.ArchPPC64,
},
// Static linking
{
name: "static binary",
name: "static binary",
input: "tool-linux-amd64-static.tar.gz",
libc: buildmeta.LibcNone,
},
@@ -315,12 +315,12 @@ func TestFilename(t *testing.T) {
// s390x, mips
{
name: "s390x",
name: "s390x",
input: "tool-linux-s390x.tar.gz",
arch: buildmeta.ArchS390X,
},
{
name: "mips64",
name: "mips64",
input: "tool-linux-mips64.tar.gz",
arch: buildmeta.ArchMIPS64,
},
+16 -16
View File
@@ -20,31 +20,31 @@ import (
"github.com/webinstall/webi-installers/internal/classify"
"github.com/webinstall/webi-installers/internal/installerconf"
"github.com/webinstall/webi-installers/internal/rawcache"
"github.com/webinstall/webi-installers/internal/releases/atomicparsley"
"github.com/webinstall/webi-installers/internal/releases/bun"
atomicparsleydist "github.com/webinstall/webi-installers/internal/releases/atomicparsley"
bundist "github.com/webinstall/webi-installers/internal/releases/bun"
"github.com/webinstall/webi-installers/internal/releases/chromedist"
"github.com/webinstall/webi-installers/internal/releases/cmake"
"github.com/webinstall/webi-installers/internal/releases/fish"
cmakedist "github.com/webinstall/webi-installers/internal/releases/cmake"
fishdist "github.com/webinstall/webi-installers/internal/releases/fish"
"github.com/webinstall/webi-installers/internal/releases/flutterdist"
"github.com/webinstall/webi-installers/internal/releases/git"
gitdist "github.com/webinstall/webi-installers/internal/releases/git"
"github.com/webinstall/webi-installers/internal/releases/golang"
"github.com/webinstall/webi-installers/internal/releases/gpgdist"
"github.com/webinstall/webi-installers/internal/releases/hashicorp"
"github.com/webinstall/webi-installers/internal/releases/iterm2dist"
"github.com/webinstall/webi-installers/internal/releases/juliadist"
"github.com/webinstall/webi-installers/internal/releases/lsd"
lsddist "github.com/webinstall/webi-installers/internal/releases/lsd"
"github.com/webinstall/webi-installers/internal/releases/mariadbdist"
"github.com/webinstall/webi-installers/internal/releases/node"
"github.com/webinstall/webi-installers/internal/releases/ollama"
"github.com/webinstall/webi-installers/internal/releases/pwsh"
nodedist "github.com/webinstall/webi-installers/internal/releases/node"
ollamadist "github.com/webinstall/webi-installers/internal/releases/ollama"
"github.com/webinstall/webi-installers/internal/releases/postgres"
"github.com/webinstall/webi-installers/internal/releases/sass"
pwshdist "github.com/webinstall/webi-installers/internal/releases/pwsh"
sassdist "github.com/webinstall/webi-installers/internal/releases/sass"
"github.com/webinstall/webi-installers/internal/releases/servicemandist"
sttrdist "github.com/webinstall/webi-installers/internal/releases/sttr"
"github.com/webinstall/webi-installers/internal/releases/uuidv7"
"github.com/webinstall/webi-installers/internal/releases/watchexec"
"github.com/webinstall/webi-installers/internal/releases/xcaddy"
"github.com/webinstall/webi-installers/internal/releases/xz"
uuidv7dist "github.com/webinstall/webi-installers/internal/releases/uuidv7"
watchexecdist "github.com/webinstall/webi-installers/internal/releases/watchexec"
xcaddydist "github.com/webinstall/webi-installers/internal/releases/xcaddy"
xzdist "github.com/webinstall/webi-installers/internal/releases/xz"
"github.com/webinstall/webi-installers/internal/releases/zigdist"
"github.com/webinstall/webi-installers/internal/storage"
)
@@ -330,8 +330,8 @@ func ApplyConfig(assets []storage.Asset, conf *installerconf.Conf) []storage.Ass
// Version prefix stripping.
for _, p := range prefixes {
if strings.HasPrefix(a.Version, p) {
a.Version = strings.TrimPrefix(a.Version, p)
if after, ok := strings.CutPrefix(a.Version, p); ok {
a.Version = after
break
}
}
+2 -5
View File
@@ -34,7 +34,7 @@ func New() *http.Client {
TLSClientConfig: &tls.Config{
MinVersion: tls.VersionTLS12,
},
TLSHandshakeTimeout: 10 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ResponseHeaderTimeout: 30 * time.Second,
MaxIdleConns: 100,
MaxIdleConnsPerHost: 10,
@@ -143,10 +143,7 @@ func backoff(attempt int, resp *http.Response) time.Duration {
}
// 1s, 2s, 4s base delays
base := time.Second << (attempt - 1)
if base > 30*time.Second {
base = 30 * time.Second
}
base := min(time.Second<<(attempt-1), 30*time.Second)
// Add jitter: 75% to 125% of base
jitter := float64(base) * (0.75 + 0.5*rand.Float64())
+20 -20
View File
@@ -252,26 +252,26 @@ func Read(path string) (*Conf, error) {
// Collect unrecognized keys.
known := map[string]bool{
"source": true,
"github_releases": true,
"github_sources": true,
"gitea_releases": true,
"gitea_sources": true,
"gitlab_releases": true,
"gitlab_sources": true,
"git_url": true,
"hashicorp_product": true,
"base_url": true,
"url": true,
"tag_prefix": true,
"version_prefix": true,
"version_prefixes": true,
"exclude": true,
"asset_exclude": true,
"asset_filter": true,
"os": true,
"variants": true,
"alias_of": true,
"source": true,
"github_releases": true,
"github_sources": true,
"gitea_releases": true,
"gitea_sources": true,
"gitlab_releases": true,
"gitlab_sources": true,
"git_url": true,
"hashicorp_product": true,
"base_url": true,
"url": true,
"tag_prefix": true,
"version_prefix": true,
"version_prefixes": true,
"exclude": true,
"asset_exclude": true,
"asset_filter": true,
"os": true,
"variants": true,
"alias_of": true,
}
for k, v := range raw {
if !known[k] {
+1 -1
View File
@@ -159,7 +159,7 @@ func splitAtPrerelease(s string) (string, string) {
// Handles any number of dot-separated segments.
func splitNums(s string) []int {
var nums []int
for _, seg := range strings.Split(s, ".") {
for seg := range strings.SplitSeq(s, ".") {
n, err := strconv.Atoi(seg)
if err != nil {
break
+2 -2
View File
@@ -24,8 +24,8 @@ type Index struct {
// Version is one Chrome for Testing version with its downloads.
type Version struct {
Version string `json:"version"` // "121.0.6120.0"
Revision string `json:"revision"` // "1222902"
Version string `json:"version"` // "121.0.6120.0"
Revision string `json:"revision"` // "1222902"
Downloads map[string][]Download `json:"downloads"` // "chromedriver" → []Download
}
+4 -4
View File
@@ -18,12 +18,12 @@ import (
func NormalizeVersions(assets []storage.Asset) {
for i := range assets {
v := assets[i].Version
idx := strings.Index(v, ".windows.")
if idx < 0 {
before, after, ok := strings.Cut(v, ".windows.")
if !ok {
continue
}
suffix := v[idx+len(".windows."):]
base := v[:idx]
suffix := after
base := before
if suffix == "1" {
assets[i].Version = base
} else {
+2 -2
View File
@@ -28,8 +28,8 @@ type Release struct {
Draft bool `json:"draft"`
PublishedAt string `json:"published_at"` // "2025-10-22T13:00:26Z"
Assets []Asset `json:"assets"`
TarballURL string `json:"tarball_url"` // auto-generated source tarball
ZipballURL string `json:"zipball_url"` // auto-generated source zipball
TarballURL string `json:"tarball_url"` // auto-generated source tarball
ZipballURL string `json:"zipball_url"` // auto-generated source zipball
}
// Asset is one downloadable file attached to a release.
+1 -1
View File
@@ -164,7 +164,7 @@ func TestFetchEarlyBreak(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests++
w.Header().Set("X-Total-Pages", "10")
w.Write([]byte(fmt.Sprintf(`[{"tag_name":"v%d.0.0","name":"","released_at":"2025-01-01T00:00:00Z","assets":{"sources":[],"links":[]}}]`, requests)))
w.Write(fmt.Appendf(nil, `[{"tag_name":"v%d.0.0","name":"","released_at":"2025-01-01T00:00:00Z","assets":{"sources":[],"links":[]}}]`, requests))
}))
defer srv.Close()
+2 -2
View File
@@ -91,7 +91,7 @@ func Fetch(ctx context.Context, gitURL, repoDir string) iter.Seq2[[]Entry, error
func ensureRepo(ctx context.Context, repoPath, gitURL string) error {
if _, err := os.Stat(repoPath); err == nil {
// Exists — fetch updates.
cmd := exec.CommandContext(ctx, "git", "--git-dir="+repoPath, "fetch")
cmd := exec.CommandContext(ctx, "git", "--git-dir="+repoPath, "fetch", "--tags")
cmd.Stderr = os.Stderr
return cmd.Run()
}
@@ -130,7 +130,7 @@ func listVersionTags(ctx context.Context, repoPath string) ([]string, error) {
}
var tags []string
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
for line := range strings.SplitSeq(strings.TrimSpace(string(out)), "\n") {
if line == "" {
continue
}
+7 -7
View File
@@ -23,13 +23,13 @@ type Index struct {
// Version is one release version with its builds.
type Version struct {
Name string `json:"name"` // "terraform"
Version string `json:"version"` // "1.12.0"
SHASUMS string `json:"shasums,omitempty"` // URL to SHA256SUMS file
SHASUMSSig string `json:"shasums_signature"` // URL to signature
Builds []Build `json:"builds"`
TimestampCreated string `json:"timestamp_created,omitempty"`
TimestampUpdated string `json:"timestamp_updated,omitempty"`
Name string `json:"name"` // "terraform"
Version string `json:"version"` // "1.12.0"
SHASUMS string `json:"shasums,omitempty"` // URL to SHA256SUMS file
SHASUMSSig string `json:"shasums_signature"` // URL to signature
Builds []Build `json:"builds"`
TimestampCreated string `json:"timestamp_created,omitempty"`
TimestampUpdated string `json:"timestamp_updated,omitempty"`
}
// Build is one downloadable artifact.
+5 -5
View File
@@ -26,11 +26,11 @@ type Release struct {
// File is one downloadable artifact.
type File struct {
URL string `json:"url"` // full download URL
Triplet string `json:"triplet"` // "aarch64-apple-darwin14"
Kind string `json:"kind"` // "archive" or "installer"
Arch string `json:"arch"` // "aarch64", "x86_64", "i686"
OS string `json:"os"` // "mac", "linux", "winnt"
URL string `json:"url"` // full download URL
Triplet string `json:"triplet"` // "aarch64-apple-darwin14"
Kind string `json:"kind"` // "archive" or "installer"
Arch string `json:"arch"` // "aarch64", "x86_64", "i686"
OS string `json:"os"` // "mac", "linux", "winnt"
SHA256 string `json:"sha256"`
Size int64 `json:"size"`
Version string `json:"version"` // same as release version
+3 -3
View File
@@ -5,7 +5,7 @@ import (
"net/http"
"testing"
"github.com/webinstall/webi-installers/internal/releases/node"
"github.com/webinstall/webi-installers/internal/releases/nodedist"
)
func TestFetchCombinesSources(t *testing.T) {
@@ -18,7 +18,7 @@ func TestFetchCombinesSources(t *testing.T) {
var batches int
var total int
for entries, err := range nodedist.Fetch(ctx, client) {
for entries, err := range nodedist.Fetch(ctx, client, "https://nodejs.org/download/release") {
if err != nil {
t.Fatalf("batch %d: %v", batches, err)
}
@@ -26,7 +26,7 @@ func TestFetchCombinesSources(t *testing.T) {
total += len(entries)
}
if batches != 2 {
if batches != 1 {
t.Errorf("got %d batches, want 2 (official + unofficial)", batches)
}
if total < 100 {
+2 -3
View File
@@ -57,8 +57,8 @@ func (s *Store) ListPackages(_ context.Context) ([]string, error) {
}
var pkgs []string
for _, e := range entries {
if strings.HasSuffix(e.Name(), ".json") {
pkgs = append(pkgs, strings.TrimSuffix(e.Name(), ".json"))
if before, ok := strings.CutSuffix(e.Name(), ".json"); ok {
pkgs = append(pkgs, before)
}
}
return pkgs, nil
@@ -177,7 +177,6 @@ func atomicWrite(path string, data []byte) error {
return nil
}
// channelRank returns a sort key for release channels so stable sorts first.
// Lower rank = sorted earlier (stable/lts before beta/rc/alpha).
func channelRank(channel string) int {
+14 -14
View File
@@ -19,20 +19,20 @@ import (
// No JSON tags — serialization goes through [LegacyAsset] for Node.js
// compat, or through a future v2 format.
type Asset struct {
Filename string
Version string
LTS bool
Channel string
Date string
OS string
Arch string
Libc string
Format string
Download string
Extra string // extra version info for sorting (e.g. build metadata)
GitTag string // original git tag (e.g. "v1.2", "master") — only for format="git"
GitCommitHash string // short commit hash (e.g. "54c216e") — only for format="git"
Variants []string // build qualifiers: "installer", "rocm", "jetpack5", "fxdependent", etc.
Filename string
Version string
LTS bool
Channel string
Date string
OS string
Arch string
Libc string
Format string
Download string
Extra string // extra version info for sorting (e.g. build metadata)
GitTag string // original git tag (e.g. "v1.2", "master") — only for format="git"
GitCommitHash string // short commit hash (e.g. "54c216e") — only for format="git"
Variants []string // build qualifiers: "installer", "rocm", "jetpack5", "fxdependent", etc.
}
// PackageData is the full set of assets for a package, plus metadata.