feat(webicached): rescan conf dir each batch, prioritize new packages

Rescans the conf directory at the start of each batch loop so new
{pkg}/releases.conf files dropped on disk are picked up without a restart.

Also runs a rescan after each individual package refresh mid-batch. If a
new conf is discovered, the inner loop breaks immediately so the outer
loop recomputes staleness — new packages have a zero timestamp and sort
to position 1, meaning they are fetched in the very next slot.
This commit is contained in:
AJ ONeal
2026-05-14 17:19:28 -06:00
parent f66822295b
commit 95418b1023

View File

@@ -206,11 +206,11 @@ func main() {
if err != nil {
log.Fatalf("discover: %v", err)
}
nameSet := make(map[string]bool, len(filterPkgs))
for _, a := range filterPkgs {
nameSet[a] = true
}
if len(filterPkgs) > 0 {
nameSet := make(map[string]bool, len(filterPkgs))
for _, a := range filterPkgs {
nameSet[a] = true
}
var filtered []pkgConf
for _, p := range packages {
if nameSet[p.name] {
@@ -227,8 +227,41 @@ func main() {
}
}
// rescanNew appends any conf files added since the last scan.
// Returns true when at least one new package was added so the caller
// can restart the batch loop and process new packages immediately.
rescanNew := func() bool {
discovered, err := discover(wc.ConfDir)
if err != nil {
log.Printf("rescan: %v", err)
return false
}
known := make(map[string]bool, len(real))
for _, p := range real {
known[p.name] = true
}
added := false
for _, p := range discovered {
if p.conf.AliasOf != "" || known[p.name] {
continue
}
if len(filterPkgs) > 0 && !nameSet[p.name] {
continue
}
log.Printf("discovered new package: %s (source=%s)", p.name, p.conf.Source)
real = append(real, p)
added = true
}
return added
}
log.Printf("refreshing %d packages, interval %s, batch size 20 (ctrl-c to stop)", len(real), cfg.interval)
for {
// Rescan before computing staleness so newly added conf files are
// included immediately. New packages have a zero timestamp and sort
// to the front of the stale list, so they are processed next.
rescanNew()
stale := wc.stalest(real)
if len(stale) == 0 {
log.Printf("all packages fresh, sleeping %s", cfg.interval)
@@ -252,6 +285,10 @@ func main() {
}
cancel()
time.Sleep(cfg.interval)
// Rescan mid-batch so new packages preempt remaining batch items.
if rescanNew() {
break
}
}
}
}