From b4cecfd4fea506ea9d0bc363ee5e1eadceec718b Mon Sep 17 00:00:00 2001 From: Thibault VINCENT Date: Tue, 5 May 2026 14:43:36 +0200 Subject: [PATCH] test(k8s): add memory smoke tests, ConfigMap delete coverage, and a sync benchmark --- internal/source/k8s/k8s_test.go | 199 ++++++++++++++++++++++++++++++++ test/e2e/e2e_test.go | 27 +++++ 2 files changed, 226 insertions(+) diff --git a/internal/source/k8s/k8s_test.go b/internal/source/k8s/k8s_test.go index 8e42cd4..7d39799 100644 --- a/internal/source/k8s/k8s_test.go +++ b/internal/source/k8s/k8s_test.go @@ -12,18 +12,24 @@ import ( "log/slog" "math/big" "regexp" + "runtime" "sync" "testing" "time" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" "github.com/enix/x509-certificate-exporter/v4/pkg/cert" "github.com/enix/x509-certificate-exporter/v4/pkg/cert/pem" ) +// kruntimeObj is an alias used by the bulk-seeding helpers to keep the +// signatures readable without dragging the long type name into every line. +type kruntimeObj = kruntime.Object + func nopLogger() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) } type fakeSink struct { @@ -215,6 +221,199 @@ func TestSecretFilterExclude(t *testing.T) { // (no informer cache, no transform), so those tests were removed alongside // the transform functions. +func TestConfigMapsWatchHandlesDelete(t *testing.T) { + pemData := string(makeCertPEM(t)) + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "c1", Namespace: "ns"}, + Data: map[string]string{"ca.crt": pemData}, + } + client := fake.NewSimpleClientset(cm) + src := New(Options{ + Name: "k", Client: client, ResyncEvery: 10 * time.Minute, + ConfigMapRules: []SecretTypeRule{{KeyRe: regexp.MustCompile(`\.crt$`), Parser: pem.New()}}, + }, nopLogger()) + sink := &fakeSink{} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = src.Run(ctx, sink) }() + waitFor(t, func() bool { + sink.mu.Lock() + defer sink.mu.Unlock() + return len(sink.upsert) >= 1 + }, "initial configmap upsert") + + if err := client.CoreV1().ConfigMaps("ns").Delete(ctx, "c1", metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { + sink.mu.Lock() + defer sink.mu.Unlock() + return len(sink.delete) >= 1 + }, "configmap delete event") +} + +// TestSecretsListPagesDoesNotCacheData is a memory-regression smoke test for +// the direct LIST+WATCH path. It seeds N secrets each carrying a sizeable +// "garbage" Data key (mimicking large Helm release secrets) alongside the +// matched tls.crt key, runs the source until first sync, then forces GC and +// asserts the post-run heap stays well under "everything cached" territory. +// +// If a regression reintroduces a per-Secret cache (or fails to release the +// LIST page), the garbage bytes stay rooted on the source's heap and the +// assertion trips. +func TestSecretsListPagesDoesNotCacheData(t *testing.T) { + const ( + nSecrets = 200 + garbageSize = 50 * 1024 // 50 KiB per secret => 10 MiB total of garbage + // Budget catches the "source caches every Secret" failure mode. A + // healthy run holds parsed certs (~5 KiB each) + Prometheus series, + // totalling under 4 MiB for 200 entries; a fully-cached regression + // would push this past 10 MiB. + budgetBytes = 6 * 1024 * 1024 + ) + pemData := makeCertPEM(t) + objs := make([]kruntimeObj, 0, nSecrets) + for i := 0; i < nSecrets; i++ { + // Each secret carries a unique 50 KiB garbage payload that the + // configured rule does NOT match, so it should never be retained + // by anything in our pipeline. + garbage := make([]byte, garbageSize) + for j := range garbage { + garbage[j] = byte(i + j) + } + objs = append(objs, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tls-" + itoa(i), + Namespace: "ns", + }, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{"tls.crt": pemData, "garbage": garbage}, + }) + } + client := fake.NewSimpleClientset(objs...) + src := New(Options{ + Name: "k", Client: client, ResyncEvery: 10 * time.Minute, ListPageSize: 50, + SecretRules: []SecretTypeRule{{ + Type: "kubernetes.io/tls", KeyRe: regexp.MustCompile(`^tls\.crt$`), Parser: pem.New(), + }}, + }, nopLogger()) + sink := &fakeSink{} + ctx, cancel := context.WithCancel(context.Background()) + + done := make(chan struct{}) + go func() { _ = src.Run(ctx, sink); close(done) }() + waitFor(t, func() bool { + sink.mu.Lock() + defer sink.mu.Unlock() + return len(sink.upsert) >= nSecrets + }, "all secrets emitted") + + // Stop the source and wait for its goroutine to fully exit so the + // fake client (rooted via src.opts.Client) is no longer reachable. + cancel() + <-done + + // Drop every reference the test still owns to the seeded data, the + // client, and the source. After GC, what's left in the heap is + // whatever was retained by the sink (parsed bundles + a tiny tracked + // map) and Go runtime overhead — definitely no per-Secret garbage. + for i := range objs { + objs[i] = nil + } + objs = nil + client = nil + src = nil + runtime.GC() + runtime.GC() // second pass to clear finalizers triggered by the first + + var after runtime.MemStats + runtime.ReadMemStats(&after) + if after.HeapAlloc > budgetBytes { + t.Fatalf("post-sync heap: %d bytes (budget %d)\n"+ + "this is the smoke check that the LIST path does not cache "+ + "per-secret data: a regression here likely means the Source "+ + "is holding references to entire Secret objects.", + after.HeapAlloc, budgetBytes) + } + t.Logf("post-sync heap: %d KiB (budget %d KiB)", after.HeapAlloc>>10, budgetBytes>>10) +} + +// BenchmarkSecretsListPages measures the cost of one full initial sync +// against a fixed-size cluster fixture. Run with: +// +// go test -bench=BenchmarkSecretsListPages -benchmem ./internal/source/k8s/ +// +// allocs/op is the long-term regression watch — a sudden jump indicates +// either an extra per-secret allocation or accidental retention. +func BenchmarkSecretsListPages(b *testing.B) { + const nSecrets = 500 + pemData := makeCertPEMB(b) + objs := make([]kruntimeObj, 0, nSecrets) + for i := 0; i < nSecrets; i++ { + objs = append(objs, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "tls-" + itoa(i), Namespace: "ns"}, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{"tls.crt": pemData}, + }) + } + rules := []SecretTypeRule{{ + Type: "kubernetes.io/tls", KeyRe: regexp.MustCompile(`^tls\.crt$`), Parser: pem.New(), + }} + + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + client := fake.NewSimpleClientset(objs...) + src := New(Options{ + Name: "k", Client: client, ResyncEvery: 10 * time.Minute, ListPageSize: 50, + SecretRules: rules, + }, nopLogger()) + sink := &fakeSink{} + ctx, cancel := context.WithCancel(context.Background()) + go func() { _ = src.Run(ctx, sink) }() + for { + sink.mu.Lock() + n := len(sink.upsert) + sink.mu.Unlock() + if n >= nSecrets { + break + } + time.Sleep(time.Millisecond) + } + cancel() + } +} + +// itoa is a tiny base-10 formatter that avoids the strconv import dance for +// the benchmark and smoke test. +func itoa(i int) string { + if i == 0 { + return "0" + } + var buf [12]byte + pos := len(buf) + for i > 0 { + pos-- + buf[pos] = byte('0' + i%10) + i /= 10 + } + return string(buf[pos:]) +} + +// makeCertPEMB mirrors makeCertPEM but accepts a *testing.B instead of *testing.T. +func makeCertPEMB(b *testing.B) []byte { + b.Helper() + key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + tpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "leaf"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + } + der, _ := x509.CreateCertificate(rand.Reader, tpl, tpl, &key.PublicKey, key) + return encpem.EncodeToMemory(&encpem.Block{Type: "CERTIFICATE", Bytes: der}) +} + func TestRunNoClient(t *testing.T) { src := New(Options{Name: "k"}, nopLogger()) if err := src.Run(context.Background(), &fakeSink{}); err == nil { diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 17e66dd..33d0f5d 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -271,6 +271,33 @@ func TestExporterCoversAllScenarios(t *testing.T) { t.Fatalf("expected x509_source_bundles total > 0, got %v", total) } }) + + // memory_under_budget is a smoke check on the exporter's resident + // memory after the initial sync has completed. The dev cluster only + // holds a handful of TLS Secrets/ConfigMaps, so 80 MiB is generous — + // well above the typical ~15 MiB but well below what a regression + // to "cache every Secret" would push the pod to. The metric is + // emitted by Prometheus's process collector; if it disappears (e.g. + // build flag change) we just skip rather than fail. + t.Run("memory_under_budget", func(t *testing.T) { + fam := fams["process_resident_memory_bytes"] + if fam == nil || len(fam.GetMetric()) == 0 { + t.Skip("process_resident_memory_bytes not exposed") + } + const budget = 80 * 1024 * 1024 // 80 MiB + var maxRSS float64 + for _, m := range fam.GetMetric() { + if v := m.GetGauge().GetValue(); v > maxRSS { + maxRSS = v + } + } + if maxRSS > budget { + t.Fatalf("exporter RSS %.0f bytes (%.0f MiB) exceeds budget of %d MiB", + maxRSS, maxRSS/1024/1024, budget>>20) + } + t.Logf("max exporter RSS across scraped pods: %.1f MiB (budget %d MiB)", + maxRSS/1024/1024, budget>>20) + }) } func assertNoSeries(t *testing.T, fam *dto.MetricFamily, sc scenarios.Scenario) {