mirror of
https://github.com/kubereboot/kured.git
synced 2026-05-03 15:07:23 +00:00
168fe81bb4
We have historically decided to support AND TEST 3 versions of kubernetes. For that, we lagged behind in terms of client versions. Lagging behind one version allowed us, thanks to kubernetes client version skew, to support 3 versions at relatively low cost. However, maintaining 3 versions has always been a pain. For that, we pinned old versions of kubernetes clients/api and test tooling. In december 2025 community meeting, we decided to only maintain two versions: the most current one, and the previous one. This allows us to bump more regularly and be more secure. We can also more easily use dependabot this way. Signed-off-by: Jean-Philippe Evrard <open-source@a.spamming.party>
426 lines
12 KiB
Go
426 lines
12 KiB
Go
package kind
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"math/rand"
|
|
"os/exec"
|
|
"strconv"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
kuredDevImage string = "kured:dev"
|
|
)
|
|
|
|
// KindTest cluster deployed by each TestMain function, prepared to run a given test scenario.
|
|
type KindTest struct {
|
|
kindConfigPath string
|
|
clusterName string
|
|
timeout time.Duration
|
|
deployManifests []string
|
|
localImages []string
|
|
logsDir string
|
|
logBuffer bytes.Buffer
|
|
testInstance *testing.T // Maybe move this to testing.TB
|
|
}
|
|
|
|
func (k *KindTest) Write(p []byte) (n int, err error) {
|
|
k.testInstance.Helper()
|
|
k.logBuffer.Write(p)
|
|
return len(p), nil
|
|
}
|
|
|
|
func (k *KindTest) FlushLog() {
|
|
k.testInstance.Helper()
|
|
k.testInstance.Log(k.logBuffer.String())
|
|
k.logBuffer.Reset()
|
|
}
|
|
|
|
func (k *KindTest) RunCmd(cmdDetails ...string) error {
|
|
cmd := exec.Command(cmdDetails[0], cmdDetails[1:]...)
|
|
// by making KindTest a Writer, we can simply wire k to logs
|
|
// writing to k will write to proper logs.
|
|
cmd.Stdout = k
|
|
cmd.Stderr = k
|
|
|
|
err := cmd.Run()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Option that can be passed to the NewKind function in order to change the configuration
|
|
// of the test cluster
|
|
type Option func(k *KindTest)
|
|
|
|
// Deploy can be passed to NewKind to deploy extra components, in addition to the base deployment.
|
|
func Deploy(manifest string) Option {
|
|
return func(k *KindTest) {
|
|
k.deployManifests = append(k.deployManifests, manifest)
|
|
}
|
|
}
|
|
|
|
// ExportLogs can be passed to NewKind to specify the folder where the kubernetes logs will be exported after the tests.
|
|
func ExportLogs(folder string) Option {
|
|
return func(k *KindTest) {
|
|
k.logsDir = folder
|
|
}
|
|
}
|
|
|
|
// Timeout for long-running operations (e.g. deployments, readiness probes...)
|
|
func Timeout(t time.Duration) Option {
|
|
return func(k *KindTest) {
|
|
k.timeout = t
|
|
}
|
|
}
|
|
|
|
// LocalImage is passed to NewKind to allow loading a local Docker image into the cluster
|
|
func LocalImage(nameTag string) Option {
|
|
return func(k *KindTest) {
|
|
k.localImages = append(k.localImages, nameTag)
|
|
}
|
|
}
|
|
|
|
// NewKind creates a kind cluster given a name and set of Option instances.
|
|
func NewKindTester(kindClusterName string, filePath string, t *testing.T, options ...Option) *KindTest {
|
|
|
|
k := &KindTest{
|
|
clusterName: kindClusterName,
|
|
timeout: 10 * time.Minute,
|
|
kindConfigPath: filePath,
|
|
testInstance: t,
|
|
}
|
|
for _, option := range options {
|
|
option(k)
|
|
}
|
|
return k
|
|
}
|
|
|
|
// Prepare the kind cluster.
|
|
func (k *KindTest) Create() error {
|
|
err := k.RunCmd("kind", "create", "cluster", "--name", k.clusterName, "--config", k.kindConfigPath)
|
|
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create cluster: %v", err)
|
|
}
|
|
|
|
for _, img := range k.localImages {
|
|
if err := k.RunCmd("kind", "load", "docker-image", "--name", k.clusterName, img); err != nil {
|
|
return fmt.Errorf("failed to load image: %v", err)
|
|
}
|
|
}
|
|
for _, mf := range k.deployManifests {
|
|
kubectlContext := fmt.Sprintf("kind-%v", k.clusterName)
|
|
if err := k.RunCmd("kubectl", "--context", kubectlContext, "apply", "-f", mf); err != nil {
|
|
return fmt.Errorf("failed to deploy manifest: %v", err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (k *KindTest) Destroy() error {
|
|
if k.logsDir != "" {
|
|
if err := k.RunCmd("kind", "export", "logs", k.logsDir, "--name", k.clusterName); err != nil {
|
|
return fmt.Errorf("failed to export logs: %v. will not teardown", err)
|
|
}
|
|
}
|
|
|
|
if err := k.RunCmd("kind", "delete", "cluster", "--name", k.clusterName); err != nil {
|
|
return fmt.Errorf("failed to destroy cluster: %v", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func TestE2EWithCommand(t *testing.T) {
|
|
t.Parallel()
|
|
if testing.Short() {
|
|
t.Skip("skipping test in short mode.")
|
|
}
|
|
|
|
var kindClusterConfigs = []string{
|
|
"previous",
|
|
"current",
|
|
}
|
|
// Iterate over each Kubernetes version
|
|
for _, version := range kindClusterConfigs {
|
|
version := version
|
|
// Define a subtest for each combination
|
|
t.Run(version, func(t *testing.T) {
|
|
t.Parallel() // Allow tests to run in parallel
|
|
|
|
randomInt := strconv.Itoa(rand.Intn(100))
|
|
kindClusterName := fmt.Sprintf("kured-e2e-command-%v-%v", version, randomInt)
|
|
kindClusterConfigFile := fmt.Sprintf("../../.github/kind-cluster-%v.yaml", version)
|
|
kindContext := fmt.Sprintf("kind-%v", kindClusterName)
|
|
|
|
k := NewKindTester(kindClusterName, kindClusterConfigFile, t, LocalImage(kuredDevImage), Deploy("../../kured-rbac.yaml"), Deploy("testfiles/kured-ds.yaml"))
|
|
defer k.FlushLog()
|
|
|
|
err := k.Create()
|
|
if err != nil {
|
|
t.Fatalf("Error creating cluster %v", err)
|
|
}
|
|
defer func(k *KindTest) {
|
|
err := k.Destroy()
|
|
if err != nil {
|
|
t.Fatalf("Error destroying cluster %v", err)
|
|
}
|
|
}(k)
|
|
|
|
k.Write([]byte("Now running e2e tests"))
|
|
|
|
if err := k.RunCmd("bash", "testfiles/create-reboot-sentinels.sh", kindContext); err != nil {
|
|
t.Fatalf("failed to create sentinels: %v", err)
|
|
}
|
|
|
|
if err := k.RunCmd("bash", "testfiles/follow-coordinated-reboot.sh", kindContext); err != nil {
|
|
t.Fatalf("failed to follow reboot: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestE2EWithSignal(t *testing.T) {
|
|
t.Parallel()
|
|
if testing.Short() {
|
|
t.Skip("skipping test in short mode.")
|
|
}
|
|
|
|
var kindClusterConfigs = []string{
|
|
"previous",
|
|
"current",
|
|
}
|
|
// Iterate over each Kubernetes version
|
|
for _, version := range kindClusterConfigs {
|
|
version := version
|
|
// Define a subtest for each combination
|
|
t.Run(version, func(t *testing.T) {
|
|
t.Parallel() // Allow tests to run in parallel
|
|
|
|
randomInt := strconv.Itoa(rand.Intn(100))
|
|
kindClusterName := fmt.Sprintf("kured-e2e-signal-%v-%v", version, randomInt)
|
|
kindClusterConfigFile := fmt.Sprintf("../../.github/kind-cluster-%v.yaml", version)
|
|
kindContext := fmt.Sprintf("kind-%v", kindClusterName)
|
|
|
|
k := NewKindTester(kindClusterName, kindClusterConfigFile, t, LocalImage(kuredDevImage), Deploy("../../kured-rbac.yaml"), Deploy("testfiles/kured-ds-signal.yaml"))
|
|
defer k.FlushLog()
|
|
|
|
err := k.Create()
|
|
if err != nil {
|
|
t.Fatalf("Error creating cluster %v", err)
|
|
}
|
|
defer func(k *KindTest) {
|
|
err := k.Destroy()
|
|
if err != nil {
|
|
t.Fatalf("Error destroying cluster %v", err)
|
|
}
|
|
}(k)
|
|
|
|
k.Write([]byte("Now running e2e tests"))
|
|
|
|
if err := k.RunCmd("bash", "testfiles/create-reboot-sentinels.sh", kindContext); err != nil {
|
|
t.Fatalf("failed to create sentinels: %v", err)
|
|
}
|
|
|
|
if err := k.RunCmd("bash", "testfiles/follow-coordinated-reboot.sh", kindContext); err != nil {
|
|
t.Fatalf("failed to follow reboot: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestE2EConcurrentWithCommand(t *testing.T) {
|
|
t.Parallel()
|
|
if testing.Short() {
|
|
t.Skip("skipping test in short mode.")
|
|
}
|
|
|
|
var kindClusterConfigs = []string{
|
|
"previous",
|
|
"current",
|
|
}
|
|
// Iterate over each Kubernetes version
|
|
for _, version := range kindClusterConfigs {
|
|
version := version
|
|
// Define a subtest for each combination
|
|
t.Run(version, func(t *testing.T) {
|
|
t.Parallel() // Allow tests to run in parallel
|
|
|
|
randomInt := strconv.Itoa(rand.Intn(100))
|
|
kindClusterName := fmt.Sprintf("kured-e2e-concurrentcommand-%v-%v", version, randomInt)
|
|
kindClusterConfigFile := fmt.Sprintf("../../.github/kind-cluster-%v.yaml", version)
|
|
kindContext := fmt.Sprintf("kind-%v", kindClusterName)
|
|
|
|
k := NewKindTester(kindClusterName, kindClusterConfigFile, t, LocalImage(kuredDevImage), Deploy("../../kured-rbac.yaml"), Deploy("testfiles/kured-ds-concurrent-command.yaml"))
|
|
defer k.FlushLog()
|
|
|
|
err := k.Create()
|
|
if err != nil {
|
|
t.Fatalf("Error creating cluster %v", err)
|
|
}
|
|
defer func(k *KindTest) {
|
|
err := k.Destroy()
|
|
if err != nil {
|
|
t.Fatalf("Error destroying cluster %v", err)
|
|
}
|
|
}(k)
|
|
|
|
k.Write([]byte("Now running e2e tests"))
|
|
|
|
if err := k.RunCmd("bash", "testfiles/create-reboot-sentinels.sh", kindContext); err != nil {
|
|
t.Fatalf("failed to create sentinels: %v", err)
|
|
}
|
|
|
|
if err := k.RunCmd("bash", "testfiles/follow-coordinated-reboot.sh", kindContext); err != nil {
|
|
t.Fatalf("failed to follow reboot: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestE2EConcurrentWithSignal(t *testing.T) {
|
|
t.Parallel()
|
|
if testing.Short() {
|
|
t.Skip("skipping test in short mode.")
|
|
}
|
|
|
|
var kindClusterConfigs = []string{
|
|
"previous",
|
|
"current",
|
|
}
|
|
// Iterate over each Kubernetes version
|
|
for _, version := range kindClusterConfigs {
|
|
version := version
|
|
// Define a subtest for each combination
|
|
t.Run(version, func(t *testing.T) {
|
|
t.Parallel() // Allow tests to run in parallel
|
|
|
|
randomInt := strconv.Itoa(rand.Intn(100))
|
|
kindClusterName := fmt.Sprintf("kured-e2e-concurrentsignal-%v-%v", version, randomInt)
|
|
kindClusterConfigFile := fmt.Sprintf("../../.github/kind-cluster-%v.yaml", version)
|
|
kindContext := fmt.Sprintf("kind-%v", kindClusterName)
|
|
|
|
k := NewKindTester(kindClusterName, kindClusterConfigFile, t, LocalImage(kuredDevImage), Deploy("../../kured-rbac.yaml"), Deploy("testfiles/kured-ds-concurrent-signal.yaml"))
|
|
defer k.FlushLog()
|
|
|
|
err := k.Create()
|
|
if err != nil {
|
|
t.Fatalf("Error creating cluster %v", err)
|
|
}
|
|
defer func(k *KindTest) {
|
|
err := k.Destroy()
|
|
if err != nil {
|
|
t.Fatalf("Error destroying cluster %v", err)
|
|
}
|
|
}(k)
|
|
|
|
k.Write([]byte("Now running e2e tests"))
|
|
|
|
if err := k.RunCmd("bash", "testfiles/create-reboot-sentinels.sh", kindContext); err != nil {
|
|
t.Fatalf("failed to create sentinels: %v", err)
|
|
}
|
|
|
|
if err := k.RunCmd("bash", "testfiles/follow-coordinated-reboot.sh", kindContext); err != nil {
|
|
t.Fatalf("failed to follow reboot: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestCordonningIsKept(t *testing.T) {
|
|
t.Parallel()
|
|
if testing.Short() {
|
|
t.Skip("skipping test in short mode.")
|
|
}
|
|
|
|
var kindClusterConfigs = []string{
|
|
"concurrency1",
|
|
"concurrency2",
|
|
}
|
|
// Iterate over each test variant
|
|
for _, variant := range kindClusterConfigs {
|
|
variant := variant
|
|
// Define a subtest for each combination
|
|
t.Run(variant, func(t *testing.T) {
|
|
t.Parallel() // Allow tests to run in parallel
|
|
|
|
randomInt := strconv.Itoa(rand.Intn(100))
|
|
kindClusterName := fmt.Sprintf("kured-e2e-cordon-%v-%v", variant, randomInt)
|
|
kindClusterConfigFile := "../../.github/kind-cluster-current.yaml"
|
|
kindContext := fmt.Sprintf("kind-%v", kindClusterName)
|
|
|
|
var manifest string
|
|
if variant == "concurrency1" {
|
|
manifest = "testfiles/kured-ds-signal.yaml"
|
|
} else {
|
|
manifest = "testfiles/kured-ds-concurrent-signal.yaml"
|
|
}
|
|
k := NewKindTester(kindClusterName, kindClusterConfigFile, t, LocalImage(kuredDevImage), Deploy("../../kured-rbac.yaml"), Deploy(manifest))
|
|
defer k.FlushLog()
|
|
|
|
err := k.Create()
|
|
if err != nil {
|
|
t.Fatalf("Error creating cluster %v", err)
|
|
}
|
|
defer func(k *KindTest) {
|
|
err := k.Destroy()
|
|
if err != nil {
|
|
t.Fatalf("Error destroying cluster %v", err)
|
|
}
|
|
}(k)
|
|
|
|
k.Write([]byte("Now running e2e tests"))
|
|
|
|
if err := k.RunCmd("bash", "testfiles/node-stays-as-cordonned.sh", kindContext); err != nil {
|
|
t.Fatalf("node did not reboot in time: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
func TestE2EBlocker(t *testing.T) {
|
|
t.Parallel()
|
|
if testing.Short() {
|
|
t.Skip("skipping test in short mode.")
|
|
}
|
|
|
|
var kindClusterConfigs = []string{
|
|
"podblocker",
|
|
}
|
|
// Iterate over each variant of the test
|
|
for _, variant := range kindClusterConfigs {
|
|
variant := variant
|
|
// Define a subtest for each combination
|
|
t.Run(variant, func(t *testing.T) {
|
|
t.Parallel() // Allow tests to run in parallel
|
|
|
|
randomInt := strconv.Itoa(rand.Intn(100))
|
|
kindClusterName := fmt.Sprintf("kured-e2e-cordon-%v-%v", variant, randomInt)
|
|
kindClusterConfigFile := "../../.github/kind-cluster-current.yaml"
|
|
kindContext := fmt.Sprintf("kind-%v", kindClusterName)
|
|
|
|
k := NewKindTester(kindClusterName, kindClusterConfigFile, t, LocalImage(kuredDevImage), Deploy("../../kured-rbac.yaml"), Deploy(fmt.Sprintf("testfiles/kured-ds-%v.yaml", variant)))
|
|
defer k.FlushLog()
|
|
|
|
err := k.Create()
|
|
if err != nil {
|
|
t.Fatalf("Error creating cluster %v", err)
|
|
}
|
|
defer func(k *KindTest) {
|
|
err := k.Destroy()
|
|
if err != nil {
|
|
t.Fatalf("Error destroying cluster %v", err)
|
|
}
|
|
}(k)
|
|
|
|
k.Write([]byte("Now running e2e tests"))
|
|
|
|
if err := k.RunCmd("bash", fmt.Sprintf("testfiles/%v.sh", variant), kindContext); err != nil {
|
|
t.Fatalf("node blocker test did not succeed: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|