From 7d28dde8d87978cbb81ddb058bab869e398d1ef3 Mon Sep 17 00:00:00 2001 From: Xuewei Zhang Date: Wed, 20 Nov 2019 17:14:14 -0800 Subject: [PATCH] Add e2e test for OOM kill and Docker hung Also fixes two minor bugs: 1. Change default Boskos wait timeout to 2 minutes. This is because the current test timeout is configured to 10 minutes. Running each test case taks 1-2 minutes, and each node will run 1-2 test cases. 5 minutes timeout on waiting for Boskos may cause a test timeout, which we want to avoid. 2. Create artifact subdir with 0755 rather than 0644. Because execution bit should be set on the directories. --- test/e2e/metriconly/e2e_npd_test.go | 2 +- test/e2e/metriconly/metrics_test.go | 28 +++++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/test/e2e/metriconly/e2e_npd_test.go b/test/e2e/metriconly/e2e_npd_test.go index 6e6108f0..f422c5ab 100644 --- a/test/e2e/metriconly/e2e_npd_test.go +++ b/test/e2e/metriconly/e2e_npd_test.go @@ -49,7 +49,7 @@ var boskosProjectType = flag.String("boskos-project-type", "gce-project", "specifies which project type to select from Boskos.") var boskosServerURL = flag.String("boskos-server-url", "http://boskos.test-pods.svc.cluster.local", "specifies Boskos server URL.") -var boskosWaitDuration = flag.Duration("boskos-wait-duration", 5*time.Minute, +var boskosWaitDuration = flag.Duration("boskos-wait-duration", 2*time.Minute, "Duration to wait before quitting getting Boskos resource.") var computeService *compute.Service diff --git a/test/e2e/metriconly/metrics_test.go b/test/e2e/metriconly/metrics_test.go index ad3dbecf..348574ec 100644 --- a/test/e2e/metriconly/metrics_test.go +++ b/test/e2e/metriconly/metrics_test.go @@ -127,6 +127,32 @@ var _ = ginkgo.Describe("NPD should export Prometheus metrics.", func() { }) }) + ginkgo.Context("When OOM kills and docker hung happen", func() { + + ginkgo.BeforeEach(func() { + err := npd.WaitForNPD(instance, []string{"problem_gauge"}, 120) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Expect NPD to become ready in 120s, but hit error: %v", err)) + instance.RunCommandOrFail("sudo /home/kubernetes/bin/problem-maker --problem OOMKill") + instance.RunCommandOrFail("sudo /home/kubernetes/bin/problem-maker --problem DockerHung") + }) + + ginkgo.It("NPD should update problem_counter and problem_gauge", func() { + time.Sleep(5 * time.Second) + assertMetricValueInBound(instance, + "problem_counter", map[string]string{"reason": "DockerHung"}, + 1.0, 1.0) + assertMetricValueInBound(instance, + "problem_counter", map[string]string{"reason": "TaskHung"}, + 1.0, 1.0) + assertMetricValueInBound(instance, + "problem_gauge", map[string]string{"reason": "DockerHung", "type": "KernelDeadlock"}, + 1.0, 1.0) + assertMetricValueInBound(instance, + "problem_counter", map[string]string{"reason": "OOMKilling"}, + 1.0, 1.0) + }) + }) + ginkgo.AfterEach(func() { defer func() { err := instance.DeleteInstance() @@ -139,7 +165,7 @@ var _ = ginkgo.Describe("NPD should export Prometheus metrics.", func() { testSubdirName := strings.Replace(testText, " ", "_", -1) artifactSubDir = path.Join(*artifactsDir, testSubdirName) - err := os.MkdirAll(artifactSubDir, os.ModeDir|0644) + err := os.MkdirAll(artifactSubDir, os.ModeDir|0755) if err != nil { fmt.Printf("Failed to create sub-directory to hold test artiface for test %s at %s\n", testText, artifactSubDir)