diff --git a/test/e2e/metriconly/e2e_npd_test.go b/test/e2e/metriconly/e2e_npd_test.go index 6e6108f0..f422c5ab 100644 --- a/test/e2e/metriconly/e2e_npd_test.go +++ b/test/e2e/metriconly/e2e_npd_test.go @@ -49,7 +49,7 @@ var boskosProjectType = flag.String("boskos-project-type", "gce-project", "specifies which project type to select from Boskos.") var boskosServerURL = flag.String("boskos-server-url", "http://boskos.test-pods.svc.cluster.local", "specifies Boskos server URL.") -var boskosWaitDuration = flag.Duration("boskos-wait-duration", 5*time.Minute, +var boskosWaitDuration = flag.Duration("boskos-wait-duration", 2*time.Minute, "Duration to wait before quitting getting Boskos resource.") var computeService *compute.Service diff --git a/test/e2e/metriconly/metrics_test.go b/test/e2e/metriconly/metrics_test.go index ad3dbecf..348574ec 100644 --- a/test/e2e/metriconly/metrics_test.go +++ b/test/e2e/metriconly/metrics_test.go @@ -127,6 +127,32 @@ var _ = ginkgo.Describe("NPD should export Prometheus metrics.", func() { }) }) + ginkgo.Context("When OOM kills and docker hung happen", func() { + + ginkgo.BeforeEach(func() { + err := npd.WaitForNPD(instance, []string{"problem_gauge"}, 120) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Expect NPD to become ready in 120s, but hit error: %v", err)) + instance.RunCommandOrFail("sudo /home/kubernetes/bin/problem-maker --problem OOMKill") + instance.RunCommandOrFail("sudo /home/kubernetes/bin/problem-maker --problem DockerHung") + }) + + ginkgo.It("NPD should update problem_counter and problem_gauge", func() { + time.Sleep(5 * time.Second) + assertMetricValueInBound(instance, + "problem_counter", map[string]string{"reason": "DockerHung"}, + 1.0, 1.0) + assertMetricValueInBound(instance, + "problem_counter", map[string]string{"reason": "TaskHung"}, + 1.0, 1.0) + assertMetricValueInBound(instance, + "problem_gauge", map[string]string{"reason": "DockerHung", "type": "KernelDeadlock"}, + 1.0, 1.0) + assertMetricValueInBound(instance, + "problem_counter", map[string]string{"reason": "OOMKilling"}, + 1.0, 1.0) + }) + }) + ginkgo.AfterEach(func() { defer func() { err := instance.DeleteInstance() @@ -139,7 +165,7 @@ var _ = ginkgo.Describe("NPD should export Prometheus metrics.", func() { testSubdirName := strings.Replace(testText, " ", "_", -1) artifactSubDir = path.Join(*artifactsDir, testSubdirName) - err := os.MkdirAll(artifactSubDir, os.ModeDir|0644) + err := os.MkdirAll(artifactSubDir, os.ModeDir|0755) if err != nil { fmt.Printf("Failed to create sub-directory to hold test artiface for test %s at %s\n", testText, artifactSubDir)