github/node-problem-detector
1
0
Fork 0
mirror of https://github.com/kubernetes/node-problem-detector.git synced 2026-08-19 04:06:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add e2e test for OOM kill and Docker hung

Browse Source 
Also fixes two minor bugs:

1. Change default Boskos wait timeout to 2 minutes.
This is because the current test timeout is configured to 10 minutes.
Running each test case taks 1-2 minutes, and each node will run 1-2 test
cases. 5 minutes timeout on waiting for Boskos may cause a test timeout,
which we want to avoid.

2. Create artifact subdir with 0755 rather than 0644.
Because execution bit should be set on the directories.
This commit is contained in:
Xuewei Zhang
2019-12-06 14:49:17 -08:00
parent 8b98d08b5f
commit 7d28dde8d8
2 changed files with 28 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/e2e/metriconly/e2e_npd_test.go
+1 -1
View File
@@ -49,7 +49,7 @@ var boskosProjectType = flag.String("boskos-project-type", "gce-project",
"specifies which project type to select from Boskos.")
var boskosServerURL = flag.String("boskos-server-url", "http://boskos.test-pods.svc.cluster.local",
"specifies Boskos server URL.")
var boskosWaitDuration = flag.Duration("boskos-wait-duration", 5*time.Minute,
var boskosWaitDuration = flag.Duration("boskos-wait-duration", 2*time.Minute,
"Duration to wait before quitting getting Boskos resource.")
var computeService *compute.Service
test/e2e/metriconly/metrics_test.go
+27 -1
View File
@@ -127,6 +127,32 @@ var _ = ginkgo.Describe("NPD should export Prometheus metrics.", func() {
})
})
ginkgo.Context("When OOM kills and docker hung happen", func() {
ginkgo.BeforeEach(func() {
err := npd.WaitForNPD(instance, []string{"problem_gauge"}, 120)
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Expect NPD to become ready in 120s, but hit error: %v", err))
instance.RunCommandOrFail("sudo /home/kubernetes/bin/problem-maker --problem OOMKill")
instance.RunCommandOrFail("sudo /home/kubernetes/bin/problem-maker --problem DockerHung")
})
ginkgo.It("NPD should update problem_counter and problem_gauge", func() {
time.Sleep(5 * time.Second)
assertMetricValueInBound(instance,
"problem_counter", map[string]string{"reason": "DockerHung"},
1.0, 1.0)
assertMetricValueInBound(instance,
"problem_counter", map[string]string{"reason": "TaskHung"},
1.0, 1.0)
assertMetricValueInBound(instance,
"problem_gauge", map[string]string{"reason": "DockerHung", "type": "KernelDeadlock"},
1.0, 1.0)
assertMetricValueInBound(instance,
"problem_counter", map[string]string{"reason": "OOMKilling"},
1.0, 1.0)
})
})
ginkgo.AfterEach(func() {
defer func() {
err := instance.DeleteInstance()
@@ -139,7 +165,7 @@ var _ = ginkgo.Describe("NPD should export Prometheus metrics.", func() {
testSubdirName := strings.Replace(testText, " ", "_", -1)
artifactSubDir = path.Join(*artifactsDir, testSubdirName)
err := os.MkdirAll(artifactSubDir, os.ModeDir|0644)
err := os.MkdirAll(artifactSubDir, os.ModeDir|0755)
if err != nil {
fmt.Printf("Failed to create sub-directory to hold test artiface for test %s at %s\n",
testText, artifactSubDir)