Check kube-proxy health on linux

This commit is contained in:
Oleg Atamanenko
2021-06-08 09:57:00 -07:00
parent 70f79831de
commit c8629cea5d
4 changed files with 102 additions and 90 deletions

View File

@@ -17,6 +17,10 @@ limitations under the License.
package healthchecker
import (
"context"
"net/http"
"os/exec"
"strings"
"time"
"github.com/golang/glog"
@@ -110,3 +114,57 @@ func logPatternHealthCheck(service, logStartTime string, logPatternsToCheck map[
}
return true, nil
}
// healthCheckEndpointOKFunc returns a function to check the status of an http endpoint
func healthCheckEndpointOKFunc(endpoint string, timeout time.Duration) func() (bool, error) {
return func() (bool, error) {
httpClient := http.Client{Timeout: timeout}
response, err := httpClient.Get(endpoint)
if err != nil || response.StatusCode != http.StatusOK {
return false, nil
}
return true, nil
}
}
// getHealthCheckFunc returns the health check function based on the component.
func getHealthCheckFunc(hco *options.HealthCheckerOptions) func() (bool, error) {
switch hco.Component {
case types.KubeletComponent:
return healthCheckEndpointOKFunc(types.KubeletHealthCheckEndpoint, hco.HealthCheckTimeout)
case types.KubeProxyComponent:
return healthCheckEndpointOKFunc(types.KubeProxyHealthCheckEndpoint, hco.HealthCheckTimeout)
case types.DockerComponent:
return func() (bool, error) {
if _, err := execCommand(hco.HealthCheckTimeout, getDockerPath(), "ps"); err != nil {
return false, nil
}
return true, nil
}
case types.CRIComponent:
return func() (bool, error) {
if _, err := execCommand(hco.HealthCheckTimeout, hco.CriCtlPath, "--runtime-endpoint="+hco.CriSocketPath, "--image-endpoint="+hco.CriSocketPath, "pods"); err != nil {
return false, nil
}
return true, nil
}
default:
glog.Warningf("Unsupported component: %v", hco.Component)
}
return nil
}
// execCommand executes the bash command and returns the (output, error) from command, error if timeout occurs.
func execCommand(timeout time.Duration, command string, args ...string) (string, error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
cmd := exec.CommandContext(ctx, command, args...)
out, err := cmd.Output()
if err != nil {
glog.Infof("command %v failed: %v, %v\n", cmd, err, out)
return "", err
}
return strings.TrimSuffix(string(out), "\n"), nil
}

View File

@@ -17,10 +17,7 @@ limitations under the License.
package healthchecker
import (
"context"
"errors"
"net/http"
"os/exec"
"strconv"
"strings"
"time"
@@ -75,49 +72,6 @@ func getRepairFunc(hco *options.HealthCheckerOptions) func() {
}
}
// getHealthCheckFunc returns the health check function based on the component.
func getHealthCheckFunc(hco *options.HealthCheckerOptions) func() (bool, error) {
switch hco.Component {
case types.KubeletComponent:
return func() (bool, error) {
httpClient := http.Client{Timeout: hco.HealthCheckTimeout}
response, err := httpClient.Get(types.KubeletHealthCheckEndpoint)
if err != nil || response.StatusCode != http.StatusOK {
return false, nil
}
return true, nil
}
case types.DockerComponent:
return func() (bool, error) {
if _, err := execCommand(hco.HealthCheckTimeout, "docker", "ps"); err != nil {
return false, nil
}
return true, nil
}
case types.CRIComponent:
return func() (bool, error) {
if _, err := execCommand(hco.HealthCheckTimeout, hco.CriCtlPath, "--runtime-endpoint="+hco.CriSocketPath, "--image-endpoint="+hco.CriSocketPath, "pods"); err != nil {
return false, nil
}
return true, nil
}
}
return nil
}
// execCommand executes the bash command and returns the (output, error) from command, error if timeout occurs.
func execCommand(timeout time.Duration, command string, args ...string) (string, error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
cmd := exec.CommandContext(ctx, command, args...)
out, err := cmd.Output()
if err != nil {
glog.Infof("command %v failed: %v, %v\n", cmd, err, out)
return "", err
}
return strings.TrimSuffix(string(out), "\n"), nil
}
// checkForPattern returns (true, nil) if logPattern occurs less than logCountThreshold number of times since last
// service restart. (false, nil) otherwise.
func checkForPattern(service, logStartTime, logPattern string, logCountThreshold int) (bool, error) {
@@ -141,3 +95,7 @@ func checkForPattern(service, logStartTime, logPattern string, logCountThreshold
}
return true, nil
}
func getDockerPath() string {
return "docker"
}

View File

@@ -20,6 +20,7 @@ import (
"testing"
"time"
"k8s.io/node-problem-detector/cmd/healthchecker/options"
"k8s.io/node-problem-detector/pkg/healthchecker/types"
)
@@ -119,3 +120,38 @@ func TestHealthCheck(t *testing.T) {
})
}
}
func TestComponentsSupported(t *testing.T) {
for _, tc := range []struct {
description string
component string
}{
{
description: "Kube Proxy should be supported",
component: types.KubeProxyComponent,
},
{
description: "Kubelet should be supported",
component: types.KubeletComponent,
},
{
description: "Docker should be supported",
component: types.DockerComponent,
},
{
description: "CRI should be supported",
component: types.CRIComponent,
},
} {
t.Run(tc.description, func(t *testing.T) {
checkFunc := getHealthCheckFunc(&options.HealthCheckerOptions{
Component: tc.component,
})
if checkFunc == nil {
t.Errorf("component %v should be supported", tc.component)
}
})
}
}

View File

@@ -18,7 +18,6 @@ package healthchecker
import (
"fmt"
"net/http"
"os/exec"
"strconv"
"strings"
@@ -64,49 +63,6 @@ func getRepairFunc(hco *options.HealthCheckerOptions) func() {
}
}
// getHealthCheckFunc returns the health check function based on the component.
func getHealthCheckFunc(hco *options.HealthCheckerOptions) func() (bool, error) {
switch hco.Component {
case types.KubeletComponent:
return healthCheckEndpointOKFunc(types.KubeletHealthCheckEndpoint, hco.HealthCheckTimeout)
case types.KubeProxyComponent:
return healthCheckEndpointOKFunc(types.KubeProxyHealthCheckEndpoint, hco.HealthCheckTimeout)
case types.DockerComponent:
return func() (bool, error) {
if _, err := execCommand("docker.exe", "ps"); err != nil {
return false, nil
}
return true, nil
}
case types.CRIComponent:
return func() (bool, error) {
if _, err := execCommand(hco.CriCtlPath, "--runtime-endpoint="+hco.CriSocketPath, "--image-endpoint="+hco.CriSocketPath, "pods"); err != nil {
return false, nil
}
return true, nil
}
}
return nil
}
// healthCheckEndpointOKFunc returns a function to check the status of an http endpoint
func healthCheckEndpointOKFunc(endpoint string, timeout time.Duration) func() (bool, error) {
return func() (bool, error) {
httpClient := http.Client{Timeout: timeout}
response, err := httpClient.Get(endpoint)
if err != nil || response.StatusCode != http.StatusOK {
return false, nil
}
return true, nil
}
}
// execCommand creates a new process, executes the command, and returns the (output, error) from command.
func execCommand(command string, args ...string) (string, error) {
cmd := util.Exec(command, args...)
return extractCommandOutput(cmd)
}
// powershell executes the arguments in powershell process and returns (output, error) from command.
func powershell(args ...string) (string, error) {
cmd := util.Powershell(args...)
@@ -143,3 +99,7 @@ func checkForPattern(service, logStartTime, logPattern string, logCountThreshold
}
return true, nil
}
func getDockerPath() string {
return "docker.exe"
}