Collector and analyzer for sysctl parameters (#441)

Collector and analyzer for sysctl parameters

examples/troubleshoot/sysctl.yaml

@@ -0,0 +1,15 @@
+apiVersion: troubleshoot.sh/v1beta2
+kind: SupportBundle
+metadata:
+  name: sysctl
+spec:
+  collectors:
+  - sysctl:
+      image: debian:buster-slim
+  analyzers:
+  - sysctl:
+      checkName: IP forwarding enabled
+      outcomes:
+      - fail:
+          when: "net.ipv4.ip_forward = 0"
+          message: "IP forwarding is not enabled"

pkg/analyze/analyzer.go

@@ -358,5 +358,23 @@ func Analyze(analyzer *troubleshootv1beta2.Analyze, getFile getCollectedFileCont
 		return results, nil
 	}
 
+	if analyzer.Sysctl != nil {
+		isExcluded, err := isExcluded(analyzer.Sysctl.Exclude)
+		if err != nil {
+			return nil, err
+		}
+		if isExcluded {
+			return nil, nil
+		}
+		result, err := analyzeSysctl(analyzer.Sysctl, findFiles)
+		if err != nil {
+			return nil, err
+		}
+		if result == nil {
+			return []*AnalyzeResult{}, nil
+		}
+		return []*AnalyzeResult{result}, nil
+	}
+
 	return nil, errors.New("invalid analyzer")
 }

pkg/analyze/sysctl.go

@@ -0,0 +1,156 @@
+package analyzer
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/pkg/errors"
+	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+)
+
+// The when condition for outcomes in this analyzer is interpreted as "for some node".
+// For example, "when: net.ipv4.ip_forward = 0" is true if at least one node has IP forwarding
+// disabled.
+func analyzeSysctl(analyzer *troubleshootv1beta2.SysctlAnalyze, findFiles func(string) (map[string][]byte, error)) (*AnalyzeResult, error) {
+	files, err := findFiles("sysctl/*")
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to find collected sysctl parameters")
+	}
+	if len(files) == 0 {
+		return nil, errors.Wrap(err, "no sysctl parameters collected")
+	}
+
+	nodeParams := map[string]map[string]string{}
+
+	for filename, parameters := range files {
+		nodeName := filepath.Base(filename)
+		nodeParams[nodeName] = parseSysctlParameters(parameters)
+	}
+
+	for _, outcome := range analyzer.Outcomes {
+		result, err := evalSysctlOutcome(nodeParams, outcome)
+		if err != nil {
+			return nil, err
+		}
+		if result != nil {
+			result.Title = analyzer.CheckName
+			if result.Title == "" {
+				result.Title = "Sysctl"
+			}
+			return result, nil
+		}
+	}
+
+	return nil, nil
+}
+
+// Example: /proc/sys/net/ipv4/ip_forward = 1
+var sysctlParamRX = regexp.MustCompile(`(^[^\s]+)\s=\s(.+)`)
+
+func parseSysctlParameters(parameters []byte) map[string]string {
+	buffer := bytes.NewBuffer(parameters)
+	scanner := bufio.NewScanner(buffer)
+
+	parsed := map[string]string{}
+
+	for scanner.Scan() {
+		matches := sysctlParamRX.FindStringSubmatch(scanner.Text())
+		if len(matches) != 3 {
+			continue
+		}
+		key := matches[1]
+		value := matches[2]
+
+		// "/proc/sys/net/ipv4/ip_forward" => "net.ipv4.ip_forward"
+		key = strings.TrimPrefix(key, "/proc/sys/")
+		parts := strings.Split(key, "/")
+		key = strings.Join(parts, ".")
+
+		parsed[key] = value
+	}
+
+	return parsed
+}
+
+func evalSysctlOutcome(nodeParams map[string]map[string]string, outcome *troubleshootv1beta2.Outcome) (*AnalyzeResult, error) {
+	result := &AnalyzeResult{}
+
+	var singleOutcome *troubleshootv1beta2.SingleOutcome
+
+	if outcome.Pass != nil {
+		singleOutcome = outcome.Pass
+		result.IsPass = true
+	}
+	if outcome.Warn != nil {
+		singleOutcome = outcome.Warn
+		result.IsWarn = true
+	}
+	if outcome.Fail != nil {
+		singleOutcome = outcome.Fail
+		result.IsFail = true
+	}
+
+	if singleOutcome.When == "" {
+		result.Message = singleOutcome.Message
+		return result, nil
+	}
+
+	nodes, err := evalSysctlWhen(nodeParams, singleOutcome.When)
+	if err != nil {
+		return nil, err
+	}
+
+	// The when for this outcome is not true for any
+	if len(nodes) == 0 {
+		return nil, nil
+	}
+
+	// The when condition is true for at least one node
+	if len(nodes) == 1 {
+		result.Message = fmt.Sprintf("Node %s: %s", nodes[0], singleOutcome.Message)
+	} else {
+		result.Message = fmt.Sprintf("Nodes %s: %s", strings.Join(nodes, ", "), singleOutcome.Message)
+	}
+
+	result.URI = singleOutcome.URI
+
+	return result, nil
+}
+
+// Example: net.ipv4.ip_forward = 0
+var sysctlWhenRX = regexp.MustCompile(`([^\s]+)\s+(=+)\s+(.+)`)
+
+// Returns the list of node names the condition is true for. The condition is not considered true
+// if the parameter is missing for the node.
+func evalSysctlWhen(nodeParams map[string]map[string]string, when string) ([]string, error) {
+	matches := sysctlWhenRX.FindStringSubmatch(when)
+	if len(matches) != 4 {
+		return nil, fmt.Errorf("Failed to parse when %q", when)
+	}
+
+	switch matches[2] {
+	case "=", "==", "===":
+		var nodes []string
+
+		for nodeName, params := range nodeParams {
+			nodeValue, ok := params[matches[1]]
+			if !ok {
+				continue
+			}
+			if nodeValue == strings.TrimSpace(matches[3]) {
+				nodes = append(nodes, nodeName)
+			}
+		}
+
+		sort.Strings(nodes)
+
+		return nodes, nil
+	default:
+		return nil, fmt.Errorf("Unknown operator %q", matches[2])
+	}
+}

pkg/analyze/sysctl_test.go

@@ -0,0 +1,287 @@
+package analyzer
+
+import (
+	"testing"
+
+	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseSysctlParameters(t *testing.T) {
+	parameters := `
+/proc/sys/net/ipv4/ip_forward = 1
+/proc/sys/net/ipv4/ip_local_port_range = 32768 60999
+/proc/sys/net/bridge/bridge-nf-call-iptables = 0
+`
+	got := parseSysctlParameters([]byte(parameters))
+	expect := map[string]string{
+		"net.ipv4.ip_forward":                "1",
+		"net.ipv4.ip_local_port_range":       "32768 60999",
+		"net.bridge.bridge-nf-call-iptables": "0",
+	}
+
+	assert.Equal(t, expect, got)
+}
+
+func TestEvalSysctlWhen(t *testing.T) {
+	tests := []struct {
+		name       string
+		when       string
+		nodeParams map[string]map[string]string
+		expect     []string
+		expectErr  bool
+	}{
+		{
+			name: "One node with IP forwarding disabled",
+			when: "net.ipv4.ip_forward = 0",
+			nodeParams: map[string]map[string]string{
+				"node-a": {"net.ipv4.ip_forward": "0"},
+			},
+			expect:    []string{"node-a"},
+			expectErr: false,
+		},
+		{
+			name: "All nodes have IP forwarding disabled",
+			when: "net.ipv4.ip_forward = 0",
+			nodeParams: map[string]map[string]string{
+				"node-a": {"net.ipv4.ip_forward": "0"},
+				"node-b": {"net.ipv4.ip_forward": "0"},
+			},
+			expect:    []string{"node-a", "node-b"},
+			expectErr: false,
+		},
+		{
+			name: "No nodes have net.ipv4.ip_forward",
+			when: "net.ipv4.ip_forward = 0",
+			nodeParams: map[string]map[string]string{
+				"node-a": {},
+			},
+			expect:    []string{},
+			expectErr: false,
+		},
+		{
+			name: "One node has IP forwarding enabled, one disabled",
+			when: "net.ipv4.ip_forward = 0",
+			nodeParams: map[string]map[string]string{
+				"node-a": {"net.ipv4.ip_forward": "1"},
+				"node-b": {"net.ipv4.ip_forward": "0"},
+			},
+			expect:    []string{"node-b"},
+			expectErr: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := evalSysctlWhen(test.nodeParams, test.when)
+			if test.expectErr {
+				assert.Error(t, err)
+				return
+			}
+
+			assert.NoError(t, err)
+
+			assert.ElementsMatch(t, test.expect, got)
+		})
+	}
+}
+
+func TestAnalyzeSysctl(t *testing.T) {
+	var tests = []struct {
+		name     string
+		files    map[string][]byte
+		analyzer *troubleshootv1beta2.SysctlAnalyze
+		expect   *AnalyzeResult
+	}{
+		{
+			name: "Fail IP forwarding disabled on one node",
+			files: map[string][]byte{
+				"a": []byte(`
+/proc/sys/net/ipv4/ip_forward = 1
+`),
+				"b": []byte(`
+/proc/sys/net/ipv4/ip_forward = 0
+`),
+			},
+			analyzer: &troubleshootv1beta2.SysctlAnalyze{
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "net.ipv4.ip_forward = 0 ",
+							Message: "IP forwarding disabled",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							Message: "IP forwarding enabled",
+						},
+					},
+				},
+			},
+			expect: &AnalyzeResult{
+				Title:   "Sysctl",
+				IsFail:  true,
+				Message: "Node b: IP forwarding disabled",
+			},
+		},
+		{
+			name: "Fail IP forwarding disabled on all nodes",
+			files: map[string][]byte{
+				"a": []byte(`
+/proc/sys/net/ipv4/ip_forward = 0
+`),
+				"b": []byte(`
+/proc/sys/net/ipv4/ip_forward = 0
+`),
+			},
+			analyzer: &troubleshootv1beta2.SysctlAnalyze{
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "net.ipv4.ip_forward = 0",
+							Message: "IP forwarding disabled",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							Message: "IP forwarding enabled",
+						},
+					},
+				},
+			},
+			expect: &AnalyzeResult{
+				Title:   "Sysctl",
+				IsFail:  true,
+				Message: "Nodes a, b: IP forwarding disabled",
+			},
+		},
+		{
+			name: "Pass IP forwarding disabled on all nodes",
+			files: map[string][]byte{
+				"a": []byte(`
+/proc/sys/net/ipv4/ip_forward = 1
+`),
+				"b": []byte(`
+/proc/sys/net/ipv4/ip_forward = 1
+`),
+			},
+			analyzer: &troubleshootv1beta2.SysctlAnalyze{
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "net.ipv4.ip_forward = 0",
+							Message: "IP forwarding disabled",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							When:    "net.ipv4.ip_forward = 1",
+							Message: "IP forwarding enabled",
+						},
+					},
+				},
+			},
+			expect: &AnalyzeResult{
+				Title:   "Sysctl",
+				IsPass:  true,
+				Message: "Nodes a, b: IP forwarding enabled",
+			},
+		},
+		{
+			name: "Pass IP forwarding enabled on one node",
+			files: map[string][]byte{
+				"a": []byte{},
+				"b": []byte(`
+/proc/sys/net/ipv4/ip_forward = 1
+`),
+			},
+			analyzer: &troubleshootv1beta2.SysctlAnalyze{
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "net.ipv4.ip_forward = 0",
+							Message: "IP forwarding disabled",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							When:    "net.ipv4.ip_forward = 1",
+							Message: "IP forwarding enabled",
+						},
+					},
+				},
+			},
+			expect: &AnalyzeResult{
+				Title:   "Sysctl",
+				IsPass:  true,
+				Message: "Node b: IP forwarding enabled",
+			},
+		},
+		{
+			name: "Default warn with empty when",
+			files: map[string][]byte{
+				"a": []byte{},
+				"b": []byte{},
+			},
+			analyzer: &troubleshootv1beta2.SysctlAnalyze{
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "net.ipv4.ip_forward = 0",
+							Message: "IP forwarding disabled",
+						},
+					},
+					{
+						Pass: &troubleshootv1beta2.SingleOutcome{
+							When:    "net.ipv4.ip_forward = 1",
+							Message: "IP forwarding enabled",
+						},
+					},
+					{
+						Warn: &troubleshootv1beta2.SingleOutcome{
+							When:    "",
+							Message: "IP forwarding kernel parameters not found",
+						},
+					},
+				},
+			},
+			expect: &AnalyzeResult{
+				Title:   "Sysctl",
+				IsWarn:  true,
+				Message: "IP forwarding kernel parameters not found",
+			},
+		},
+		{
+			name: "No result",
+			files: map[string][]byte{
+				"a": []byte(`
+/proc/sys/net/ipv4/ip_forward = 1
+`),
+			},
+			analyzer: &troubleshootv1beta2.SysctlAnalyze{
+				Outcomes: []*troubleshootv1beta2.Outcome{
+					{
+						Fail: &troubleshootv1beta2.SingleOutcome{
+							When:    "net.ipv4.ip_forward = 0",
+							Message: "IP forwarding disabled",
+						},
+					},
+				},
+			},
+			expect: nil,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			var findFiles = func(glob string) (map[string][]byte, error) {
+				return test.files, nil
+			}
+			got, err := analyzeSysctl(test.analyzer, findFiles)
+
+			assert.NoError(t, err)
+
+			assert.Equal(t, test.expect, got)
+		})
+	}
+}

pkg/apis/troubleshoot/v1beta2/analyzer_shared.go

@@ -142,6 +142,11 @@ type RegistryImagesAnalyze struct {
 	CollectorName string     `json:"collectorName" yaml:"collectorName"`
 }
 
+type SysctlAnalyze struct {
+	AnalyzeMeta `json:",inline" yaml:",inline"`
+	Outcomes    []*Outcome `json:"outcomes" yaml:"outcomes"`
+}
+
 type AnalyzeMeta struct {
 	CheckName string                 `json:"checkName,omitempty" yaml:"checkName,omitempty"`
 	Exclude   multitype.BoolOrString `json:"exclude,omitempty" yaml:"exclude,omitempty"`
@@ -168,4 +173,5 @@ type Analyze struct {
 	Longhorn                 *LonghornAnalyze          `json:"longhorn,omitempty" yaml:"longhorn,omitempty"`
 	RegistryImages           *RegistryImagesAnalyze    `json:"registryImages,omitempty" yaml:"registryImages,omitempty"`
 	WeaveReport              *WeaveReportAnalyze       `json:"weaveReport,omitempty" yaml:"weaveReport,omitempty"`
+	Sysctl                   *SysctlAnalyze            `json:"sysctl,omitempty" yaml:"sysctl,omitempty"`
 }

pkg/apis/troubleshoot/v1beta2/collector_shared.go

@@ -115,6 +115,16 @@ type CopyFromHost struct {
 	ExtractArchive  bool              `json:"extractArchive,omitempty" yaml:"extractArchive,omitempty"`
 }
 
+type Sysctl struct {
+	CollectorMeta   `json:",inline" yaml:",inline"`
+	Name            string            `json:"name,omitempty" yaml:"name,omitempty"`
+	Namespace       string            `json:"namespace" yaml:"namespace"`
+	Image           string            `json:"image" yaml:"image"`
+	ImagePullPolicy string            `json:"imagePullPolicy,omitempty" yaml:"imagePullPolicy,omitempty"`
+	ImagePullSecret *ImagePullSecrets `json:"imagePullSecret,omitempty" yaml:"imagePullSecret,omitempty"`
+	Timeout         string            `json:"timeout,omitempty" yaml:"timeout,omitempty"`
+}
+
 type HTTP struct {
 	CollectorMeta `json:",inline" yaml:",inline"`
 	Name          string `json:"name,omitempty" yaml:"name,omitempty"`
@@ -196,6 +206,7 @@ type Collect struct {
 	Ceph             *Ceph             `json:"ceph,omitempty" yaml:"ceph,omitempty"`
 	Longhorn         *Longhorn         `json:"longhorn,omitempty" yaml:"longhorn,omitempty"`
 	RegistryImages   *RegistryImages   `json:"registryImages,omitempty" yaml:"registryImages,omitempty"`
+	Sysctl           *Sysctl           `json:"sysctl,omitempty" yaml:"sysctl,omitempty"`
 }
 
 func (c *Collect) AccessReviewSpecs(overrideNS string) []authorizationv1.SelfSubjectAccessReviewSpec {
@@ -387,6 +398,8 @@ func (c *Collect) AccessReviewSpecs(overrideNS string) []authorizationv1.SelfSub
 			},
 			NonResourceAttributes: nil,
 		})
+	} else if c.Sysctl != nil {
+		// TODO
 	}
 
 	return result
@@ -457,6 +470,10 @@ func (c *Collect) GetName() string {
 		collector = "registry-images"
 		name = c.RegistryImages.CollectorName
 	}
+	if c.Sysctl != nil {
+		collector = "sysctl"
+		name = c.Sysctl.Name
+	}
 
 	if collector == "" {
 		return "<none>"

pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go

@@ -152,6 +152,11 @@ func (in *Analyze) DeepCopyInto(out *Analyze) {
 		*out = new(WeaveReportAnalyze)
 		**out = **in
 	}
+	if in.Sysctl != nil {
+		in, out := &in.Sysctl, &out.Sysctl
+		*out = new(SysctlAnalyze)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Analyze.
@@ -647,6 +652,11 @@ func (in *Collect) DeepCopyInto(out *Collect) {
 		*out = new(RegistryImages)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.Sysctl != nil {
+		in, out := &in.Sysctl, &out.Sysctl
+		*out = new(Sysctl)
+		(*in).DeepCopyInto(*out)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Collect.
@@ -2763,6 +2773,54 @@ func (in *SupportBundleVersionSpec) DeepCopy() *SupportBundleVersionSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Sysctl) DeepCopyInto(out *Sysctl) {
+	*out = *in
+	out.CollectorMeta = in.CollectorMeta
+	if in.ImagePullSecret != nil {
+		in, out := &in.ImagePullSecret, &out.ImagePullSecret
+		*out = new(ImagePullSecrets)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Sysctl.
+func (in *Sysctl) DeepCopy() *Sysctl {
+	if in == nil {
+		return nil
+	}
+	out := new(Sysctl)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SysctlAnalyze) DeepCopyInto(out *SysctlAnalyze) {
+	*out = *in
+	out.AnalyzeMeta = in.AnalyzeMeta
+	if in.Outcomes != nil {
+		in, out := &in.Outcomes, &out.Outcomes
+		*out = make([]*Outcome, len(*in))
+		for i := range *in {
+			if (*in)[i] != nil {
+				in, out := &(*in)[i], &(*out)[i]
+				*out = new(Outcome)
+				(*in).DeepCopyInto(*out)
+			}
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SysctlAnalyze.
+func (in *SysctlAnalyze) DeepCopy() *SysctlAnalyze {
+	if in == nil {
+		return nil
+	}
+	out := new(SysctlAnalyze)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TCPConnect) DeepCopyInto(out *TCPConnect) {
 	*out = *in

pkg/collect/collector.go

@@ -182,7 +182,16 @@ func (c *Collector) IsExcluded() bool {
 		if isExcludedResult {
 			return true
 		}
+	} else if c.Collect.Sysctl != nil {
+		isExcludedResult, err := isExcluded(c.Collect.Sysctl.Exclude)
+		if err != nil {
+			return true
+		}
+		if isExcludedResult {
+			return true
+		}
 	}
+
 	return false
 }
 
@@ -251,6 +260,16 @@ func (c *Collector) RunCollectorSync(clientConfig *rest.Config, client kubernete
 		result, err = Longhorn(c, c.Collect.Longhorn)
 	} else if c.Collect.RegistryImages != nil {
 		result, err = Registry(c, c.Collect.RegistryImages)
+	} else if c.Collect.Sysctl != nil {
+		if c.Collect.Sysctl.Namespace == "" {
+			c.Collect.Sysctl.Namespace = c.Namespace
+		}
+		if c.Collect.Sysctl.Namespace == "" {
+			kubeconfig := k8sutil.GetKubeconfig()
+			namespace, _, _ := kubeconfig.Namespace()
+			c.Collect.Sysctl.Namespace = namespace
+		}
+		result, err = Sysctl(ctx, c, client, c.Collect.Sysctl)
 	} else {
 		err = errors.New("no spec found to run")
 		return

pkg/collect/run-pod.go

@@ -0,0 +1,144 @@
+package collect
+
+import (
+	"context"
+	"io/ioutil"
+	"sync"
+
+	"github.com/pkg/errors"
+	"github.com/replicatedhq/troubleshoot/pkg/k8sutil"
+	"github.com/replicatedhq/troubleshoot/pkg/logger"
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/client-go/kubernetes/typed/core/v1"
+
+	kuberneteserrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type RunPodOptions struct {
+	Image               string
+	ImagePullPolicy     string
+	Namespace           string
+	Command             []string
+	ImagePullSecretName string
+	HostNetwork         bool
+}
+
+func RunPodsReadyNodes(ctx context.Context, client v1.CoreV1Interface, opts RunPodOptions) (map[string][]byte, error) {
+	wg := sync.WaitGroup{}
+	mtx := sync.Mutex{}
+	nodeLogs := map[string][]byte{}
+
+	nodes, err := client.Nodes().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return nil, errors.Wrapf(err, "list nodes")
+	}
+
+	for _, node := range nodes.Items {
+		if !k8sutil.NodeIsReady(node) {
+			continue
+		}
+
+		wg.Add(1)
+
+		go func(node string) {
+			defer wg.Done()
+
+			pod := &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					GenerateName: "run-pod-",
+					Namespace:    opts.Namespace,
+				},
+				Spec: corev1.PodSpec{
+					NodeSelector: map[string]string{
+						"kubernetes.io/hostname": node,
+					},
+					RestartPolicy: corev1.RestartPolicyNever,
+					HostNetwork:   opts.HostNetwork,
+					Containers: []corev1.Container{
+						{
+							Name:            "run",
+							Image:           opts.Image,
+							ImagePullPolicy: corev1.PullPolicy(opts.ImagePullPolicy),
+							Command:         opts.Command,
+						},
+					},
+					Tolerations: []corev1.Toleration{
+						{
+							Key:      "node-role.kubernetes.io/master",
+							Operator: "Exists",
+							Effect:   "NoSchedule",
+						},
+					},
+				},
+			}
+			if opts.ImagePullSecretName != "" {
+				pod.Spec.ImagePullSecrets = append(pod.Spec.ImagePullSecrets, corev1.LocalObjectReference{Name: opts.ImagePullSecretName})
+			}
+			logs, err := RunPodLogs(ctx, client, pod)
+			if err != nil {
+				logger.Printf("Failed to run pod on node %s: %v", node, err)
+				return
+			}
+
+			mtx.Lock()
+			defer mtx.Unlock()
+			nodeLogs[node] = logs
+		}(node.Name)
+	}
+
+	wg.Wait()
+
+	return nodeLogs, nil
+}
+
+// RunPodLogs runs a pod to completion on a node and returns its logs
+func RunPodLogs(ctx context.Context, client v1.CoreV1Interface, pod *corev1.Pod) ([]byte, error) {
+	// 1. Create
+	pod, err := client.Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{})
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create pod")
+	}
+	defer func() {
+		go func() {
+			// use context.background for the after-completion cleanup, as the parent context might already be over
+			err := client.Pods(pod.Namespace).Delete(context.Background(), pod.Name, metav1.DeleteOptions{})
+			if err != nil && !kuberneteserrors.IsNotFound(err) {
+				logger.Printf("Failed to delete pod %s: %v\n", pod.Name, err)
+			}
+		}()
+	}()
+
+	// 2. Wait
+	for {
+		pod, err := client.Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to get pod")
+		}
+
+		if pod.Status.Phase == corev1.PodFailed || pod.Status.Phase == corev1.PodSucceeded {
+			break
+		}
+
+		if pod.Status.Phase == corev1.PodPending {
+			for _, v := range pod.Status.ContainerStatuses {
+				if v.State.Waiting != nil && v.State.Waiting.Reason == "ImagePullBackOff" {
+					return nil, errors.New("wait for pod aborted after getting pod status 'ImagePullBackOff'")
+				}
+			}
+		}
+	}
+
+	// 3. Logs
+	podLogOpts := corev1.PodLogOptions{
+		Container: pod.Spec.Containers[0].Name,
+	}
+	req := client.Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts)
+	logs, err := req.Stream(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get log stream")
+	}
+	defer logs.Close()
+
+	return ioutil.ReadAll(logs)
+}

pkg/collect/sysctl.go

@@ -0,0 +1,76 @@
+package collect
+
+import (
+	"bytes"
+	"context"
+	"path/filepath"
+	"time"
+
+	"github.com/pkg/errors"
+	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/replicatedhq/troubleshoot/pkg/logger"
+	kuberneteserrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+func Sysctl(ctx context.Context, c *Collector, client kubernetes.Interface, collector *troubleshootv1beta2.Sysctl) (CollectorResult, error) {
+
+	if collector.Timeout != "" {
+		timeout, err := time.ParseDuration(collector.Timeout)
+		if err != nil {
+			return nil, errors.Wrap(err, "parse timeout")
+		}
+		if timeout == 0 {
+			timeout = time.Minute
+		}
+		childCtx, cancel := context.WithTimeout(ctx, timeout)
+		defer cancel()
+		ctx = childCtx
+	}
+
+	runPodOptions := RunPodOptions{
+		Image:           collector.Image,
+		ImagePullPolicy: collector.ImagePullPolicy,
+		Namespace:       collector.Namespace,
+		HostNetwork:     true,
+	}
+
+	command := `
+find /proc/sys/net/ipv4 -type f | while read f; do v=$(cat $f 2>/dev/null); echo "$f = $v"; done
+find /proc/sys/net/bridge -type f | while read f; do v=$(cat $f 2>/dev/null); echo "$f = $v"; done
+`
+	runPodOptions.Command = []string{"sh", "-c", command}
+
+	if collector.ImagePullSecret != nil {
+		runPodOptions.ImagePullSecretName = collector.ImagePullSecret.Name
+
+		if collector.ImagePullSecret.Data != nil {
+			secretName, err := createSecret(ctx, client, collector.Namespace, collector.ImagePullSecret)
+			if err != nil {
+				return nil, errors.Wrap(err, "create image pull secret")
+			}
+			defer func() {
+				err := client.CoreV1().Secrets(collector.Namespace).Delete(ctx, collector.ImagePullSecret.Name, metav1.DeleteOptions{})
+				if err != nil && !kuberneteserrors.IsNotFound(err) {
+					logger.Printf("Failed to delete secret %s: %v", collector.ImagePullSecret.Name, err)
+				}
+			}()
+
+			runPodOptions.ImagePullSecretName = secretName
+		}
+	}
+
+	results, err := RunPodsReadyNodes(ctx, client.CoreV1(), runPodOptions)
+	if err != nil {
+		return nil, err
+	}
+
+	output := NewResult()
+
+	for k, v := range results {
+		output.SaveResult(c.BundlePath, filepath.Join("sysctl", k), bytes.NewBuffer(v))
+	}
+
+	return output, nil
+}

pkg/k8sutil/node.go

@@ -0,0 +1,21 @@
+package k8sutil
+
+import v1 "k8s.io/api/core/v1"
+
+const UnreachableTaint = "node.kubernetes.io/unreachable"
+const NotReadyTaint = "node.kubernetes.io/not-ready"
+const UnschedulableTaint = "node.kubernetes.io/unschedulable"
+
+func NodeIsReady(node v1.Node) bool {
+	for _, taint := range node.Spec.Taints {
+		switch taint.Key {
+		case NotReadyTaint:
+			return false
+		case UnreachableTaint:
+			return false
+		case UnschedulableTaint:
+			return false
+		}
+	}
+	return true
+}