From a090a296fab450be55600ca58be8a2d6a480640d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20BIDON?= Date: Sun, 12 Mar 2023 18:57:22 +0100 Subject: [PATCH] refact(hostsensorutils): unexported fields that don't need to be exposed Also: * declared scanner resources as an enum type * replaced stdlib json, added uit tests for skipped resources * unexported worker pool * more unexported methods (i.e. everything that is not part of the interface) * refact(core): clarified mock injection logic and added a few unit tests at the caller's (CLI init utils) Signed-off-by: Frederic BIDON --- core/core/initutils.go | 34 ++++-- core/core/initutils_test.go | 71 +++++++++++ core/core/scan.go | 2 +- core/pkg/hostsensorutils/hostsensordeploy.go | 53 ++++---- .../hostsensorutils/hostsensordeploy_test.go | 37 +++--- .../hostsensorutils/hostsensorgetfrompod.go | 115 +++++++++--------- core/pkg/hostsensorutils/hostsensormock.go | 6 + .../hostsensorutils/hostsensorworkerpool.go | 6 +- core/pkg/hostsensorutils/json.go | 15 +++ core/pkg/hostsensorutils/utils.go | 75 +++++++----- core/pkg/hostsensorutils/utils_test.go | 68 +++++++++++ 11 files changed, 334 insertions(+), 148 deletions(-) create mode 100644 core/pkg/hostsensorutils/json.go create mode 100644 core/pkg/hostsensorutils/utils_test.go diff --git a/core/core/initutils.go b/core/core/initutils.go index 08a55b54..545e4c4e 100644 --- a/core/core/initutils.go +++ b/core/core/initutils.go @@ -101,26 +101,38 @@ func getResourceHandler(ctx context.Context, scanInfo *cautils.ScanInfo, tenantC return resourcehandler.NewK8sResourceHandler(k8s, getFieldSelector(scanInfo), hostSensorHandler, rbacObjects, registryAdaptors) } +// getHostSensorHandler yields a IHostSensor that knows how to collect a host's scanned resources. +// +// A noop sensor is returned whenever host scanning is disabled or an error prevented the scanner to properly deploy. func getHostSensorHandler(ctx context.Context, scanInfo *cautils.ScanInfo, k8s *k8sinterface.KubernetesApi) hostsensorutils.IHostSensor { - if !k8sinterface.IsConnectedToCluster() || k8s == nil { - return &hostsensorutils.HostSensorHandlerMock{} - } + const wantsHostSensorControls = true // defaults to disabling the scanner if not explictly enabled (TODO(fredbi): should be addressed by injecting ScanInfo defaults) + hostSensorVal := scanInfo.HostSensorEnabled.Get() - hasHostSensorControls := true - // we need to determined which controls needs host scanner - if scanInfo.HostSensorEnabled.Get() == nil && hasHostSensorControls { - scanInfo.HostSensorEnabled.SetBool(false) // default - do not run host scanner - } - if hostSensorVal := scanInfo.HostSensorEnabled.Get(); hostSensorVal != nil && *hostSensorVal { + switch { + case !k8sinterface.IsConnectedToCluster() || k8s == nil: // TODO(fred): fix race condition on global KSConfig there + return hostsensorutils.NewHostSensorHandlerMock() + + case hostSensorVal != nil && *hostSensorVal: hostSensorHandler, err := hostsensorutils.NewHostSensorHandler(k8s, scanInfo.HostSensorYamlPath) if err != nil { logger.L().Ctx(ctx).Warning(fmt.Sprintf("failed to create host scanner: %s", err.Error())) - return &hostsensorutils.HostSensorHandlerMock{} + + return hostsensorutils.NewHostSensorHandlerMock() } + return hostSensorHandler + + case hostSensorVal == nil && wantsHostSensorControls: + // TODO: we need to determine which controls need the host scanner + scanInfo.HostSensorEnabled.SetBool(false) + + fallthrough + + default: + return hostsensorutils.NewHostSensorHandlerMock() } - return &hostsensorutils.HostSensorHandlerMock{} } + func getFieldSelector(scanInfo *cautils.ScanInfo) resourcehandler.IFieldSelector { if scanInfo.IncludeNamespaces != "" { return resourcehandler.NewIncludeSelector(scanInfo.IncludeNamespaces) diff --git a/core/core/initutils_test.go b/core/core/initutils_test.go index 40f5454d..408a900c 100644 --- a/core/core/initutils_test.go +++ b/core/core/initutils_test.go @@ -7,8 +7,11 @@ import ( "github.com/kubescape/go-logger" "github.com/kubescape/go-logger/helpers" + "github.com/kubescape/k8s-interface/k8sinterface" "github.com/kubescape/kubescape/v2/core/cautils" + "github.com/kubescape/kubescape/v2/core/pkg/hostsensorutils" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test_getUIPrinter(t *testing.T) { @@ -105,3 +108,71 @@ func Test_getUIPrinter(t *testing.T) { }) } } + +func TestGetSensorHandler(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + t.Run("should return mock sensor if not k8s interface is provided", func(t *testing.T) { + t.Parallel() + + scanInfo := &cautils.ScanInfo{} + var k8s *k8sinterface.KubernetesApi + + sensor := getHostSensorHandler(ctx, scanInfo, k8s) + require.NotNil(t, sensor) + + _, isMock := sensor.(*hostsensorutils.HostSensorHandlerMock) + require.True(t, isMock) + }) + + t.Run("should return mock sensor if the sensor is not enabled", func(t *testing.T) { + t.Parallel() + + scanInfo := &cautils.ScanInfo{} + k8s := &k8sinterface.KubernetesApi{} + + sensor := getHostSensorHandler(ctx, scanInfo, k8s) + require.NotNil(t, sensor) + + _, isMock := sensor.(*hostsensorutils.HostSensorHandlerMock) + require.True(t, isMock) + }) + + t.Run("should return mock sensor if the sensor is disabled", func(t *testing.T) { + t.Parallel() + + falseFlag := cautils.NewBoolPtr(nil) + falseFlag.SetBool(false) + scanInfo := &cautils.ScanInfo{ + HostSensorEnabled: falseFlag, + } + k8s := &k8sinterface.KubernetesApi{} + + sensor := getHostSensorHandler(ctx, scanInfo, k8s) + require.NotNil(t, sensor) + + _, isMock := sensor.(*hostsensorutils.HostSensorHandlerMock) + require.True(t, isMock) + }) + + t.Run("should return mock sensor if the sensor is enabled, but can't deploy (nil)", func(t *testing.T) { + t.Parallel() + + falseFlag := cautils.NewBoolPtr(nil) + falseFlag.SetBool(true) + scanInfo := &cautils.ScanInfo{ + HostSensorEnabled: falseFlag, + } + var k8s *k8sinterface.KubernetesApi + + sensor := getHostSensorHandler(ctx, scanInfo, k8s) + require.NotNil(t, sensor) + + _, isMock := sensor.(*hostsensorutils.HostSensorHandlerMock) + require.True(t, isMock) + }) + + // TODO(fredbi): need to share the k8s client mock to test a happy path / deployment failure path +} diff --git a/core/core/scan.go b/core/core/scan.go index 1486123d..b893c709 100644 --- a/core/core/scan.go +++ b/core/core/scan.go @@ -74,7 +74,7 @@ func getInterfaces(ctx context.Context, scanInfo *cautils.ScanInfo) componentInt hostSensorHandler := getHostSensorHandler(ctx, scanInfo, k8s) if err := hostSensorHandler.Init(ctxHostScanner); err != nil { logger.L().Ctx(ctxHostScanner).Error("failed to init host scanner", helpers.Error(err)) - hostSensorHandler = &hostsensorutils.HostSensorHandlerMock{} + hostSensorHandler = hostsensorutils.NewHostSensorHandlerMock() } spanHostScanner.End() diff --git a/core/pkg/hostsensorutils/hostsensordeploy.go b/core/pkg/hostsensorutils/hostsensordeploy.go index 0132b50d..10de70d7 100644 --- a/core/pkg/hostsensorutils/hostsensordeploy.go +++ b/core/pkg/hostsensorutils/hostsensordeploy.go @@ -3,7 +3,6 @@ package hostsensorutils import ( "context" _ "embed" - "encoding/json" "fmt" "os" "sync" @@ -27,23 +26,23 @@ var ( namespaceWasPresent bool ) -const PortName string = "scanner" +const portName string = "scanner" // HostSensorHandler is a client that interacts with a host-scanner component deployed on nodes. // // The API exposed by the host sensor is defined here: https://github.com/kubescape/host-scanner type HostSensorHandler struct { - HostSensorPort int32 - HostSensorPodNames map[string]string //map from pod names to node names - HostSensorUnscheduledPodNames map[string]string //map from pod names to node names - IsReady <-chan bool //readonly chan + hostSensorPort int32 + hostSensorPodNames map[string]string //map from pod names to node names + hostSensorUnscheduledPodNames map[string]string //map from pod names to node names k8sObj *k8sinterface.KubernetesApi - DaemonSet *appsv1.DaemonSet + daemonSet *appsv1.DaemonSet podListLock sync.RWMutex gracePeriod int64 workerPool workerPool } +// NewHostSensorHandler builds a new http client to the host-scanner API. func NewHostSensorHandler(k8sObj *k8sinterface.KubernetesApi, hostSensorYAMLFile string) (*HostSensorHandler, error) { if k8sObj == nil { return nil, fmt.Errorf("nil k8s interface received") @@ -52,17 +51,17 @@ func NewHostSensorHandler(k8sObj *k8sinterface.KubernetesApi, hostSensorYAMLFile if hostSensorYAMLFile != "" { d, err := loadHostSensorFromFile(hostSensorYAMLFile) if err != nil { - return nil, fmt.Errorf("failed to load host-scan yaml file, reason: %s", err.Error()) + return nil, fmt.Errorf("failed to load host-scan yaml file, reason: %w", err) } hostSensorYAML = d } hsh := &HostSensorHandler{ k8sObj: k8sObj, - HostSensorPodNames: map[string]string{}, - HostSensorUnscheduledPodNames: map[string]string{}, + hostSensorPodNames: map[string]string{}, + hostSensorUnscheduledPodNames: map[string]string{}, gracePeriod: int64(15), - workerPool: NewWorkerPool(), + workerPool: newWorkerPool(), } // Don't deploy on a cluster with no nodes. Some cloud providers prevent the termination of K8s objects for cluster with no nodes!!! @@ -163,8 +162,8 @@ func (hsh *HostSensorHandler) applyYAML(ctx context.Context) error { } for j := range containers { for k := range containers[j].Ports { - if containers[j].Ports[k].Name == PortName { - hsh.HostSensorPort = containers[j].Ports[k].ContainerPort + if containers[j].Ports[k].Name == portName { + hsh.hostSensorPort = containers[j].Ports[k].ContainerPort } } } @@ -202,7 +201,7 @@ func (hsh *HostSensorHandler) applyYAML(ctx context.Context) error { } return fmt.Errorf("failed to Unmarshal YAML of DaemonSet, reason: %v", err) } - hsh.DaemonSet = &ds + hsh.daemonSet = &ds } } @@ -218,8 +217,8 @@ func (hsh *HostSensorHandler) checkPodForEachNode() error { } hsh.podListLock.RLock() - podsNum := len(hsh.HostSensorPodNames) - unschedPodNum := len(hsh.HostSensorUnscheduledPodNames) + podsNum := len(hsh.hostSensorPodNames) + unschedPodNum := len(hsh.hostSensorUnscheduledPodNames) hsh.podListLock.RUnlock() if len(nodesList.Items) <= podsNum+unschedPodNum { break @@ -227,7 +226,7 @@ func (hsh *HostSensorHandler) checkPodForEachNode() error { if time.Now().After(deadline) { hsh.podListLock.RLock() - podsMap := hsh.HostSensorPodNames + podsMap := hsh.hostSensorPodNames hsh.podListLock.RUnlock() return fmt.Errorf("host-sensor pods number (%d) differ than nodes number (%d) after deadline exceeded. Kubescape will take data only from the pods below: %v", podsNum, len(nodesList.Items), podsMap) @@ -243,9 +242,9 @@ func (hsh *HostSensorHandler) populatePodNamesToNodeNames(ctx context.Context) { go func() { var watchRes watch.Interface var err error - watchRes, err = hsh.k8sObj.KubernetesClient.CoreV1().Pods(hsh.DaemonSet.Namespace).Watch(hsh.k8sObj.Context, metav1.ListOptions{ + watchRes, err = hsh.k8sObj.KubernetesClient.CoreV1().Pods(hsh.daemonSet.Namespace).Watch(hsh.k8sObj.Context, metav1.ListOptions{ Watch: true, - LabelSelector: fmt.Sprintf("name=%s", hsh.DaemonSet.Spec.Template.Labels["name"]), + LabelSelector: fmt.Sprintf("name=%s", hsh.daemonSet.Spec.Template.Labels["name"]), }) if err != nil { logger.L().Ctx(ctx).Warning("failed to watch over DaemonSet pods - are we missing watch pods permissions?", helpers.Error(err)) @@ -273,8 +272,8 @@ func (hsh *HostSensorHandler) updatePodInListAtomic(ctx context.Context, eventTy case watch.Added, watch.Modified: if podObj.Status.Phase == corev1.PodRunning && len(podObj.Status.ContainerStatuses) > 0 && podObj.Status.ContainerStatuses[0].Ready { - hsh.HostSensorPodNames[podObj.ObjectMeta.Name] = podObj.Spec.NodeName - delete(hsh.HostSensorUnscheduledPodNames, podObj.ObjectMeta.Name) + hsh.hostSensorPodNames[podObj.ObjectMeta.Name] = podObj.Spec.NodeName + delete(hsh.hostSensorUnscheduledPodNames, podObj.ObjectMeta.Name) } else { if podObj.Status.Phase == corev1.PodPending && len(podObj.Status.Conditions) > 0 && podObj.Status.Conditions[0].Reason == corev1.PodReasonUnschedulable { @@ -291,14 +290,14 @@ func (hsh *HostSensorHandler) updatePodInListAtomic(ctx context.Context, eventTy helpers.String("nodeName", nodeName), helpers.String("podName", podObj.ObjectMeta.Name)) if nodeName != "" { - hsh.HostSensorUnscheduledPodNames[podObj.ObjectMeta.Name] = nodeName + hsh.hostSensorUnscheduledPodNames[podObj.ObjectMeta.Name] = nodeName } } else { - delete(hsh.HostSensorPodNames, podObj.ObjectMeta.Name) + delete(hsh.hostSensorPodNames, podObj.ObjectMeta.Name) } } default: - delete(hsh.HostSensorPodNames, podObj.ObjectMeta.Name) + delete(hsh.hostSensorPodNames, podObj.ObjectMeta.Name) } } @@ -318,7 +317,7 @@ func (hsh *HostSensorHandler) tearDownNamespace(namespace string) error { func (hsh *HostSensorHandler) TearDown() error { namespace := hsh.GetNamespace() // delete DaemonSet - if err := hsh.k8sObj.KubernetesClient.AppsV1().DaemonSets(hsh.GetNamespace()).Delete(hsh.k8sObj.Context, hsh.DaemonSet.Name, metav1.DeleteOptions{GracePeriodSeconds: &hsh.gracePeriod}); err != nil { + if err := hsh.k8sObj.KubernetesClient.AppsV1().DaemonSets(hsh.GetNamespace()).Delete(hsh.k8sObj.Context, hsh.daemonSet.Name, metav1.DeleteOptions{GracePeriodSeconds: &hsh.gracePeriod}); err != nil { return fmt.Errorf("failed to delete host-sensor daemonset: %v", err) } // delete Namespace @@ -331,10 +330,10 @@ func (hsh *HostSensorHandler) TearDown() error { } func (hsh *HostSensorHandler) GetNamespace() string { - if hsh.DaemonSet == nil { + if hsh.daemonSet == nil { return "" } - return hsh.DaemonSet.Namespace + return hsh.daemonSet.Namespace } func loadHostSensorFromFile(hostSensorYAMLFile string) (string, error) { diff --git a/core/pkg/hostsensorutils/hostsensordeploy_test.go b/core/pkg/hostsensorutils/hostsensordeploy_test.go index 5ed22527..0ba07cb6 100644 --- a/core/pkg/hostsensorutils/hostsensordeploy_test.go +++ b/core/pkg/hostsensorutils/hostsensordeploy_test.go @@ -26,11 +26,11 @@ func TestHostSensorHandler(t *testing.T) { t.Run("should initialize host sensor", func(t *testing.T) { require.NoError(t, h.Init(ctx)) - w, err := k8s.KubernetesClient.CoreV1().Pods(h.DaemonSet.Namespace).Watch(ctx, metav1.ListOptions{}) + w, err := k8s.KubernetesClient.CoreV1().Pods(h.daemonSet.Namespace).Watch(ctx, metav1.ListOptions{}) require.NoError(t, err) w.Stop() - require.Len(t, h.HostSensorPodNames, 2) + require.Len(t, h.hostSensorPodNames, 2) }) t.Run("should return namespace", func(t *testing.T) { @@ -46,10 +46,10 @@ func TestHostSensorHandler(t *testing.T) { foundControl, foundProvider := false, false for _, sensed := range envelope { - if sensed.Kind == ControlPlaneInfo { + if sensed.Kind == ControlPlaneInfo.String() { foundControl = true } - if sensed.Kind == CloudProviderInfo { + if sensed.Kind == CloudProviderInfo.String() { foundProvider = hasCloudProviderInfo([]hostsensor.HostSensorDataEnvelope{sensed}) } } @@ -68,23 +68,22 @@ func TestHostSensorHandler(t *testing.T) { t.Run("should initialize host sensor", func(t *testing.T) { require.NoError(t, h.Init(ctx)) - w, err := k8s.KubernetesClient.CoreV1().Pods(h.DaemonSet.Namespace).Watch(ctx, metav1.ListOptions{}) + w, err := k8s.KubernetesClient.CoreV1().Pods(h.daemonSet.Namespace).Watch(ctx, metav1.ListOptions{}) require.NoError(t, err) w.Stop() - require.Len(t, h.HostSensorPodNames, 2) + require.Len(t, h.hostSensorPodNames, 2) }) t.Run("should get version", func(t *testing.T) { - version, err := h.GetVersion() + version, err := h.getVersion() require.NoError(t, err) require.Equal(t, "v1.0.45", version) }) t.Run("ForwardToPod is a stub, not implemented", func(t *testing.T) { - // NOTE(fredbi): IMHO we should rather return some ErrNotImplemented sentinel error and make it explicit. - resp, err := h.ForwardToPod("pod1", "/version") - require.NoError(t, err) + resp, err := h.forwardToPod("pod1", "/version") + require.Contains(t, err.Error(), "not implemented") require.Nil(t, resp) }) @@ -97,10 +96,10 @@ func TestHostSensorHandler(t *testing.T) { foundControl, foundProvider := false, false for _, sensed := range envelope { - if sensed.Kind == ControlPlaneInfo { + if sensed.Kind == ControlPlaneInfo.String() { foundControl = true } - if sensed.Kind == CloudProviderInfo { + if sensed.Kind == CloudProviderInfo.String() { foundProvider = hasCloudProviderInfo([]hostsensor.HostSensorDataEnvelope{sensed}) } } @@ -126,17 +125,17 @@ func TestHostSensorHandler(t *testing.T) { t.Run("should initialize host sensor", func(t *testing.T) { require.NoError(t, h.Init(ctx)) - w, err := k8s.KubernetesClient.CoreV1().Pods(h.DaemonSet.Namespace).Watch(ctx, metav1.ListOptions{}) + w, err := k8s.KubernetesClient.CoreV1().Pods(h.daemonSet.Namespace).Watch(ctx, metav1.ListOptions{}) require.NoError(t, err) w.Stop() - require.Len(t, h.HostSensorPodNames, 2) + require.Len(t, h.hostSensorPodNames, 2) }) t.Run("should NOT be able to get version", func(t *testing.T) { // NOTE: GetVersion might be successful if only one pod responds successfully. // In order to ensure an error, we need ALL pods to error. - _, err := h.GetVersion() + _, err := h.getVersion() require.Error(t, err) require.Contains(t, err.Error(), "mock") }) @@ -157,11 +156,11 @@ func TestHostSensorHandler(t *testing.T) { t.Run("should initialize host sensor", func(t *testing.T) { require.NoError(t, h.Init(ctx)) - w, err := k8s.KubernetesClient.CoreV1().Pods(h.DaemonSet.Namespace).Watch(ctx, metav1.ListOptions{}) + w, err := k8s.KubernetesClient.CoreV1().Pods(h.daemonSet.Namespace).Watch(ctx, metav1.ListOptions{}) require.NoError(t, err) w.Stop() - require.Len(t, h.HostSensorPodNames, 2) + require.Len(t, h.hostSensorPodNames, 2) }) t.Run("should collect resources from pods, with some errors", func(t *testing.T) { @@ -197,11 +196,11 @@ func TestHostSensorHandler(t *testing.T) { t.Run("should initialize host sensor", func(t *testing.T) { require.NoError(t, h.Init(ctx)) - w, err := k8s.KubernetesClient.CoreV1().Pods(h.DaemonSet.Namespace).Watch(ctx, metav1.ListOptions{}) + w, err := k8s.KubernetesClient.CoreV1().Pods(h.daemonSet.Namespace).Watch(ctx, metav1.ListOptions{}) require.NoError(t, err) w.Stop() - require.Len(t, h.HostSensorPodNames, 2) + require.Len(t, h.hostSensorPodNames, 2) }) }) }) diff --git a/core/pkg/hostsensorutils/hostsensorgetfrompod.go b/core/pkg/hostsensorutils/hostsensorgetfrompod.go index 72a1d30c..5a90ba4b 100644 --- a/core/pkg/hostsensorutils/hostsensorgetfrompod.go +++ b/core/pkg/hostsensorutils/hostsensorgetfrompod.go @@ -2,7 +2,8 @@ package hostsensorutils import ( "context" - "encoding/json" + stdjson "encoding/json" + "errors" "fmt" "reflect" "strings" @@ -20,8 +21,8 @@ import ( // getPodList clones the internal list of pods being watched as a map of pod names. func (hsh *HostSensorHandler) getPodList() map[string]string { hsh.podListLock.RLock() - res := make(map[string]string, len(hsh.HostSensorPodNames)) - for k, v := range hsh.HostSensorPodNames { + res := make(map[string]string, len(hsh.hostSensorPodNames)) + for k, v := range hsh.hostSensorPodNames { res[k] = v } hsh.podListLock.RUnlock() @@ -29,31 +30,31 @@ func (hsh *HostSensorHandler) getPodList() map[string]string { return res } -// HTTPGetToPod send the request to a pod using the HostSensorPort. -func (hsh *HostSensorHandler) HTTPGetToPod(podName, path string) ([]byte, error) { - restProxy := hsh.k8sObj.KubernetesClient.CoreV1().Pods(hsh.DaemonSet.Namespace).ProxyGet("http", podName, fmt.Sprintf("%d", hsh.HostSensorPort), path, map[string]string{}) +// httpGetToPod sends the request to a pod using the HostSensorPort. +func (hsh *HostSensorHandler) httpGetToPod(podName, path string) ([]byte, error) { + restProxy := hsh.k8sObj.KubernetesClient.CoreV1().Pods(hsh.daemonSet.Namespace).ProxyGet("http", podName, fmt.Sprintf("%d", hsh.hostSensorPort), path, map[string]string{}) return restProxy.DoRaw(hsh.k8sObj.Context) } -func (hsh *HostSensorHandler) getResourcesFromPod(podName, nodeName, resourceKind, path string) (hostsensor.HostSensorDataEnvelope, error) { +func (hsh *HostSensorHandler) getResourcesFromPod(podName, nodeName string, resourceKind scannerResource, path string) (hostsensor.HostSensorDataEnvelope, error) { // send the request and pack the response as an hostSensorDataEnvelope - resBytes, err := hsh.HTTPGetToPod(podName, path) + resBytes, err := hsh.httpGetToPod(podName, path) if err != nil { return hostsensor.HostSensorDataEnvelope{}, err } hostSensorDataEnvelope := hostsensor.HostSensorDataEnvelope{} hostSensorDataEnvelope.SetApiVersion(k8sinterface.JoinGroupVersion(hostsensor.GroupHostSensor, hostsensor.Version)) - hostSensorDataEnvelope.SetKind(resourceKind) + hostSensorDataEnvelope.SetKind(resourceKind.String()) hostSensorDataEnvelope.SetName(nodeName) hostSensorDataEnvelope.SetData(resBytes) return hostSensorDataEnvelope, nil } -// ForwardToPod is not currently implemented. -func (hsh *HostSensorHandler) ForwardToPod(podName, path string) ([]byte, error) { +// forwardToPod is currently not implemented. +func (hsh *HostSensorHandler) forwardToPod(podName, path string) ([]byte, error) { // NOT IN USE: // --- // spawn port forwarding @@ -72,7 +73,7 @@ func (hsh *HostSensorHandler) ForwardToPod(podName, path string) ([]byte, error) // } // hostIP := strings.TrimLeft(req.RestConfig.Host, "htps:/") // dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, http.MethodPost, &url.URL{Scheme: "http", Path: path, Host: hostIP}) - return nil, nil + return nil, errors.New("not implemented") } // sendAllPodsHTTPGETRequest fills the raw bytes response in the envelope and the node name, but not the GroupVersionKind @@ -82,7 +83,7 @@ func (hsh *HostSensorHandler) ForwardToPod(podName, path string) ([]byte, error) // // For each node the request is pushed to the jobs channel, the worker sends the request and pushes the result to the result channel. // When all workers have finished, the function returns a list of results -func (hsh *HostSensorHandler) sendAllPodsHTTPGETRequest(ctx context.Context, path, requestKind string) ([]hostsensor.HostSensorDataEnvelope, error) { +func (hsh *HostSensorHandler) sendAllPodsHTTPGETRequest(ctx context.Context, path string, requestKind scannerResource) ([]hostsensor.HostSensorDataEnvelope, error) { podList := hsh.getPodList() res := make([]hostsensor.HostSensorDataEnvelope, 0, len(podList)) var wg sync.WaitGroup @@ -97,10 +98,10 @@ func (hsh *HostSensorHandler) sendAllPodsHTTPGETRequest(ctx context.Context, pat return res, nil } -// GetVersion returns the version of the deployed host scanner. +// getVersion returns the version of the deployed host scanner. // // NOTE: we pick the version from the first responding pod. -func (hsh *HostSensorHandler) GetVersion() (string, error) { +func (hsh *HostSensorHandler) getVersion() (string, error) { // loop over pods and port-forward it to each of them podList := hsh.getPodList() @@ -108,7 +109,7 @@ func (hsh *HostSensorHandler) GetVersion() (string, error) { hsh.workerPool.init(len(podList)) hsh.workerPool.hostSensorApplyJobs(podList, "/version", "version") for job := range hsh.workerPool.jobs { - resBytes, err := hsh.HTTPGetToPod(job.podName, job.path) + resBytes, err := hsh.httpGetToPod(job.podName, job.path) if err != nil { return "", err } else { @@ -122,50 +123,50 @@ func (hsh *HostSensorHandler) GetVersion() (string, error) { return "", nil } -// GetKernelVariables returns the list of Linux Kernel variables. -func (hsh *HostSensorHandler) GetKernelVariables(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { +// getKernelVariables returns the list of Linux Kernel variables. +func (hsh *HostSensorHandler) getKernelVariables(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { // loop over pods and port-forward it to each of them return hsh.sendAllPodsHTTPGETRequest(ctx, "/LinuxKernelVariables", LinuxKernelVariables) } -// GetOpenPortsList returns the list of open ports. -func (hsh *HostSensorHandler) GetOpenPortsList(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { +// getOpenPortsList returns the list of open ports. +func (hsh *HostSensorHandler) getOpenPortsList(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { // loop over pods and port-forward it to each of them return hsh.sendAllPodsHTTPGETRequest(ctx, "/openedPorts", OpenPortsList) } -// GetLinuxSecurityHardeningStatus returns the list of LinuxSecurityHardeningStatus metadata. -func (hsh *HostSensorHandler) GetLinuxSecurityHardeningStatus(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { +// getLinuxSecurityHardeningStatus returns the list of LinuxSecurityHardeningStatus metadata. +func (hsh *HostSensorHandler) getLinuxSecurityHardeningStatus(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { // loop over pods and port-forward it to each of them return hsh.sendAllPodsHTTPGETRequest(ctx, "/linuxSecurityHardening", LinuxSecurityHardeningStatus) } -// GetKubeletInfo returns the list of kubelet metadata. -func (hsh *HostSensorHandler) GetKubeletInfo(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { +// getKubeletInfo returns the list of kubelet metadata. +func (hsh *HostSensorHandler) getKubeletInfo(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { // loop over pods and port-forward it to each of them return hsh.sendAllPodsHTTPGETRequest(ctx, "/kubeletInfo", KubeletInfo) } -// GetKubeProxyInfo returns the list of kubeProxy metadata. -func (hsh *HostSensorHandler) GetKubeProxyInfo(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { +// getKubeProxyInfo returns the list of kubeProxy metadata. +func (hsh *HostSensorHandler) getKubeProxyInfo(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { // loop over pods and port-forward it to each of them return hsh.sendAllPodsHTTPGETRequest(ctx, "/kubeProxyInfo", KubeProxyInfo) } -// return list of controlPlaneInfo -func (hsh *HostSensorHandler) GetControlPlaneInfo(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { +// getControlPlanInfo returns the list of controlPlaneInfo metadata +func (hsh *HostSensorHandler) getControlPlaneInfo(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { // loop over pods and port-forward it to each of them return hsh.sendAllPodsHTTPGETRequest(ctx, "/controlPlaneInfo", ControlPlaneInfo) } -// GetCloudProviderInfo returns the list of cloudProviderInfo metadata. -func (hsh *HostSensorHandler) GetCloudProviderInfo(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { +// getCloudProviderInfo returns the list of cloudProviderInfo metadata. +func (hsh *HostSensorHandler) getCloudProviderInfo(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { // loop over pods and port-forward it to each of them return hsh.sendAllPodsHTTPGETRequest(ctx, "/cloudProviderInfo", CloudProviderInfo) } -// GetKubeletCommandLine returns the list of kubelet command lines. -func (hsh *HostSensorHandler) GetKubeletCommandLine(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { +// getKubeletCommandLine returns the list of kubelet command lines. +func (hsh *HostSensorHandler) getKubeletCommandLine(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { // loop over pods and port-forward it to each of them resps, err := hsh.sendAllPodsHTTPGETRequest(ctx, "/kubeletCommandLine", KubeletCommandLine) if err != nil { @@ -178,33 +179,33 @@ func (hsh *HostSensorHandler) GetKubeletCommandLine(ctx context.Context) ([]host resBytesMarshal, err := json.Marshal(data) // TODO catch error if err == nil { - resps[resp].Data = json.RawMessage(resBytesMarshal) + resps[resp].Data = stdjson.RawMessage(resBytesMarshal) } } return resps, nil } -// GetCNIInfo returns the list of CNI metadata -func (hsh *HostSensorHandler) GetCNIInfo(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { +// getCNIInfo returns the list of CNI metadata +func (hsh *HostSensorHandler) getCNIInfo(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { // loop over pods and port-forward it to each of them return hsh.sendAllPodsHTTPGETRequest(ctx, "/CNIInfo", CNIInfo) } -// GetKernelVersion returns the list of kernelVersion metadata. -func (hsh *HostSensorHandler) GetKernelVersion(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { +// getKernelVersion returns the list of kernelVersion metadata. +func (hsh *HostSensorHandler) getKernelVersion(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { // loop over pods and port-forward it to each of them return hsh.sendAllPodsHTTPGETRequest(ctx, "/kernelVersion", "KernelVersion") } -// GetOsReleaseFile returns the list of osRelease metadata. -func (hsh *HostSensorHandler) GetOsReleaseFile(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { +// getOsReleaseFile returns the list of osRelease metadata. +func (hsh *HostSensorHandler) getOsReleaseFile(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { // loop over pods and port-forward it to each of them return hsh.sendAllPodsHTTPGETRequest(ctx, "/osRelease", "OsReleaseFile") } -// GetKubeletConfigurations returns the list of kubelet configurations. -func (hsh *HostSensorHandler) GetKubeletConfigurations(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { +// getKubeletConfigurations returns the list of kubelet configurations. +func (hsh *HostSensorHandler) getKubeletConfigurations(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, error) { // loop over pods and port-forward it to each of them res, err := hsh.sendAllPodsHTTPGETRequest(ctx, "/kubeletConfigurations", "KubeletConfiguration") // empty kind, will be overridden for resIdx := range res { @@ -224,7 +225,7 @@ func (hsh *HostSensorHandler) GetKubeletConfigurations(ctx context.Context) ([]h // If information are found, then return true. Return false otherwise. func hasCloudProviderInfo(cpi []hostsensor.HostSensorDataEnvelope) bool { for index := range cpi { - if !reflect.DeepEqual(cpi[index].GetData(), json.RawMessage("{}\n")) { + if !reflect.DeepEqual(cpi[index].GetData(), stdjson.RawMessage("{}\n")) { return true } } @@ -236,12 +237,12 @@ func hasCloudProviderInfo(cpi []hostsensor.HostSensorDataEnvelope) bool { func (hsh *HostSensorHandler) CollectResources(ctx context.Context) ([]hostsensor.HostSensorDataEnvelope, map[string]apis.StatusInfo, error) { res := make([]hostsensor.HostSensorDataEnvelope, 0) infoMap := make(map[string]apis.StatusInfo) - if hsh.DaemonSet == nil { + if hsh.daemonSet == nil { return res, nil, nil } logger.L().Debug("Accessing host scanner") - version, err := hsh.GetVersion() + version, err := hsh.getVersion() if err != nil { logger.L().Ctx(ctx).Warning(err.Error()) } @@ -254,58 +255,58 @@ func (hsh *HostSensorHandler) CollectResources(ctx context.Context) ([]hostsenso var hasCloudProvider bool for _, toPin := range []struct { - Resource string + Resource scannerResource Query func(context.Context) ([]hostsensor.HostSensorDataEnvelope, error) }{ // queries to the deployed host-scanner { Resource: KubeletConfiguration, - Query: hsh.GetKubeletConfigurations, + Query: hsh.getKubeletConfigurations, }, { Resource: KubeletCommandLine, - Query: hsh.GetKubeletCommandLine, + Query: hsh.getKubeletCommandLine, }, { Resource: OsReleaseFile, - Query: hsh.GetOsReleaseFile, + Query: hsh.getOsReleaseFile, }, { Resource: KernelVersion, - Query: hsh.GetKernelVersion, + Query: hsh.getKernelVersion, }, { Resource: LinuxSecurityHardeningStatus, - Query: hsh.GetLinuxSecurityHardeningStatus, + Query: hsh.getLinuxSecurityHardeningStatus, }, { Resource: OpenPortsList, - Query: hsh.GetOpenPortsList, + Query: hsh.getOpenPortsList, }, { Resource: LinuxKernelVariables, - Query: hsh.GetKernelVariables, + Query: hsh.getKernelVariables, }, { Resource: KubeletInfo, - Query: hsh.GetKubeletInfo, + Query: hsh.getKubeletInfo, }, { Resource: KubeProxyInfo, - Query: hsh.GetKubeProxyInfo, + Query: hsh.getKubeProxyInfo, }, { Resource: CloudProviderInfo, - Query: hsh.GetCloudProviderInfo, + Query: hsh.getCloudProviderInfo, }, { Resource: CNIInfo, - Query: hsh.GetCNIInfo, + Query: hsh.getCNIInfo, }, { // ControlPlaneInfo is queried _after_ CloudProviderInfo. Resource: ControlPlaneInfo, - Query: hsh.GetControlPlaneInfo, + Query: hsh.getControlPlaneInfo, }, } { k8sInfo := toPin diff --git a/core/pkg/hostsensorutils/hostsensormock.go b/core/pkg/hostsensorutils/hostsensormock.go index 0261e844..6fb945bb 100644 --- a/core/pkg/hostsensorutils/hostsensormock.go +++ b/core/pkg/hostsensorutils/hostsensormock.go @@ -7,9 +7,15 @@ import ( "github.com/kubescape/opa-utils/reporthandling/apis" ) +// HostSensorHandlerMock is a noop sensor when the host scanner is disabled. type HostSensorHandlerMock struct { } +// NewHostSensorHandlerMock yields a dummy host sensor. +func NewHostSensorHandlerMock() *HostSensorHandlerMock { + return &HostSensorHandlerMock{} +} + func (hshm *HostSensorHandlerMock) Init(_ context.Context) error { return nil } diff --git a/core/pkg/hostsensorutils/hostsensorworkerpool.go b/core/pkg/hostsensorutils/hostsensorworkerpool.go index 2d231d69..13624f3a 100644 --- a/core/pkg/hostsensorutils/hostsensorworkerpool.go +++ b/core/pkg/hostsensorutils/hostsensorworkerpool.go @@ -14,7 +14,7 @@ const noOfWorkers int = 10 type job struct { podName string nodeName string - requestKind string + requestKind scannerResource path string } @@ -25,7 +25,7 @@ type workerPool struct { noOfWorkers int } -func NewWorkerPool() workerPool { +func newWorkerPool() workerPool { wp := workerPool{} wp.noOfWorkers = noOfWorkers wp.init() @@ -80,7 +80,7 @@ func (wp *workerPool) hostSensorGetResults(result *[]hostsensor.HostSensorDataEn }() } -func (wp *workerPool) hostSensorApplyJobs(podList map[string]string, path, requestKind string) { +func (wp *workerPool) hostSensorApplyJobs(podList map[string]string, path string, requestKind scannerResource) { go func() { for podName, nodeName := range podList { thisJob := job{ diff --git a/core/pkg/hostsensorutils/json.go b/core/pkg/hostsensorutils/json.go new file mode 100644 index 00000000..d35b4cab --- /dev/null +++ b/core/pkg/hostsensorutils/json.go @@ -0,0 +1,15 @@ +package hostsensorutils + +import ( + jsoniter "github.com/json-iterator/go" +) + +var ( + json jsoniter.API +) + +func init() { + // NOTE(fredbi): attention, this configuration rounds floats down to 6 digits + // For finer-grained config, see: https://pkg.go.dev/github.com/json-iterator/go#section-readme + json = jsoniter.ConfigFastest +} diff --git a/core/pkg/hostsensorutils/utils.go b/core/pkg/hostsensorutils/utils.go index a6862c06..64e763c8 100644 --- a/core/pkg/hostsensorutils/utils.go +++ b/core/pkg/hostsensorutils/utils.go @@ -5,39 +5,54 @@ import ( "github.com/kubescape/opa-utils/reporthandling/apis" ) -var ( - KubeletConfiguration = "KubeletConfiguration" - OsReleaseFile = "OsReleaseFile" - KernelVersion = "KernelVersion" - LinuxSecurityHardeningStatus = "LinuxSecurityHardeningStatus" - OpenPortsList = "OpenPortsList" - LinuxKernelVariables = "LinuxKernelVariables" - KubeletCommandLine = "KubeletCommandLine" - KubeletInfo = "KubeletInfo" - KubeProxyInfo = "KubeProxyInfo" - ControlPlaneInfo = "ControlPlaneInfo" - CloudProviderInfo = "CloudProviderInfo" - CNIInfo = "CNIInfo" +// scannerResource is the enumerated type listing all resources from the host-scanner. +type scannerResource string - MapHostSensorResourceToApiGroup = map[string]string{ - KubeletConfiguration: "hostdata.kubescape.cloud/v1beta0", - OsReleaseFile: "hostdata.kubescape.cloud/v1beta0", - KubeletCommandLine: "hostdata.kubescape.cloud/v1beta0", - KernelVersion: "hostdata.kubescape.cloud/v1beta0", - LinuxSecurityHardeningStatus: "hostdata.kubescape.cloud/v1beta0", - OpenPortsList: "hostdata.kubescape.cloud/v1beta0", - LinuxKernelVariables: "hostdata.kubescape.cloud/v1beta0", - KubeletInfo: "hostdata.kubescape.cloud/v1beta0", - KubeProxyInfo: "hostdata.kubescape.cloud/v1beta0", - ControlPlaneInfo: "hostdata.kubescape.cloud/v1beta0", - CloudProviderInfo: "hostdata.kubescape.cloud/v1beta0", - CNIInfo: "hostdata.kubescape.cloud/v1beta0", - } +const ( + // host-scanner resources + + KubeletConfiguration scannerResource = "KubeletConfiguration" + OsReleaseFile scannerResource = "OsReleaseFile" + KernelVersion scannerResource = "KernelVersion" + LinuxSecurityHardeningStatus scannerResource = "LinuxSecurityHardeningStatus" + OpenPortsList scannerResource = "OpenPortsList" + LinuxKernelVariables scannerResource = "LinuxKernelVariables" + KubeletCommandLine scannerResource = "KubeletCommandLine" + KubeletInfo scannerResource = "KubeletInfo" + KubeProxyInfo scannerResource = "KubeProxyInfo" + ControlPlaneInfo scannerResource = "ControlPlaneInfo" + CloudProviderInfo scannerResource = "CloudProviderInfo" + CNIInfo scannerResource = "CNIInfo" ) -func addInfoToMap(resource string, infoMap map[string]apis.StatusInfo, err error) { - group, version := k8sinterface.SplitApiVersion(MapHostSensorResourceToApiGroup[resource]) - r := k8sinterface.JoinResourceTriplets(group, version, resource) +func mapHostSensorResourceToApiGroup(r scannerResource) string { + switch r { + case + KubeletConfiguration, + OsReleaseFile, + KubeletCommandLine, + KernelVersion, + LinuxSecurityHardeningStatus, + OpenPortsList, + LinuxKernelVariables, + KubeletInfo, + KubeProxyInfo, + ControlPlaneInfo, + CloudProviderInfo, + CNIInfo: + return "hostdata.kubescape.cloud/v1beta0" + default: + return "" + } +} + +func (r scannerResource) String() string { + return string(r) +} + +func addInfoToMap(resource scannerResource, infoMap map[string]apis.StatusInfo, err error) { + group, version := k8sinterface.SplitApiVersion(mapHostSensorResourceToApiGroup(resource)) + r := k8sinterface.JoinResourceTriplets(group, version, resource.String()) infoMap[r] = apis.StatusInfo{ InnerStatus: apis.StatusSkipped, InnerInfo: err.Error(), diff --git a/core/pkg/hostsensorutils/utils_test.go b/core/pkg/hostsensorutils/utils_test.go new file mode 100644 index 00000000..95545837 --- /dev/null +++ b/core/pkg/hostsensorutils/utils_test.go @@ -0,0 +1,68 @@ +package hostsensorutils + +import ( + "errors" + "fmt" + "testing" + + "github.com/kubescape/opa-utils/reporthandling/apis" + "github.com/stretchr/testify/require" +) + +func TestAddInfoToMap(t *testing.T) { + t.Parallel() + + // NOTE: the function being tested is hard to test, because + // the worker pool mutes most errors. + // + // Essentially, unless we hit some extreme edge case, we never get an error to be added to the map. + testErr := errors.New("test error") + + for _, toPin := range []struct { + Resource scannerResource + Err error + Expected map[string]apis.StatusInfo + }{ + { + Resource: KubeletConfiguration, + Err: testErr, + Expected: map[string]apis.StatusInfo{ + "hostdata.kubescape.cloud/v1beta0/KubeletConfiguration": { + InnerStatus: apis.StatusSkipped, + InnerInfo: testErr.Error(), + }, + }, + }, + { + Resource: CNIInfo, + Err: testErr, + Expected: map[string]apis.StatusInfo{ + "hostdata.kubescape.cloud/v1beta0/CNIInfo": { + InnerStatus: apis.StatusSkipped, + InnerInfo: testErr.Error(), + }, + }, + }, + { + Resource: scannerResource("invalid"), + Err: testErr, + Expected: map[string]apis.StatusInfo{ + "//invalid": { // no group, no version + InnerStatus: apis.StatusSkipped, + InnerInfo: testErr.Error(), + }, + }, + }, + } { + tc := toPin + + t.Run(fmt.Sprintf("should expect a status for resource %s", tc.Resource), func(t *testing.T) { + t.Parallel() + + result := make(map[string]apis.StatusInfo, 1) + addInfoToMap(tc.Resource, result, tc.Err) + + require.EqualValues(t, tc.Expected, result) + }) + } +}