diff --git a/cautils/scaninfo.go b/cautils/scaninfo.go index e65e0b56..989c4115 100644 --- a/cautils/scaninfo.go +++ b/cautils/scaninfo.go @@ -68,7 +68,6 @@ type ScanInfo struct { ScanAll bool // true if scan all frameworks } - type Getters struct { ExceptionsGetter getter.IExceptionsGetter ControlsInputsGetter getter.IControlsInputsGetter @@ -131,13 +130,6 @@ func (scanInfo *ScanInfo) GetScanningEnvironment() string { return ScanCluster } -func (scanInfo *ScanInfo) GetScanningEnvironment() string { - if len(scanInfo.InputPatterns) != 0 { - return ScanLocalFiles - } - return ScanCluster -} - func (scanInfo *ScanInfo) SetPolicyIdentifiers(policies []string, kind reporthandling.NotificationPolicyKind) { for _, policy := range policies { if !scanInfo.contains(policy) { diff --git a/clihandler/initcli.go b/clihandler/initcli.go index 47834d68..6b4ede54 100644 --- a/clihandler/initcli.go +++ b/clihandler/initcli.go @@ -18,6 +18,7 @@ import ( "github.com/armosec/kubescape/resultshandling/printer" "github.com/armosec/kubescape/resultshandling/reporter" "github.com/armosec/opa-utils/reporthandling" + "github.com/golang/glog" ) type componentInterfaces struct { @@ -41,10 +42,7 @@ func initHostSensor(scanInfo *cautils.ScanInfo, k8s *k8sinterface.KubernetesApi) glog.Errorf("failed to create host sensor: %v", err) return &hostsensorutils.HostSensorHandlerMock{} } - scanInfo.ExcludedNamespaces = fmt.Sprintf("%s,%s", scanInfo.ExcludedNamespaces, hostSensorHandler.DaemonSet.Namespace) return hostSensorHandler - } else { - fmt.Printf("Skipping nodes scanning\n") } return &hostsensorutils.HostSensorHandlerMock{} } @@ -54,6 +52,7 @@ func getInterfaces(scanInfo *cautils.ScanInfo) componentInterfaces { var hostSensorHandler hostsensorutils.IHostSensor var tenantConfig cautils.ITenantConfig + hostSensorHandler = &hostsensorutils.HostSensorHandlerMock{} // scanning environment scanningTarget := scanInfo.GetScanningEnvironment() switch scanningTarget { @@ -72,11 +71,11 @@ func getInterfaces(scanInfo *cautils.ScanInfo) componentInterfaces { resourceHandler = resourcehandler.NewK8sResourceHandler(k8s, getFieldSelector(scanInfo)) // use clusterConfig struct tenantConfig = cautils.NewClusterConfig(k8s, getter.GetArmoAPIConnector(), scanInfo.Account) - hostSensorHandler = initHostSensor(scanInfo, k8s) + hostSensorHandler = initHostSensor(scanInfo, k8s) } // reporting behavior - setup reporter reportHandler := getReporter(scanInfo, tenantConfig) - + v := cautils.NewIVersionCheckHandler() v.CheckLatestVersion(cautils.NewVersionCheckRequest(cautils.BuildNumber, policyIdentifierNames(scanInfo.PolicyIdentifier), "", scanningTarget)) @@ -107,13 +106,32 @@ func ScanCliSetup(scanInfo *cautils.ScanInfo) error { interfaces.report.SetClusterName(interfaces.tenantConfig.GetClusterName()) interfaces.report.SetCustomerGUID(interfaces.tenantConfig.GetCustomerGUID()) + if err := interfaces.hostSensorHandler.Init(); err != nil { + errMsg := "failed to init host sensor" + if scanInfo.VerboseMode { + errMsg = fmt.Sprintf("%s: %v", errMsg, err) + } + cautils.ErrorDisplay(errMsg) + } else if len(scanInfo.IncludeNamespaces) == 0 && interfaces.hostSensorHandler.GetNamespace() != "" { + scanInfo.ExcludedNamespaces = fmt.Sprintf("%s,%s", scanInfo.ExcludedNamespaces, interfaces.hostSensorHandler) + defer func() { + if err := interfaces.hostSensorHandler.TearDown(); err != nil { + errMsg := "failed to tear down host sensor" + if scanInfo.VerboseMode { + errMsg = fmt.Sprintf("%s: %v", errMsg, err) + } + cautils.ErrorDisplay(errMsg) + } + }() + } + // set policy getter only after setting the customerGUID setPolicyGetter(scanInfo, interfaces.tenantConfig.GetCustomerGUID()) // cli handler setup go func() { // policy handler setup - policyHandler := policyhandler.NewPolicyHandler(&processNotification, interfaces.resourceHandler) + policyHandler := policyhandler.NewPolicyHandler(&processNotification, interfaces.resourceHandler, interfaces.hostSensorHandler) if err := Scan(policyHandler, scanInfo); err != nil { fmt.Println(err) diff --git a/hostsensorutils/hostsensordataenvelope.go b/hostsensorutils/hostsensordataenvelope.go index 7c68f96b..de394157 100644 --- a/hostsensorutils/hostsensordataenvelope.go +++ b/hostsensorutils/hostsensordataenvelope.go @@ -8,7 +8,7 @@ import ( ) type HostSensorDataEnvelope struct { - schema.GroupVersionKind + schema.GroupVersionResource NodeName string `json:"nodeName"` Data json.RawMessage `json:"data"` } @@ -22,7 +22,7 @@ func (hsde *HostSensorDataEnvelope) SetName(val string) { } func (hsde *HostSensorDataEnvelope) SetKind(val string) { - hsde.Kind = val + hsde.Resource = val } @@ -43,7 +43,7 @@ func (hsde *HostSensorDataEnvelope) GetName() string { } func (hsde *HostSensorDataEnvelope) GetKind() string { - return hsde.Kind + return hsde.Resource } func (hsde *HostSensorDataEnvelope) GetApiVersion() string { diff --git a/hostsensorutils/hostsensordeploy.go b/hostsensorutils/hostsensordeploy.go index 74fbc678..6911da56 100644 --- a/hostsensorutils/hostsensordeploy.go +++ b/hostsensorutils/hostsensordeploy.go @@ -8,6 +8,7 @@ import ( "time" "github.com/armosec/k8s-interface/k8sinterface" + "github.com/armosec/kubescape/cautils" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -18,22 +19,16 @@ import ( ) type HostSensorHandler struct { - HostSensorNamespace string - HostSensorPort int32 - HostSensorDaemonSetName string - HostSensorPodNames map[string]string //map from pod names to node names - IsReady <-chan bool //readonly chan - k8sObj *k8sinterface.KubernetesApi - DaemonSet *appsv1.DaemonSet - podListLock sync.RWMutex + HostSensorPort int32 + HostSensorPodNames map[string]string //map from pod names to node names + IsReady <-chan bool //readonly chan + k8sObj *k8sinterface.KubernetesApi + DaemonSet *appsv1.DaemonSet + podListLock sync.RWMutex + gracePeriod int64 } func NewHostSensorHandler(k8sObj *k8sinterface.KubernetesApi) (*HostSensorHandler, error) { - // deploy the YAML - // store namespace + port - // store pod names - // make sure all pods are running, after X seconds treat has running anyway, and log an error on the pods not running yet - // return the object if k8sObj == nil { return nil, fmt.Errorf("nil k8s interface received") @@ -41,6 +36,7 @@ func NewHostSensorHandler(k8sObj *k8sinterface.KubernetesApi) (*HostSensorHandle hsh := &HostSensorHandler{ k8sObj: k8sObj, HostSensorPodNames: map[string]string{}, + gracePeriod: int64(15), } // Don't deploy on cluster with no nodes. Some cloud providers prevents termination of K8s objects for cluster with no nodes!!! if nodeList, err := k8sObj.KubernetesClient.NodeV1().RuntimeClasses().List(k8sObj.Context, metav1.ListOptions{}); err != nil || len(nodeList.Items) == 0 { @@ -54,6 +50,13 @@ func NewHostSensorHandler(k8sObj *k8sinterface.KubernetesApi) (*HostSensorHandle } func (hsh *HostSensorHandler) Init() error { + // deploy the YAML + // store namespace + port + // store pod names + // make sure all pods are running, after X seconds treat has running anyway, and log an error on the pods not running yet + cautils.ProgressTextDisplay("Installing host sensor") + cautils.StartSpinner() + defer cautils.StopSpinner() if err := hsh.applyYAML(); err != nil { return fmt.Errorf("in HostSensorHandler init failed to apply YAML: %v", err) } @@ -77,32 +80,41 @@ func (hsh *HostSensorHandler) applyYAML() error { if err := yaml.Unmarshal(singleYAMLBytes, namespaceAC); err != nil { return fmt.Errorf("failed to Unmarshal YAML of namespace: %v", err) } + namespaceName := "" if ns, err := hsh.k8sObj.KubernetesClient.CoreV1().Namespaces().Apply(hsh.k8sObj.Context, namespaceAC, metav1.ApplyOptions{ FieldManager: "kubescape", }); err != nil { return fmt.Errorf("failed to apply YAML of namespace: %v", err) } else { - hsh.HostSensorNamespace = ns.Name + namespaceName = ns.Name } - // apply deamonset - deamonAC := &appsapplyv1.DaemonSetApplyConfiguration{} + // apply DaemonSet + daemonAC := &appsapplyv1.DaemonSetApplyConfiguration{} singleYAMLBytes = make([]byte, 4096) if readLen, err := dec.Read(singleYAMLBytes); err != nil { - return fmt.Errorf("failed to read YAML of deamonset: %v", err) + if erra := hsh.tearDownNamesapce(namespaceName); erra != nil { + err = fmt.Errorf("%v; In addidtion %v", err, erra) + } + return fmt.Errorf("failed to read YAML of DaemonSet: %v", err) } else { singleYAMLBytes = singleYAMLBytes[:readLen] } - if err := yaml.Unmarshal(singleYAMLBytes, deamonAC); err != nil { - return fmt.Errorf("failed to Unmarshal YAML of deamonset: %v", err) + if err := yaml.Unmarshal(singleYAMLBytes, daemonAC); err != nil { + if erra := hsh.tearDownNamesapce(namespaceName); erra != nil { + err = fmt.Errorf("%v; In addidtion %v", err, erra) + } + return fmt.Errorf("failed to Unmarshal YAML of DaemonSet: %v", err) } - deamonAC.Namespace = &hsh.HostSensorNamespace - if ds, err := hsh.k8sObj.KubernetesClient.AppsV1().DaemonSets(hsh.HostSensorNamespace).Apply(hsh.k8sObj.Context, deamonAC, metav1.ApplyOptions{ + daemonAC.Namespace = &namespaceName + if ds, err := hsh.k8sObj.KubernetesClient.AppsV1().DaemonSets(namespaceName).Apply(hsh.k8sObj.Context, daemonAC, metav1.ApplyOptions{ FieldManager: "kubescape", }); err != nil { - return fmt.Errorf("failed to apply YAML of deamonset: %v", err) + if erra := hsh.tearDownNamesapce(namespaceName); erra != nil { + err = fmt.Errorf("%v; In addidtion %v", err, erra) + } + return fmt.Errorf("failed to apply YAML of DaemonSet: %v", err) } else { - hsh.HostSensorDaemonSetName = ds.Name hsh.HostSensorPort = ds.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort hsh.DaemonSet = ds } @@ -144,7 +156,6 @@ func (hsh *HostSensorHandler) populatePodNamesToNodeNames() { for eve := range watchRes.ResultChan() { pod, ok := eve.Object.(*corev1.Pod) if !ok { - fmt.Printf("Failed to watch over daemonset pods: not a Pod") continue } go hsh.updatePodInListAtomic(eve.Type, pod) @@ -168,17 +179,27 @@ func (hsh *HostSensorHandler) updatePodInListAtomic(eventType watch.EventType, p } } -func (hsh *HostSensorHandler) TearDown() error { - // remove the namespace - gracePeriod := int64(15) - if err := hsh.k8sObj.KubernetesClient.AppsV1().DaemonSets(hsh.HostSensorNamespace).Delete(hsh.k8sObj.Context, hsh.HostSensorDaemonSetName, metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod}); err != nil { - return fmt.Errorf("failed to delete host-sensor daemonset: %v", err) - } - if err := hsh.k8sObj.KubernetesClient.CoreV1().Namespaces().Delete(hsh.k8sObj.Context, hsh.HostSensorNamespace, - metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod}); err != nil { +func (hsh *HostSensorHandler) tearDownNamesapce(namespace string) error { + + if err := hsh.k8sObj.KubernetesClient.CoreV1().Namespaces().Delete(hsh.k8sObj.Context, namespace, metav1.DeleteOptions{GracePeriodSeconds: &hsh.gracePeriod}); err != nil { return fmt.Errorf("failed to delete host-sensor namespace: %v", err) } - // TODO: wait for termination + return nil +} + +func (hsh *HostSensorHandler) TearDown() error { + namespace := hsh.GetNamespace() + if err := hsh.k8sObj.KubernetesClient.AppsV1().DaemonSets(hsh.GetNamespace()).Delete(hsh.k8sObj.Context, hsh.DaemonSet.Name, metav1.DeleteOptions{GracePeriodSeconds: &hsh.gracePeriod}); err != nil { + return fmt.Errorf("failed to delete host-sensor daemonset: %v", err) + } + if err := hsh.tearDownNamesapce(namespace); err != nil { + return fmt.Errorf("failed to delete host-sensor daemonset: %v", err) + } + // TODO: wait for termination? may take up to 120 seconds!!! return nil } + +func (hsh *HostSensorHandler) GetNamespace() string { + return hsh.DaemonSet.Namespace +} diff --git a/hostsensorutils/hostsensorgetfrompod.go b/hostsensorutils/hostsensorgetfrompod.go index 11a36374..509eb747 100644 --- a/hostsensorutils/hostsensorgetfrompod.go +++ b/hostsensorutils/hostsensorgetfrompod.go @@ -5,7 +5,8 @@ import ( "fmt" "sync" - "k8s.io/apimachinery/pkg/runtime/schema" + "github.com/armosec/kubescape/cautils" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/yaml" ) @@ -93,20 +94,31 @@ func (hsh *HostSensorHandler) GetKubeletConfigurations() ([]HostSensorDataEnvelo continue } res[resIdx].Data = jsonBytes - kindDet := schema.GroupVersionKind{} + kindDet := metav1.TypeMeta{} if err = json.Unmarshal(jsonBytes, &kindDet); err != nil { fmt.Printf("In GetKubeletConfigurations failed to Unmarshal GroupVersionKind: %v;\n%v", err, jsonBytes) continue } - res[resIdx].GroupVersionKind = kindDet + res[resIdx].GroupVersionResource.Resource = kindDet.Kind + res[resIdx].GroupVersionResource.Group = kindDet.GroupVersionKind().Group + res[resIdx].GroupVersionResource.Version = kindDet.GroupVersionKind().Version } return res, err } func (hsh *HostSensorHandler) CollectResources() ([]HostSensorDataEnvelope, error) { + res := make([]HostSensorDataEnvelope, 0) + if hsh.DaemonSet == nil { + return res, nil + } + cautils.ProgressTextDisplay("Accessing host sensor") + cautils.StartSpinner() + defer cautils.StopSpinner() kcData, err := hsh.GetKubeletConfigurations() if err != nil { return kcData, err } - return kcData, nil + res = append(res, kcData...) + cautils.SuccessTextDisplay("Read host information from host sensor") + return res, nil } diff --git a/hostsensorutils/hostsensorinterface.go b/hostsensorutils/hostsensorinterface.go index 33f578d8..6b9e4170 100644 --- a/hostsensorutils/hostsensorinterface.go +++ b/hostsensorutils/hostsensorinterface.go @@ -4,4 +4,5 @@ type IHostSensor interface { Init() error TearDown() error CollectResources() ([]HostSensorDataEnvelope, error) + GetNamespace() string } diff --git a/hostsensorutils/hostsensormock.go b/hostsensorutils/hostsensormock.go index 6c790410..a8610a33 100644 --- a/hostsensorutils/hostsensormock.go +++ b/hostsensorutils/hostsensormock.go @@ -14,3 +14,7 @@ func (hshm *HostSensorHandlerMock) TearDown() error { func (hshm *HostSensorHandlerMock) CollectResources() ([]HostSensorDataEnvelope, error) { return []HostSensorDataEnvelope{}, nil } + +func (hshm *HostSensorHandlerMock) GetNamespace() string { + return "" +} diff --git a/policyhandler/handlenotification.go b/policyhandler/handlenotification.go index 8db8963e..6346af3d 100644 --- a/policyhandler/handlenotification.go +++ b/policyhandler/handlenotification.go @@ -3,24 +3,29 @@ package policyhandler import ( "fmt" + "github.com/armosec/k8s-interface/k8sinterface" + "github.com/armosec/k8s-interface/workloadinterface" "github.com/armosec/kubescape/cautils" + "github.com/armosec/kubescape/hostsensorutils" "github.com/armosec/kubescape/resourcehandler" "github.com/armosec/opa-utils/reporthandling" ) // PolicyHandler - type PolicyHandler struct { - resourceHandler resourcehandler.IResourceHandler + resourceHandler resourcehandler.IResourceHandler + hostSensorHandler hostsensorutils.IHostSensor // we are listening on this chan in opaprocessor/processorhandler.go/ProcessRulesListenner func processPolicy *chan *cautils.OPASessionObj getters *cautils.Getters } // CreatePolicyHandler Create ws-handler obj -func NewPolicyHandler(processPolicy *chan *cautils.OPASessionObj, resourceHandler resourcehandler.IResourceHandler) *PolicyHandler { +func NewPolicyHandler(processPolicy *chan *cautils.OPASessionObj, resourceHandler resourcehandler.IResourceHandler, hostSensorHandler hostsensorutils.IHostSensor) *PolicyHandler { return &PolicyHandler{ - resourceHandler: resourceHandler, - processPolicy: processPolicy, + resourceHandler: resourceHandler, + processPolicy: processPolicy, + hostSensorHandler: hostSensorHandler, } } @@ -43,7 +48,12 @@ func (policyHandler *PolicyHandler) HandleNotificationRequest(notification *repo return fmt.Errorf("empty list of resources") } opaSessionObj.K8SResources = k8sResources - + for i := range *k8sResources { + for resourceIdx := range (*k8sResources)[i] { + // TODO: add remove data function + opaSessionObj.AllResources[(*k8sResources)[i][resourceIdx].GetID()] = (*k8sResources)[i][resourceIdx] + } + } // update channel *policyHandler.processPolicy <- opaSessionObj return nil @@ -52,5 +62,25 @@ func (policyHandler *PolicyHandler) HandleNotificationRequest(notification *repo func (policyHandler *PolicyHandler) getResources(notification *reporthandling.PolicyNotification, opaSessionObj *cautils.OPASessionObj, scanInfo *cautils.ScanInfo) (*cautils.K8SResources, error) { opaSessionObj.PostureReport.ClusterAPIServerInfo = policyHandler.resourceHandler.GetClusterAPIServerInfo() - return policyHandler.resourceHandler.GetResources(opaSessionObj.Frameworks, ¬ification.Designators) + resourcesMap, err := policyHandler.resourceHandler.GetResources(opaSessionObj.Frameworks, ¬ification.Designators) + if err != nil { + return resourcesMap, err + } + hostResources, err := policyHandler.hostSensorHandler.CollectResources() + if err != nil { + return resourcesMap, err + } + for rscIdx := range hostResources { + groupResources := k8sinterface.ResourceGroupToString(hostResources[rscIdx].Group, hostResources[rscIdx].GetApiVersion(), hostResources[rscIdx].GetKind()) + for _, groupResource := range groupResources { + grpReasorceList, ok := (*resourcesMap)[groupResource] + if !ok { + grpReasorceList = make([]workloadinterface.IMetadata, 0) + } + grpReasorceList = append(grpReasorceList, &hostResources[rscIdx]) + (*resourcesMap)[groupResource] = grpReasorceList + } + } + cautils.SuccessTextDisplay("Let’s start!!!") + return resourcesMap, nil } diff --git a/resourcehandler/k8sresources.go b/resourcehandler/k8sresources.go index 149df06d..cd0a8c9f 100644 --- a/resourcehandler/k8sresources.go +++ b/resourcehandler/k8sresources.go @@ -47,7 +47,7 @@ func (k8sHandler *K8sResourceHandler) GetResources(frameworks []reporthandling.F return k8sResourcesMap, err } - cautils.SuccessTextDisplay("Accessed successfully to Kubernetes objects, let’s start!!!") + cautils.SuccessTextDisplay("Accessed successfully to Kubernetes objects") return k8sResourcesMap, nil }