From 1aeb2b96e25fff4d2567099dc2be1046c3faae62 Mon Sep 17 00:00:00 2001 From: dwertent Date: Tue, 7 Dec 2021 10:30:03 +0200 Subject: [PATCH] store data only once --- cautils/datastructures.go | 4 +- clihandler/cmd/framework.go | 27 ++++++++- opaprocessor/processorhandler.go | 50 +++++++++-------- opaprocessor/processorhandler_test.go | 8 ++- opaprocessor/processorhandlerutils.go | 55 ++++--------------- policyhandler/handlenotification.go | 44 ++++++++------- resourcehandler/filesloader.go | 31 +++++++---- resourcehandler/k8sresources.go | 23 +++++--- resourcehandler/resourceshandler.go | 3 +- .../reporter/reporteventreceiver.go | 4 -- 10 files changed, 136 insertions(+), 113 deletions(-) diff --git a/cautils/datastructures.go b/cautils/datastructures.go index 2f9c2b71..918edc4a 100644 --- a/cautils/datastructures.go +++ b/cautils/datastructures.go @@ -6,8 +6,8 @@ import ( "github.com/armosec/opa-utils/reporthandling" ) -// K8SResources map[//]<[]resource objects> -type K8SResources map[string][]workloadinterface.IMetadata +// K8SResources map[//][] +type K8SResources map[string][]string type OPASessionObj struct { K8SResources *K8SResources // input k8s objects diff --git a/clihandler/cmd/framework.go b/clihandler/cmd/framework.go index e381ecff..03d2e1d2 100644 --- a/clihandler/cmd/framework.go +++ b/clihandler/cmd/framework.go @@ -13,9 +13,34 @@ import ( "github.com/spf13/cobra" ) +var ( + frameworkExample = ` + # Scan all frameworks and submit the results + kubescape scan --submit + + # Scan the NSA framework + kubescape scan framework nsa + + # Scan the NSA and MITRE framework + kubescape scan framework nsa,mitre + + # Scan kubernetes YAML manifest files + kubescape scan framework nsa *.yaml + + # Scan and save the results in the JSON format + kubescape scan --format json --output results.json + + # Save scan results in JSON format + kubescape scan --format json --output results.json + + # Display all resources + kubescape scan --verbose +` +) var frameworkCmd = &cobra.Command{ - Use: fmt.Sprintf("framework [``/`-`] [flags]\nExamples:\n$ kubescape scan framework nsa [flags]\n$ kubescape scan framework mitre,nsa [flags]\n$ kubescape scan framework 'nsa, mitre' [flags]\nSupported frameworks: %s", getter.NativeFrameworks), + Use: "framework [``/`-`] [flags]", Short: fmt.Sprintf("The framework you wish to use. Supported frameworks: %s", strings.Join(getter.NativeFrameworks, ", ")), + Example: frameworkExample, Long: "Execute a scan on a running Kubernetes cluster or `yaml`/`json` files (use glob) or `-` for stdin", ValidArgs: getter.NativeFrameworks, Args: func(cmd *cobra.Command, args []string) error { diff --git a/opaprocessor/processorhandler.go b/opaprocessor/processorhandler.go index dfde0555..f1573f85 100644 --- a/opaprocessor/processorhandler.go +++ b/opaprocessor/processorhandler.go @@ -8,12 +8,12 @@ import ( "github.com/armosec/kubescape/cautils" "github.com/armosec/opa-utils/exceptions" "github.com/armosec/opa-utils/reporthandling" + "github.com/golang/glog" "github.com/armosec/k8s-interface/k8sinterface" "github.com/armosec/k8s-interface/workloadinterface" "github.com/armosec/opa-utils/resources" - "github.com/golang/glog" "github.com/open-policy-agent/opa/ast" "github.com/open-policy-agent/opa/rego" uuid "github.com/satori/go.uuid" @@ -58,7 +58,7 @@ func (opaHandler *OPAProcessorHandler) ProcessRulesListenner() { // process if err := opap.Process(); err != nil { - fmt.Println(err) + // fmt.Println(err) } // edit results @@ -81,7 +81,7 @@ func (opap *OPAProcessor) Process() error { for i := range opap.Frameworks { frameworkReport, err := opap.processFramework(&opap.Frameworks[i]) if err != nil { - errs = fmt.Errorf("%v\n%s", errs, err.Error()) + appendError(&errs, err) } frameworkReports = append(frameworkReports, *frameworkReport) } @@ -95,6 +95,16 @@ func (opap *OPAProcessor) Process() error { return errs } +func appendError(errs *error, err error) { + if err == nil { + return + } + if errs == nil { + errs = &err + } else { + *errs = fmt.Errorf("%v\n%s", *errs, err.Error()) + } +} func (opap *OPAProcessor) processFramework(framework *reporthandling.Framework) (*reporthandling.FrameworkReport, error) { var errs error @@ -105,7 +115,8 @@ func (opap *OPAProcessor) processFramework(framework *reporthandling.Framework) for i := range framework.Controls { controlReport, err := opap.processControl(&framework.Controls[i]) if err != nil { - errs = fmt.Errorf("%v\n%s", errs, err.Error()) + appendError(&errs, err) + // errs = fmt.Errorf("%v\n%s", errs, err.Error()) } if controlReport != nil { controlReports = append(controlReports, *controlReport) @@ -133,7 +144,7 @@ func (opap *OPAProcessor) processControl(control *reporthandling.Control) (*repo for i := range control.Rules { ruleReport, err := opap.processRule(&control.Rules[i]) if err != nil { - errs = fmt.Errorf("%v\n%s", errs, err.Error()) + appendError(&errs, err) } if ruleReport != nil { ruleReports = append(ruleReports, *ruleReport) @@ -151,7 +162,7 @@ func (opap *OPAProcessor) processRule(rule *reporthandling.PolicyRule) (*reporth return nil, nil } - inputResources, err := reporthandling.RegoResourcesAggregator(rule, getKubernetesObjects(opap.K8SResources, rule.Match)) + inputResources, err := reporthandling.RegoResourcesAggregator(rule, getKubernetesObjects(opap.K8SResources, opap.AllResources, rule.Match)) if err != nil { return nil, fmt.Errorf("error getting aggregated k8sObjects: %s", err.Error()) } @@ -167,8 +178,12 @@ func (opap *OPAProcessor) processRule(rule *reporthandling.PolicyRule) (*reporth } ruleReport.ListInputKinds = workloadinterface.ListMetaIDs(inputResources) + // remove all data from responses, leave only the metadata + keepFields := []string{"kind", "apiVersion", "metadata"} + keepMetadataFields := []string{"name", "namespace", "labels"} + ruleReport.RemoveData(keepFields, keepMetadataFields) + for i := range inputResources { - removeData(inputResources[i]) opap.AllResources[inputResources[i].GetID()] = inputResources[i] } @@ -228,11 +243,9 @@ func (opap *OPAProcessor) regoEval(inputObj []map[string]interface{}, compiledRe // Run evaluation resultSet, err := rego.Eval(context.Background()) if err != nil { - return nil, fmt.Errorf("in 'regoEval', failed to evaluate rule, reason: %s", err.Error()) + return nil, err } results, err := reporthandling.ParseRegoResult(&resultSet) - - // results, err := ParseRegoResult(&resultSet) if err != nil { return results, err } @@ -241,6 +254,11 @@ func (opap *OPAProcessor) regoEval(inputObj []map[string]interface{}, compiledRe } func (opap *OPAProcessor) updateResults() { + // remove data from all objects + for i := range opap.AllResources { + removeData(opap.AllResources[i]) + } + for f := range opap.PostureReport.FrameworkReports { // set exceptions exceptions.SetFrameworkExceptions(&opap.PostureReport.FrameworkReports[f], opap.Exceptions, cautils.ClusterName) @@ -250,17 +268,5 @@ func (opap *OPAProcessor) updateResults() { // set default score reporthandling.SetDefaultScore(&opap.PostureReport.FrameworkReports[f]) - - // edit results - remove data - - // TODO - move function to pkg - use RemoveData - for c := range opap.PostureReport.FrameworkReports[f].ControlReports { - for r, ruleReport := range opap.PostureReport.FrameworkReports[f].ControlReports[c].RuleReports { - // editing the responses -> removing duplications, clearing secret data, etc. - opap.PostureReport.FrameworkReports[f].ControlReports[c].RuleReports[r].RuleResponses = editRuleResponses(ruleReport.RuleResponses) - } - } - } - } diff --git a/opaprocessor/processorhandler_test.go b/opaprocessor/processorhandler_test.go index afa73d14..320f2baf 100644 --- a/opaprocessor/processorhandler_test.go +++ b/opaprocessor/processorhandler_test.go @@ -19,12 +19,18 @@ func TestProcess(t *testing.T) { // set k8s k8sResources := make(cautils.K8SResources) - k8sResources["/v1/pods"] = workloadinterface.ListMapToMeta(k8sinterface.ConvertUnstructuredSliceToMap(k8sinterface.V1KubeSystemNamespaceMock().Items)) + allResources := make(map[string]workloadinterface.IMetadata) + imetaObj := workloadinterface.ListMapToMeta(k8sinterface.ConvertUnstructuredSliceToMap(k8sinterface.V1KubeSystemNamespaceMock().Items)) + for i := range imetaObj { + allResources[imetaObj[i].GetID()] = imetaObj[i] + } + k8sResources["/v1/pods"] = workloadinterface.ListMetaIDs(imetaObj) // set opaSessionObj opaSessionObj := cautils.NewOPASessionObjMock() opaSessionObj.Frameworks = []reporthandling.Framework{*reporthandling.MockFrameworkA()} opaSessionObj.K8SResources = &k8sResources + opaSessionObj.AllResources = allResources opap := NewOPAProcessor(opaSessionObj, resources.NewRegoDependenciesDataMock()) opap.Process() diff --git a/opaprocessor/processorhandlerutils.go b/opaprocessor/processorhandlerutils.go index a6f5ad2a..b5724cc1 100644 --- a/opaprocessor/processorhandlerutils.go +++ b/opaprocessor/processorhandlerutils.go @@ -13,7 +13,7 @@ import ( "github.com/golang/glog" ) -func getKubernetesObjects(k8sResources *cautils.K8SResources, match []reporthandling.RuleMatchObjects) []workloadinterface.IMetadata { +func getKubernetesObjects(k8sResources *cautils.K8SResources, allResources map[string]workloadinterface.IMetadata, match []reporthandling.RuleMatchObjects) []workloadinterface.IMetadata { k8sObjects := []workloadinterface.IMetadata{} for m := range match { for _, groups := range match[m].APIGroups { @@ -26,7 +26,9 @@ func getKubernetesObjects(k8sResources *cautils.K8SResources, match []reporthand continue // glog.Errorf("Resource '%s' is nil, probably failed to pull the resource", groupResource) } - k8sObjects = append(k8sObjects, k8sObj...) + for i := range k8sObj { + k8sObjects = append(k8sObjects, allResources[k8sObj[i]]) + } } } } @@ -45,28 +47,6 @@ func getRuleDependencies() (map[string]string, error) { return modules, nil } -//editRuleResponses editing the responses -> removing duplications, clearing secret data, etc. -func editRuleResponses(ruleResponses []reporthandling.RuleResponse) []reporthandling.RuleResponse { - lenRuleResponses := len(ruleResponses) - for i := 0; i < lenRuleResponses; i++ { - for j := range ruleResponses[i].AlertObject.K8SApiObjects { - w := workloadinterface.NewWorkloadObj(ruleResponses[i].AlertObject.K8SApiObjects[j]) - if w == nil { - continue - } - - cleanRuleResponses(w) - ruleResponses[i].AlertObject.K8SApiObjects[j] = w.GetWorkload() - } - } - return ruleResponses -} -func cleanRuleResponses(workload k8sinterface.IWorkload) { - if workload.GetKind() == "Secret" { - workload.RemoveSecretData() - } -} - func ruleWithArmoOpaDependency(annotations map[string]interface{}) bool { if annotations == nil { return false @@ -107,36 +87,25 @@ func removeData(obj workloadinterface.IMetadata) { workload := workloadinterface.NewWorkloadObj(obj.GetObject()) switch workload.GetKind() { case "Secret": - removeSecretData(obj) + removeSecretData(workload) case "ConfigMap": - removeConfigMapData(obj) + removeConfigMapData(workload) default: - removePodData(obj) + removePodData(workload) } } -func removeConfigMapData(obj workloadinterface.IMetadata) { - if !workloadinterface.IsTypeWorkload(obj.GetObject()) { - return // remove data only from kubernetes objects - } - workload := workloadinterface.NewWorkloadObj(obj.GetObject()) +func removeConfigMapData(workload workloadinterface.IWorkload) { workload.RemoveAnnotation("kubectl.kubernetes.io/last-applied-configuration") workloadinterface.RemoveFromMap(workload.GetObject(), "data") workloadinterface.RemoveFromMap(workload.GetObject(), "metadata", "managedFields") } -func removeSecretData(obj workloadinterface.IMetadata) { - if !workloadinterface.IsTypeWorkload(obj.GetObject()) { - return // remove data only from kubernetes objects - } - workloadinterface.NewWorkloadObj(obj.GetObject()).RemoveSecretData() - workloadinterface.RemoveFromMap(obj.GetObject(), "metadata", "managedFields") +func removeSecretData(workload workloadinterface.IWorkload) { + workloadinterface.NewWorkloadObj(workload.GetObject()).RemoveSecretData() + workloadinterface.RemoveFromMap(workload.GetObject(), "metadata", "managedFields") } -func removePodData(obj workloadinterface.IMetadata) { - if !workloadinterface.IsTypeWorkload(obj.GetObject()) { - return // remove data only from kubernetes objects - } - workload := workloadinterface.NewWorkloadObj(obj.GetObject()) +func removePodData(workload workloadinterface.IWorkload) { workload.RemoveAnnotation("kubectl.kubernetes.io/last-applied-configuration") workloadinterface.RemoveFromMap(workload.GetObject(), "metadata", "managedFields") diff --git a/policyhandler/handlenotification.go b/policyhandler/handlenotification.go index 6346af3d..27bad7a6 100644 --- a/policyhandler/handlenotification.go +++ b/policyhandler/handlenotification.go @@ -40,47 +40,53 @@ func (policyHandler *PolicyHandler) HandleNotificationRequest(notification *repo return err } - k8sResources, err := policyHandler.getResources(notification, opaSessionObj, scanInfo) + err := policyHandler.getResources(notification, opaSessionObj, scanInfo) if err != nil { return err } - if k8sResources == nil || len(*k8sResources) == 0 { + if opaSessionObj.K8SResources == nil || len(*opaSessionObj.K8SResources) == 0 { return fmt.Errorf("empty list of resources") } - opaSessionObj.K8SResources = k8sResources - for i := range *k8sResources { - for resourceIdx := range (*k8sResources)[i] { - // TODO: add remove data function - opaSessionObj.AllResources[(*k8sResources)[i][resourceIdx].GetID()] = (*k8sResources)[i][resourceIdx] - } - } + // update channel *policyHandler.processPolicy <- opaSessionObj return nil } -func (policyHandler *PolicyHandler) getResources(notification *reporthandling.PolicyNotification, opaSessionObj *cautils.OPASessionObj, scanInfo *cautils.ScanInfo) (*cautils.K8SResources, error) { +func (policyHandler *PolicyHandler) getResources(notification *reporthandling.PolicyNotification, opaSessionObj *cautils.OPASessionObj, scanInfo *cautils.ScanInfo) error { opaSessionObj.PostureReport.ClusterAPIServerInfo = policyHandler.resourceHandler.GetClusterAPIServerInfo() - resourcesMap, err := policyHandler.resourceHandler.GetResources(opaSessionObj.Frameworks, ¬ification.Designators) + resourcesMap, allResources, err := policyHandler.resourceHandler.GetResources(opaSessionObj.Frameworks, ¬ification.Designators) if err != nil { - return resourcesMap, err + return err } + + if err := policyHandler.collectHostResources(allResources, resourcesMap); err != nil { + return err + } + opaSessionObj.K8SResources = resourcesMap + opaSessionObj.AllResources = allResources + + cautils.SuccessTextDisplay("Let’s start!!!") + return nil +} + +func (policyHandler *PolicyHandler) collectHostResources(allResources map[string]workloadinterface.IMetadata, resourcesMap *cautils.K8SResources) error { hostResources, err := policyHandler.hostSensorHandler.CollectResources() if err != nil { - return resourcesMap, err + return err } for rscIdx := range hostResources { groupResources := k8sinterface.ResourceGroupToString(hostResources[rscIdx].Group, hostResources[rscIdx].GetApiVersion(), hostResources[rscIdx].GetKind()) for _, groupResource := range groupResources { - grpReasorceList, ok := (*resourcesMap)[groupResource] + allResources[hostResources[rscIdx].GetID()] = &hostResources[rscIdx] + + grpResourceList, ok := (*resourcesMap)[groupResource] if !ok { - grpReasorceList = make([]workloadinterface.IMetadata, 0) + grpResourceList = make([]string, 0) } - grpReasorceList = append(grpReasorceList, &hostResources[rscIdx]) - (*resourcesMap)[groupResource] = grpReasorceList + (*resourcesMap)[groupResource] = append(grpResourceList, hostResources[rscIdx].GetID()) } } - cautils.SuccessTextDisplay("Let’s start!!!") - return resourcesMap, nil + return nil } diff --git a/resourcehandler/filesloader.go b/resourcehandler/filesloader.go index 2a268ea5..0cbe8a41 100644 --- a/resourcehandler/filesloader.go +++ b/resourcehandler/filesloader.go @@ -43,13 +43,19 @@ func NewFileResourceHandler(inputPatterns []string) *FileResourceHandler { } } -func (fileHandler *FileResourceHandler) GetResources(frameworks []reporthandling.Framework, designator *armotypes.PortalDesignator) (*cautils.K8SResources, error) { +func (fileHandler *FileResourceHandler) GetResources(frameworks []reporthandling.Framework, designator *armotypes.PortalDesignator) (*cautils.K8SResources, map[string]workloadinterface.IMetadata, error) { + + // build resources map + // map resources based on framework required resources: map["/group/version/kind"][] + k8sResources := setResourceMap(frameworks) + allResources := map[string]workloadinterface.IMetadata{} + workloads := []workloadinterface.IMetadata{} // load resource from local file system w, err := loadResourcesFromFiles(fileHandler.inputPatterns) if err != nil { - return nil, err + return nil, allResources, err } if w != nil { workloads = append(workloads, w...) @@ -58,31 +64,32 @@ func (fileHandler *FileResourceHandler) GetResources(frameworks []reporthandling // load resources from url w, err = loadResourcesFromUrl(fileHandler.inputPatterns) if err != nil { - return nil, err + return nil, allResources, err } if w != nil { workloads = append(workloads, w...) } if len(workloads) == 0 { - return nil, fmt.Errorf("empty list of workloads - no workloads found") + return nil, allResources, fmt.Errorf("empty list of workloads - no workloads found") } // map all resources: map["/group/version/kind"][] - allResources := mapResources(workloads) - - // build resources map - // map resources based on framework required resources: map["/group/version/kind"][] - k8sResources := setResourceMap(frameworks) // TODO - support designators + mappedResources := mapResources(workloads) // save only relevant resources - for i := range allResources { + for i := range mappedResources { if _, ok := (*k8sResources)[i]; ok { - (*k8sResources)[i] = allResources[i] + ids := []string{} + for j := range mappedResources[i] { + ids = append(ids, mappedResources[i][j].GetID()) + allResources[mappedResources[i][j].GetID()] = mappedResources[i][j] + } + (*k8sResources)[i] = ids } } - return k8sResources, nil + return k8sResources, allResources, nil } diff --git a/resourcehandler/k8sresources.go b/resourcehandler/k8sresources.go index cd0a8c9f..7d8ed658 100644 --- a/resourcehandler/k8sresources.go +++ b/resourcehandler/k8sresources.go @@ -32,23 +32,26 @@ func NewK8sResourceHandler(k8s *k8sinterface.KubernetesApi, fieldSelector IField } } -func (k8sHandler *K8sResourceHandler) GetResources(frameworks []reporthandling.Framework, designator *armotypes.PortalDesignator) (*cautils.K8SResources, error) { +func (k8sHandler *K8sResourceHandler) GetResources(frameworks []reporthandling.Framework, designator *armotypes.PortalDesignator) (*cautils.K8SResources, map[string]workloadinterface.IMetadata, error) { + allResources := map[string]workloadinterface.IMetadata{} + // get k8s resources cautils.ProgressTextDisplay("Accessing Kubernetes objects") // build resources map + // map resources based on framework required resources: map["/group/version/kind"][] k8sResourcesMap := setResourceMap(frameworks) // get namespace and labels from designator (ignore cluster labels) _, namespace, labels := armotypes.DigestPortalDesignator(designator) // pull k8s recourses - if err := k8sHandler.pullResources(k8sResourcesMap, namespace, labels); err != nil { - return k8sResourcesMap, err + if err := k8sHandler.pullResources(k8sResourcesMap, allResources, namespace, labels); err != nil { + return k8sResourcesMap, allResources, err } cautils.SuccessTextDisplay("Accessed successfully to Kubernetes objects") - return k8sResourcesMap, nil + return k8sResourcesMap, allResources, nil } func (k8sHandler *K8sResourceHandler) GetClusterAPIServerInfo() *version.Info { @@ -59,7 +62,7 @@ func (k8sHandler *K8sResourceHandler) GetClusterAPIServerInfo() *version.Info { } return clusterAPIServerInfo } -func (k8sHandler *K8sResourceHandler) pullResources(k8sResources *cautils.K8SResources, namespace string, labels map[string]string) error { +func (k8sHandler *K8sResourceHandler) pullResources(k8sResources *cautils.K8SResources, allResources map[string]workloadinterface.IMetadata, namespace string, labels map[string]string) error { var errs error for groupResource := range *k8sResources { @@ -73,10 +76,14 @@ func (k8sHandler *K8sResourceHandler) pullResources(k8sResources *cautils.K8SRes } else { errs = fmt.Errorf("%s\n%s", errs, err.Error()) } - } else { - // store result as []map[string]interface{} - (*k8sResources)[groupResource] = ConvertMapListToMeta(k8sinterface.ConvertUnstructuredSliceToMap(k8sinterface.FilterOutOwneredResources(result))) + continue } + // store result as []map[string]interface{} + metaObjs := ConvertMapListToMeta(k8sinterface.ConvertUnstructuredSliceToMap(k8sinterface.FilterOutOwneredResources(result))) + for i := range metaObjs { + allResources[metaObjs[i].GetID()] = metaObjs[i] + } + (*k8sResources)[groupResource] = workloadinterface.ListMetaIDs(metaObjs) } return errs } diff --git a/resourcehandler/resourceshandler.go b/resourcehandler/resourceshandler.go index c2ddf15f..723066eb 100644 --- a/resourcehandler/resourceshandler.go +++ b/resourcehandler/resourceshandler.go @@ -2,12 +2,13 @@ package resourcehandler import ( "github.com/armosec/armoapi-go/armotypes" + "github.com/armosec/k8s-interface/workloadinterface" "github.com/armosec/kubescape/cautils" "github.com/armosec/opa-utils/reporthandling" "k8s.io/apimachinery/pkg/version" ) type IResourceHandler interface { - GetResources(frameworks []reporthandling.Framework, designator *armotypes.PortalDesignator) (*cautils.K8SResources, error) + GetResources([]reporthandling.Framework, *armotypes.PortalDesignator) (*cautils.K8SResources, map[string]workloadinterface.IMetadata, error) GetClusterAPIServerInfo() *version.Info } diff --git a/resultshandling/reporter/reporteventreceiver.go b/resultshandling/reporter/reporteventreceiver.go index 8373f8fe..05ef1fe9 100644 --- a/resultshandling/reporter/reporteventreceiver.go +++ b/resultshandling/reporter/reporteventreceiver.go @@ -42,10 +42,6 @@ func NewReportEventReceiver(tenantConfig *cautils.ConfigObj) *ReportEventReceive } func (report *ReportEventReceiver) ActionSendReport(opaSessionObj *cautils.OPASessionObj) error { - // Remove data before reporting - keepFields := []string{"kind", "apiVersion", "metadata"} - keepMetadataFields := []string{"name", "namespace", "labels"} - opaSessionObj.PostureReport.RemoveData(keepFields, keepMetadataFields) if err := report.prepareReport(opaSessionObj.PostureReport, opaSessionObj.AllResources); err != nil { return err