From 83751e22cc8d39f6f3aa015c16df368a69361413 Mon Sep 17 00:00:00 2001 From: dwertent Date: Sun, 27 Mar 2022 15:37:11 +0300 Subject: [PATCH] fixed report sending --- cmd/go.mod | 2 +- cmd/go.sum | 4 +- core/cautils/customerloader.go | 10 ++- core/cautils/environments.go | 8 +- core/go.mod | 2 +- core/go.sum | 4 +- core/pkg/resourcehandler/repositoryscanner.go | 4 + .../reporter/v2/reporteventreceiver.go | 84 +++++++++++-------- .../reporter/v2/reporteventreceiverutils.go | 15 ++-- httphandler/go.mod | 2 +- httphandler/go.sum | 4 +- 11 files changed, 75 insertions(+), 64 deletions(-) diff --git a/cmd/go.mod b/cmd/go.mod index 64b4ec87..f9bd0560 100644 --- a/cmd/go.mod +++ b/cmd/go.mod @@ -7,7 +7,7 @@ replace github.com/armosec/kubescape/core => ../core require ( github.com/armosec/k8s-interface v0.0.68 github.com/armosec/kubescape/core v0.0.0-00010101000000-000000000000 - github.com/armosec/opa-utils v0.0.126 + github.com/armosec/opa-utils v0.0.127 github.com/armosec/rbac-utils v0.0.14 github.com/google/uuid v1.3.0 github.com/mattn/go-isatty v0.0.14 diff --git a/cmd/go.sum b/cmd/go.sum index 725838af..a56d5ba2 100644 --- a/cmd/go.sum +++ b/cmd/go.sum @@ -109,8 +109,8 @@ github.com/armosec/k8s-interface v0.0.66/go.mod h1:vwprS8qn/iowd5yf0JHpqDsLA5I8W github.com/armosec/k8s-interface v0.0.68 h1:6CtSakISiI47YHkxh+Va9FzZQIBkWa6g9sbiNxq1Zkk= github.com/armosec/k8s-interface v0.0.68/go.mod h1:PeWn41C2uenZi+xfZdyFF/zG5wXACA00htQyknDUWDE= github.com/armosec/opa-utils v0.0.64/go.mod h1:6tQP8UDq2EvEfSqh8vrUdr/9QVSCG4sJfju1SXQOn4c= -github.com/armosec/opa-utils v0.0.126 h1:AVqVWdYcdaqZ8jK92nmZhVy43vh//ITBBjLA4722Vl8= -github.com/armosec/opa-utils v0.0.126/go.mod h1:gap+EaLG5rnyqvIRGxtdNDC9y7VvoGNm90zK8Ls7avQ= +github.com/armosec/opa-utils v0.0.127 h1:uVyH/+pjpiA0oBO6QODjtdCiPgHx7KQ/TcxxkPvAl4c= +github.com/armosec/opa-utils v0.0.127/go.mod h1:gap+EaLG5rnyqvIRGxtdNDC9y7VvoGNm90zK8Ls7avQ= github.com/armosec/rbac-utils v0.0.1/go.mod h1:pQ8CBiij8kSKV7aeZm9FMvtZN28VgA7LZcYyTWimq40= github.com/armosec/rbac-utils v0.0.14 h1:CKYKcgqJEXWF2Hen/B1pVGtS3nDAG1wp9dDv6oNtq90= github.com/armosec/rbac-utils v0.0.14/go.mod h1:Ex/IdGWhGv9HZq6Hs8N/ApzCKSIvpNe/ETqDfnuyah0= diff --git a/core/cautils/customerloader.go b/core/cautils/customerloader.go index 12e9e220..04076bd4 100644 --- a/core/cautils/customerloader.go +++ b/core/cautils/customerloader.go @@ -11,6 +11,7 @@ import ( "github.com/armosec/k8s-interface/k8sinterface" "github.com/armosec/kubescape/core/cautils/getter" + "github.com/armosec/kubescape/core/cautils/logger" corev1 "k8s.io/api/core/v1" ) @@ -135,7 +136,10 @@ func (lc *LocalConfig) UpdateCachedConfig() error { } func (lc *LocalConfig) DeleteCachedConfig() error { - return DeleteConfigFile() + if err := DeleteConfigFile(); err != nil { + logger.L().Warning(err.Error()) + } + return nil } func getTenantConfigFromBE(backendAPI getter.IBackend, configObj *ConfigObj) error { @@ -257,10 +261,10 @@ func (c *ClusterConfig) UpdateCachedConfig() error { func (c *ClusterConfig) DeleteCachedConfig() error { if err := c.deleteConfigMap(); err != nil { - return err + logger.L().Warning(err.Error()) } if err := DeleteConfigFile(); err != nil { - return err + logger.L().Warning(err.Error()) } return nil } diff --git a/core/cautils/environments.go b/core/cautils/environments.go index b2d3c0cf..44e904e3 100644 --- a/core/cautils/environments.go +++ b/core/cautils/environments.go @@ -2,10 +2,6 @@ package cautils // CA environment vars var ( - CustomerGUID = "" - ClusterName = "" - EventReceiverURL = "" - NotificationServerURL = "" - DashboardBackendURL = "" - RestAPIPort = "4001" + CustomerGUID = "" + ClusterName = "" ) diff --git a/core/go.mod b/core/go.mod index fb780423..79aa77d9 100644 --- a/core/go.mod +++ b/core/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/armosec/armoapi-go v0.0.58 github.com/armosec/k8s-interface v0.0.68 - github.com/armosec/opa-utils v0.0.126 + github.com/armosec/opa-utils v0.0.127 github.com/armosec/rbac-utils v0.0.14 github.com/armosec/utils-go v0.0.3 github.com/armosec/utils-k8s-go v0.0.3 diff --git a/core/go.sum b/core/go.sum index fe237b31..f6ac1fc5 100644 --- a/core/go.sum +++ b/core/go.sum @@ -109,8 +109,8 @@ github.com/armosec/k8s-interface v0.0.66/go.mod h1:vwprS8qn/iowd5yf0JHpqDsLA5I8W github.com/armosec/k8s-interface v0.0.68 h1:6CtSakISiI47YHkxh+Va9FzZQIBkWa6g9sbiNxq1Zkk= github.com/armosec/k8s-interface v0.0.68/go.mod h1:PeWn41C2uenZi+xfZdyFF/zG5wXACA00htQyknDUWDE= github.com/armosec/opa-utils v0.0.64/go.mod h1:6tQP8UDq2EvEfSqh8vrUdr/9QVSCG4sJfju1SXQOn4c= -github.com/armosec/opa-utils v0.0.126 h1:AVqVWdYcdaqZ8jK92nmZhVy43vh//ITBBjLA4722Vl8= -github.com/armosec/opa-utils v0.0.126/go.mod h1:gap+EaLG5rnyqvIRGxtdNDC9y7VvoGNm90zK8Ls7avQ= +github.com/armosec/opa-utils v0.0.127 h1:uVyH/+pjpiA0oBO6QODjtdCiPgHx7KQ/TcxxkPvAl4c= +github.com/armosec/opa-utils v0.0.127/go.mod h1:gap+EaLG5rnyqvIRGxtdNDC9y7VvoGNm90zK8Ls7avQ= github.com/armosec/rbac-utils v0.0.1/go.mod h1:pQ8CBiij8kSKV7aeZm9FMvtZN28VgA7LZcYyTWimq40= github.com/armosec/rbac-utils v0.0.14 h1:CKYKcgqJEXWF2Hen/B1pVGtS3nDAG1wp9dDv6oNtq90= github.com/armosec/rbac-utils v0.0.14/go.mod h1:Ex/IdGWhGv9HZq6Hs8N/ApzCKSIvpNe/ETqDfnuyah0= diff --git a/core/pkg/resourcehandler/repositoryscanner.go b/core/pkg/resourcehandler/repositoryscanner.go index 0ba5e9da..6cbc09ef 100644 --- a/core/pkg/resourcehandler/repositoryscanner.go +++ b/core/pkg/resourcehandler/repositoryscanner.go @@ -194,6 +194,10 @@ func (g *GitHubRepository) getHeaders() map[string]string { return map[string]string{"Authorization": fmt.Sprintf("token %s", g.token)} } func (g *GitHubRepository) setTree() error { + if g.isFile { + return nil + } + body, err := getter.HttpGetter(&http.Client{}, g.treeAPI(), g.getHeaders()) if err != nil { return err diff --git a/core/pkg/resultshandling/reporter/v2/reporteventreceiver.go b/core/pkg/resultshandling/reporter/v2/reporteventreceiver.go index d4f996ac..48f4c407 100644 --- a/core/pkg/resultshandling/reporter/v2/reporteventreceiver.go +++ b/core/pkg/resultshandling/reporter/v2/reporteventreceiver.go @@ -7,6 +7,7 @@ import ( "net/url" "os" + "github.com/armosec/k8s-interface/workloadinterface" "github.com/armosec/kubescape/core/cautils" "github.com/armosec/kubescape/core/cautils/getter" "github.com/armosec/kubescape/core/cautils/logger" @@ -109,62 +110,73 @@ func (report *ReportEventReceiver) sendResources(host string, opaSessionObj *cau splittedPostureReport := report.setSubReport(opaSessionObj) counter := 0 reportCounter := 0 - - for _, v := range opaSessionObj.Report.Resources { - r, err := json.Marshal(v) - if err != nil { - return fmt.Errorf("failed to unmarshal resource '%s', reason: %v", v.ResourceID, err) - } - - if counter+len(r) >= MAX_REPORT_SIZE && len(splittedPostureReport.Resources) > 0 { - - // send report - if err := report.sendReport(host, splittedPostureReport, reportCounter, false); err != nil { - return err - } - reportCounter++ - - // delete resources - splittedPostureReport.Resources = []reporthandling.Resource{} - splittedPostureReport.Results = []resourcesresults.Result{} - - // restart counter - counter = 0 - } - - counter += len(r) - splittedPostureReport.Resources = append(splittedPostureReport.Resources, v) + if err := report.setResources(splittedPostureReport, opaSessionObj.AllResources, &counter, &reportCounter, host); err != nil { + return err + } + if err := report.setResults(splittedPostureReport, opaSessionObj.ResourcesResult, &counter, &reportCounter, host); err != nil { + return err } - for _, v := range opaSessionObj.Report.Results { + return report.sendReport(host, splittedPostureReport, reportCounter, true) +} +func (report *ReportEventReceiver) setResults(reportObj *reporthandlingv2.PostureReport, results map[string]resourcesresults.Result, counter, reportCounter *int, host string) error { + for _, v := range results { r, err := json.Marshal(v) if err != nil { return fmt.Errorf("failed to unmarshal resource '%s', reason: %v", v.GetResourceID(), err) } - if counter+len(r) >= MAX_REPORT_SIZE && len(splittedPostureReport.Results) > 0 { + if *counter+len(r) >= MAX_REPORT_SIZE && len(reportObj.Results) > 0 { // send report - if err := report.sendReport(host, splittedPostureReport, reportCounter, false); err != nil { + if err := report.sendReport(host, reportObj, *reportCounter, false); err != nil { return err } - reportCounter++ + *reportCounter++ // delete results - splittedPostureReport.Results = []resourcesresults.Result{} - splittedPostureReport.Resources = []reporthandling.Resource{} + reportObj.Results = []resourcesresults.Result{} + reportObj.Resources = []reporthandling.Resource{} // restart counter - counter = 0 + *counter = 0 } - counter += len(r) - splittedPostureReport.Results = append(splittedPostureReport.Results, v) + *counter += len(r) + reportObj.Results = append(reportObj.Results, v) } - - return report.sendReport(host, splittedPostureReport, reportCounter, true) + return nil } +func (report *ReportEventReceiver) setResources(reportObj *reporthandlingv2.PostureReport, allResources map[string]workloadinterface.IMetadata, counter, reportCounter *int, host string) error { + for resourceID, v := range allResources { + resource := reporthandling.NewResourceIMetadata(v) + r, err := json.Marshal(resource) + if err != nil { + return fmt.Errorf("failed to unmarshal resource '%s', reason: %v", resourceID, err) + } + + if *counter+len(r) >= MAX_REPORT_SIZE && len(reportObj.Resources) > 0 { + + // send report + if err := report.sendReport(host, reportObj, *reportCounter, false); err != nil { + return err + } + *reportCounter++ + + // delete resources + reportObj.Resources = []reporthandling.Resource{} + reportObj.Results = []resourcesresults.Result{} + + // restart counter + *counter = 0 + } + + *counter += len(r) + reportObj.Resources = append(reportObj.Resources, *resource) + } + return nil +} func (report *ReportEventReceiver) sendReport(host string, postureReport *reporthandlingv2.PostureReport, counter int, isLastReport bool) error { postureReport.PaginationInfo = reporthandlingv2.PaginationMarks{ ReportNumber: counter, diff --git a/core/pkg/resultshandling/reporter/v2/reporteventreceiverutils.go b/core/pkg/resultshandling/reporter/v2/reporteventreceiverutils.go index 438637fd..00f643f2 100644 --- a/core/pkg/resultshandling/reporter/v2/reporteventreceiverutils.go +++ b/core/pkg/resultshandling/reporter/v2/reporteventreceiverutils.go @@ -3,10 +3,8 @@ package v2 import ( "net/url" - "github.com/armosec/k8s-interface/workloadinterface" "github.com/armosec/kubescape/core/cautils" "github.com/armosec/kubescape/core/cautils/getter" - "github.com/armosec/opa-utils/reporthandling" reporthandlingv2 "github.com/armosec/opa-utils/reporthandling/v2" "github.com/google/uuid" ) @@ -35,21 +33,18 @@ func hostToString(host *url.URL, reportID string) string { } func (report *ReportEventReceiver) setSubReport(opaSessionObj *cautils.OPASessionObj) *reporthandlingv2.PostureReport { - return &reporthandlingv2.PostureReport{ + reportObj := &reporthandlingv2.PostureReport{ CustomerGUID: report.customerGUID, ClusterName: report.clusterName, ReportID: report.reportID, ReportGenerationTime: opaSessionObj.Report.ReportGenerationTime, SummaryDetails: opaSessionObj.Report.SummaryDetails, Attributes: opaSessionObj.Report.Attributes, - ClusterCloudProvider: opaSessionObj.Report.ClusterCloudProvider, ClusterAPIServerInfo: opaSessionObj.Report.ClusterAPIServerInfo, - Metadata: *opaSessionObj.Metadata, } -} -func iMetaToResource(obj workloadinterface.IMetadata) *reporthandling.Resource { - return &reporthandling.Resource{ - ResourceID: obj.GetID(), - Object: obj.GetObject(), + if opaSessionObj.Metadata != nil { + reportObj.ClusterCloudProvider = opaSessionObj.Metadata.ClusterMetadata.CloudProvider + reportObj.Metadata = *opaSessionObj.Metadata } + return reportObj } diff --git a/httphandler/go.mod b/httphandler/go.mod index 69367374..959b3dc5 100644 --- a/httphandler/go.mod +++ b/httphandler/go.mod @@ -6,7 +6,7 @@ replace github.com/armosec/kubescape/core => ../core require ( github.com/armosec/kubescape/core v0.0.0-00010101000000-000000000000 - github.com/armosec/opa-utils v0.0.126 + github.com/armosec/opa-utils v0.0.127 github.com/armosec/utils-go v0.0.3 github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 diff --git a/httphandler/go.sum b/httphandler/go.sum index 8cf26b2e..e0f03bb9 100644 --- a/httphandler/go.sum +++ b/httphandler/go.sum @@ -109,8 +109,8 @@ github.com/armosec/k8s-interface v0.0.66/go.mod h1:vwprS8qn/iowd5yf0JHpqDsLA5I8W github.com/armosec/k8s-interface v0.0.68 h1:6CtSakISiI47YHkxh+Va9FzZQIBkWa6g9sbiNxq1Zkk= github.com/armosec/k8s-interface v0.0.68/go.mod h1:PeWn41C2uenZi+xfZdyFF/zG5wXACA00htQyknDUWDE= github.com/armosec/opa-utils v0.0.64/go.mod h1:6tQP8UDq2EvEfSqh8vrUdr/9QVSCG4sJfju1SXQOn4c= -github.com/armosec/opa-utils v0.0.126 h1:AVqVWdYcdaqZ8jK92nmZhVy43vh//ITBBjLA4722Vl8= -github.com/armosec/opa-utils v0.0.126/go.mod h1:gap+EaLG5rnyqvIRGxtdNDC9y7VvoGNm90zK8Ls7avQ= +github.com/armosec/opa-utils v0.0.127 h1:uVyH/+pjpiA0oBO6QODjtdCiPgHx7KQ/TcxxkPvAl4c= +github.com/armosec/opa-utils v0.0.127/go.mod h1:gap+EaLG5rnyqvIRGxtdNDC9y7VvoGNm90zK8Ls7avQ= github.com/armosec/rbac-utils v0.0.1/go.mod h1:pQ8CBiij8kSKV7aeZm9FMvtZN28VgA7LZcYyTWimq40= github.com/armosec/rbac-utils v0.0.14 h1:CKYKcgqJEXWF2Hen/B1pVGtS3nDAG1wp9dDv6oNtq90= github.com/armosec/rbac-utils v0.0.14/go.mod h1:Ex/IdGWhGv9HZq6Hs8N/ApzCKSIvpNe/ETqDfnuyah0=