diff --git a/cautils/datastructures.go b/cautils/datastructures.go index ed4e0644..63f13664 100644 --- a/cautils/datastructures.go +++ b/cautils/datastructures.go @@ -6,21 +6,23 @@ import ( "github.com/armosec/opa-utils/reporthandling" ) -// K8SResources map[//] +// K8SResources map[//]<[]resource objects> type K8SResources map[string][]workloadinterface.IMetadata type OPASessionObj struct { - Frameworks []reporthandling.Framework - K8SResources *K8SResources - Exceptions []armotypes.PostureExceptionPolicy - PostureReport *reporthandling.PostureReport - RegoInputData RegoInputData // map[][] + K8SResources *K8SResources // input k8s objects + Frameworks []reporthandling.Framework // list of frameworks to scan + AllResources map[string]workloadinterface.IMetadata // all scanned resources, map[] + PostureReport *reporthandling.PostureReport // scan results + Exceptions []armotypes.PostureExceptionPolicy // list of exceptions to apply on scan results + RegoInputData RegoInputData // input passed to rgo for scanning. map[][] } func NewOPASessionObj(frameworks []reporthandling.Framework, k8sResources *K8SResources) *OPASessionObj { return &OPASessionObj{ Frameworks: frameworks, K8SResources: k8sResources, + AllResources: make(map[string]workloadinterface.IMetadata), PostureReport: &reporthandling.PostureReport{ ClusterName: ClusterName, CustomerGUID: CustomerGUID, diff --git a/go.mod b/go.mod index 33dd80bb..95236d8f 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/armosec/armoapi-go v0.0.23 - github.com/armosec/k8s-interface v0.0.19 + github.com/armosec/k8s-interface v0.0.21 github.com/armosec/opa-utils v0.0.53 github.com/armosec/rbac-utils v0.0.1 github.com/armosec/utils-go v0.0.3 @@ -18,6 +18,7 @@ require ( github.com/open-policy-agent/opa v0.33.1 github.com/satori/go.uuid v1.2.0 github.com/spf13/cobra v1.2.1 + github.com/stretchr/testify v1.7.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.22.2 k8s.io/apimachinery v0.22.2 @@ -64,6 +65,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pquerna/cachecontrol v0.1.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/go.sum b/go.sum index 6dd16adc..6145b571 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,9 @@ github.com/armosec/armoapi-go v0.0.2/go.mod h1:vIK17yoKbJRQyZXWWLe3AqfqCRITxW8qm github.com/armosec/armoapi-go v0.0.23 h1:jqoLIWM5CR7DCD9fpFgN0ePqtHvOCoZv/XzCwsUluJU= github.com/armosec/armoapi-go v0.0.23/go.mod h1:iaVVGyc23QGGzAdv4n+szGQg3Rbpixn9yQTU3qWRpaw= github.com/armosec/k8s-interface v0.0.8/go.mod h1:xxS+V5QT3gVQTwZyAMMDrYLWGrfKOpiJ7Jfhfa0w9sM= -github.com/armosec/k8s-interface v0.0.19 h1:GhWxAyBCWLx2Qqw8uFSZ4rpZf3IV5k922mBtpQsoW2g= github.com/armosec/k8s-interface v0.0.19/go.mod h1:0ztwEWLez2lpMMKHbs0DoZV64S0YIA4s72vCs9iAAiE= +github.com/armosec/k8s-interface v0.0.21 h1:x+UCdU7SEKU1E4fPkPQbGneC4BQljMmUcFdyRNAUry4= +github.com/armosec/k8s-interface v0.0.21/go.mod h1:0ztwEWLez2lpMMKHbs0DoZV64S0YIA4s72vCs9iAAiE= github.com/armosec/opa-utils v0.0.53 h1:ibmKPK15kEcm53ddFIu8ZcL+hks1l/3ujMDYL1EXz2Q= github.com/armosec/opa-utils v0.0.53/go.mod h1:9uMk9NbbJydxVztXcwhSgX307xMdGRD2R4pYwzSPi28= github.com/armosec/rbac-utils v0.0.1 h1:N2MI98F/0zbDjmRZ29CNElU1AXkFLk5csd/qAHOBdXY= diff --git a/opaprocessor/processorhandler.go b/opaprocessor/processorhandler.go index 65862645..dfde0555 100644 --- a/opaprocessor/processorhandler.go +++ b/opaprocessor/processorhandler.go @@ -147,26 +147,31 @@ func (opap *OPAProcessor) processControl(control *reporthandling.Control) (*repo } func (opap *OPAProcessor) processRule(rule *reporthandling.PolicyRule) (*reporthandling.RuleReport, error) { - if ruleWithArmoOpaDependency(rule.Attributes) { - return nil, nil - } - if !isRuleKubescapeVersionCompatible(rule) { + if ruleWithArmoOpaDependency(rule.Attributes) || !isRuleKubescapeVersionCompatible(rule) { return nil, nil } + inputResources, err := reporthandling.RegoResourcesAggregator(rule, getKubernetesObjects(opap.K8SResources, rule.Match)) if err != nil { - glog.Error(err) return nil, fmt.Errorf("error getting aggregated k8sObjects: %s", err.Error()) } + ruleReport, err := opap.runOPAOnSingleRule(rule, workloadinterface.ListMetaToMap(inputResources)) if err != nil { + // ruleReport.RuleStatus.Status = reporthandling.StatusFailed ruleReport.RuleStatus.Status = "failure" ruleReport.RuleStatus.Message = err.Error() glog.Error(err) } else { - ruleReport.RuleStatus.Status = "success" + ruleReport.RuleStatus.Status = reporthandling.StatusPassed } ruleReport.ListInputKinds = workloadinterface.ListMetaIDs(inputResources) + + for i := range inputResources { + removeData(inputResources[i]) + opap.AllResources[inputResources[i].GetID()] = inputResources[i] + } + return &ruleReport, err } diff --git a/opaprocessor/processorhandlerutils.go b/opaprocessor/processorhandlerutils.go index 7eeff4a3..77fa8795 100644 --- a/opaprocessor/processorhandlerutils.go +++ b/opaprocessor/processorhandlerutils.go @@ -23,6 +23,7 @@ func getKubernetesObjects(k8sResources *cautils.K8SResources, match []reporthand for _, groupResource := range groupResources { if k8sObj, ok := (*k8sResources)[groupResource]; ok { if k8sObj == nil { + continue // glog.Errorf("Resource '%s' is nil, probably failed to pull the resource", groupResource) } k8sObjects = append(k8sObjects, k8sObj...) @@ -95,3 +96,53 @@ func isRuleKubescapeVersionCompatible(rule *reporthandling.PolicyRule) bool { } return true } + +func removeData(obj workloadinterface.IMetadata) { + if !workloadinterface.IsTypeWorkload(obj.GetObject()) { + return // remove data only from kubernetes objects + } + workload := workloadinterface.NewWorkloadObj(obj.GetObject()) + switch workload.GetKind() { + case "Secret": + removeSecretData(obj) + case "ConfigMap": + removeConfigMapData(obj) + default: + removePodData(obj) + } +} + +func removeConfigMapData(obj workloadinterface.IMetadata) { + if !workloadinterface.IsTypeWorkload(obj.GetObject()) { + return // remove data only from kubernetes objects + } + workload := workloadinterface.NewWorkloadObj(obj.GetObject()) + workload.RemoveAnnotation("kubectl.kubernetes.io/last-applied-configuration") + workloadinterface.RemoveFromMap(workload.GetObject(), "data") + +} +func removeSecretData(obj workloadinterface.IMetadata) { + if !workloadinterface.IsTypeWorkload(obj.GetObject()) { + return // remove data only from kubernetes objects + } + workloadinterface.NewWorkloadObj(obj.GetObject()).RemoveSecretData() + +} +func removePodData(obj workloadinterface.IMetadata) { + if !workloadinterface.IsTypeWorkload(obj.GetObject()) { + return // remove data only from kubernetes objects + } + workload := workloadinterface.NewWorkloadObj(obj.GetObject()) + workload.RemoveAnnotation("kubectl.kubernetes.io/last-applied-configuration") + + containers, err := workload.GetContainers() + if err != nil || len(containers) == 0 { + return + } + for i := range containers { + for j := range containers[i].Env { + containers[i].Env[j].Value = "" + } + } + workloadinterface.SetInMap(workload.GetObject(), workloadinterface.PodSpec(workload.GetKind()), "containers", containers) +} diff --git a/opaprocessor/processorhandlerutils_test.go b/opaprocessor/processorhandlerutils_test.go index 90bf5bbd..7ade114b 100644 --- a/opaprocessor/processorhandlerutils_test.go +++ b/opaprocessor/processorhandlerutils_test.go @@ -3,7 +3,10 @@ package opaprocessor import ( "testing" + "github.com/stretchr/testify/assert" + "github.com/armosec/armoapi-go/armotypes" + "github.com/armosec/k8s-interface/workloadinterface" "github.com/armosec/kubescape/cautils" "github.com/armosec/opa-utils/reporthandling" ) @@ -52,3 +55,17 @@ func TestIsRuleKubescapeVersionCompatible(t *testing.T) { t.Error("error in isRuleKubescapeVersionCompatible") } } + +func TestRemoveData(t *testing.T) { + w := `{"apiVersion":"apps/v1","kind":"Deployment","metadata":{"name":"demoservice-server"},"spec":{"replicas":1,"selector":{"matchLabels":{"app":"demoservice-server"}},"template":{"metadata":{"creationTimestamp":null,"labels":{"app":"demoservice-server"}},"spec":{"containers":[{"env":[{"name":"SERVER_PORT","value":"8089"},{"name":"SLEEP_DURATION","value":"1"},{"name":"DEMO_FOLDERS","value":"/app"},{"name":"ARMO_TEST_NAME","value":"auto_attach_deployment"},{"name":"CAA_ENABLE_CRASH_REPORTER","value":"1"}],"image":"quay.io/armosec/demoservice:v25","imagePullPolicy":"IfNotPresent","name":"demoservice","ports":[{"containerPort":8089,"protocol":"TCP"}],"resources":{},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File"}],"dnsPolicy":"ClusterFirst","restartPolicy":"Always","schedulerName":"default-scheduler","securityContext":{},"terminationGracePeriodSeconds":30}}}}` + obj, _ := workloadinterface.NewWorkload([]byte(w)) + removeData(obj) + + workload := workloadinterface.NewWorkloadObj(obj.GetObject()) + c, _ := workload.GetContainers() + for i := range c { + for _, e := range c[i].Env { + assert.Equal(t, "", e.Value) + } + } +} diff --git a/resultshandling/reporter/reporteventreceiver.go b/resultshandling/reporter/reporteventreceiver.go index 2bd98392..ce8a7b69 100644 --- a/resultshandling/reporter/reporteventreceiver.go +++ b/resultshandling/reporter/reporteventreceiver.go @@ -4,12 +4,16 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" + "github.com/armosec/k8s-interface/workloadinterface" "github.com/armosec/kubescape/cautils" "github.com/armosec/kubescape/cautils/getter" "github.com/armosec/opa-utils/reporthandling" ) +const MAX_REPORT_SIZE = 2097152 // 2 MB + type IReport interface { ActionSendReport(opaSessionObj *cautils.OPASessionObj) error SetCustomerGUID(customerGUID string) @@ -17,9 +21,10 @@ type IReport interface { } type ReportEventReceiver struct { - httpClient *http.Client - clusterName string - customerGUID string + httpClient *http.Client + clusterName string + customerGUID string + eventReceiverURL *url.URL } func NewReportEventReceiver(customerGUID, clusterName string) *ReportEventReceiver { @@ -36,7 +41,7 @@ func (report *ReportEventReceiver) ActionSendReport(opaSessionObj *cautils.OPASe keepMetadataFields := []string{"name", "namespace", "labels"} opaSessionObj.PostureReport.RemoveData(keepFields, keepMetadataFields) - if err := report.send(opaSessionObj.PostureReport); err != nil { + if err := report.prepareReport(opaSessionObj.PostureReport, opaSessionObj.AllResources); err != nil { return err } return nil @@ -50,17 +55,75 @@ func (report *ReportEventReceiver) SetClusterName(clusterName string) { report.clusterName = clusterName } -func (report *ReportEventReceiver) send(postureReport *reporthandling.PostureReport) error { +func (report *ReportEventReceiver) prepareReport(postureReport *reporthandling.PostureReport, allResources map[string]workloadinterface.IMetadata) error { + report.initEventReceiverURL() + host := hostToString(report.eventReceiverURL, postureReport.ReportID) - reqBody, err := json.Marshal(*postureReport) - if err != nil { - return fmt.Errorf("in 'Send' failed to json.Marshal, reason: %v", err) + // send framework results + if err := report.sendReport(host, postureReport); err != nil { + return err } - host := hostToString(report.initEventReceiverURL(), postureReport.ReportID) + // send resources + if err := report.sendResources(host, postureReport, allResources); err != nil { + return err + } + return nil +} + +func (report *ReportEventReceiver) sendResources(host string, postureReport *reporthandling.PostureReport, allResources map[string]workloadinterface.IMetadata) error { + splittedPostureReport := setPaginationReport(postureReport) + counter := 0 + + for _, v := range allResources { + r, err := json.Marshal(*iMetaToResource(v)) + if err != nil { + return fmt.Errorf("failed to unmarshal resource '%s', reason: %v", v.GetID(), err) + } + + if counter+len(r) >= MAX_REPORT_SIZE && len(splittedPostureReport.Resources) > 0 { + + // send report + if err := report.sendReport(host, splittedPostureReport); err != nil { + return err + } + + // delete resources + splittedPostureReport.Resources = []reporthandling.Resource{} + + // restart counter + counter = 0 + } + + counter += len(r) + splittedPostureReport.Resources = append(splittedPostureReport.Resources, *iMetaToResource(v)) + } + + return report.sendReport(host, splittedPostureReport) +} +func (report *ReportEventReceiver) sendReport(host string, postureReport *reporthandling.PostureReport) error { + reqBody, err := json.Marshal(postureReport) + if err != nil { + return fmt.Errorf("in 'sendReport' failed to json.Marshal, reason: %v", err) + } msg, err := getter.HttpPost(report.httpClient, host, nil, reqBody) if err != nil { return fmt.Errorf("%s, %v:%s", host, err, msg) } return err } + +// func (report *ReportEventReceiver) send(postureReport *reporthandling.PostureReport) error { +// report.initEventReceiverURL() +// reqBody, err := json.Marshal(*postureReport) +// if err != nil { +// return fmt.Errorf("in 'Send' failed to json.Marshal, reason: %v", err) +// } +// host := hostToString(report.eventReceiverURL, postureReport.ReportID) + +// msg, err := getter.HttpPost(report.httpClient, host, nil, reqBody) +// if err != nil { +// return fmt.Errorf("%s, %v:%s", host, err, msg) +// } +// return err +// } diff --git a/resultshandling/reporter/reporteventreceiverutils.go b/resultshandling/reporter/reporteventreceiverutils.go index 0f90dbae..8133b5ee 100644 --- a/resultshandling/reporter/reporteventreceiverutils.go +++ b/resultshandling/reporter/reporteventreceiverutils.go @@ -3,11 +3,13 @@ package reporter import ( "net/url" + "github.com/armosec/k8s-interface/workloadinterface" "github.com/armosec/kubescape/cautils/getter" + "github.com/armosec/opa-utils/reporthandling" "github.com/gofrs/uuid" ) -func (report *ReportEventReceiver) initEventReceiverURL() *url.URL { +func (report *ReportEventReceiver) initEventReceiverURL() { urlObj := url.URL{} urlObj.Scheme = "https" @@ -19,7 +21,7 @@ func (report *ReportEventReceiver) initEventReceiverURL() *url.URL { urlObj.RawQuery = q.Encode() - return &urlObj + report.eventReceiverURL = &urlObj } func hostToString(host *url.URL, reportID string) string { @@ -28,3 +30,18 @@ func hostToString(host *url.URL, reportID string) string { host.RawQuery = q.Encode() return host.String() } + +func setPaginationReport(postureReport *reporthandling.PostureReport) *reporthandling.PostureReport { + return &reporthandling.PostureReport{ + CustomerGUID: postureReport.CustomerGUID, + ClusterName: postureReport.ClusterName, + ReportID: postureReport.ReportID, + ReportGenerationTime: postureReport.ReportGenerationTime, + } +} +func iMetaToResource(obj workloadinterface.IMetadata) *reporthandling.Resource { + return &reporthandling.Resource{ + ResourceID: obj.GetID(), + Object: obj.GetObject(), + } +}