Merge pull request #694 from Moshe-Rappaport-CA/support-scanning-several-files

Support scanning several files

cmd/scan/control.go

@@ -70,7 +70,7 @@ func getControlCmd(ks meta.IKubescape, scanInfo *cautils.ScanInfo) *cobra.Comman
 
 				if len(args) > 1 {
 					if len(args[1:]) == 0 || args[1] != "-" {
-						scanInfo.InputPatterns = []string{args[1]}
+						scanInfo.InputPatterns = args[1:]
 					} else { // store stdin to file - do NOT move to separate function !!
 						tempFile, err := os.CreateTemp(".", "tmp-kubescape*.yaml")
 						if err != nil {

cmd/scan/framework.go

@@ -80,7 +80,7 @@ func getFrameworkCmd(ks meta.IKubescape, scanInfo *cautils.ScanInfo) *cobra.Comm
 				}
 				if len(args) > 1 {
 					if len(args[1:]) == 0 || args[1] != "-" {
-						scanInfo.InputPatterns = []string{args[1]}
+						scanInfo.InputPatterns = args[1:]
 					} else { // store stdin to file - do NOT move to separate function !!
 						tempFile, err := os.CreateTemp(".", "tmp-kubescape*.yaml")
 						if err != nil {

core/cautils/fileutils.go

@@ -13,7 +13,6 @@ import (
 
 	logger "github.com/kubescape/go-logger"
 	"github.com/kubescape/opa-utils/objectsenvelopes"
-	"github.com/kubescape/opa-utils/objectsenvelopes/listworkloads"
 	"github.com/kubescape/opa-utils/objectsenvelopes/localworkload"
 
 	"gopkg.in/yaml.v2"
@@ -191,8 +190,10 @@ func readYamlFile(yamlFile []byte) ([]workloadinterface.IMetadata, error) {
 		}
 		if obj, ok := j.(map[string]interface{}); ok {
 			if o := objectsenvelopes.NewObject(obj); o != nil {
-				if list, ok := o.(*listworkloads.ListWorkloads); ok && list != nil {
-					yamlObjs = append(yamlObjs, handleListObject(list)...)
+				if o.GetObjectType() == workloadinterface.TypeListWorkloads {
+					if list := workloadinterface.NewListWorkloadsObj(o.GetObject()); list != nil {
+						yamlObjs = append(yamlObjs, list.GetItems()...)
+					}
 				} else {
 					yamlObjs = append(yamlObjs, o)
 				}
@@ -324,19 +325,3 @@ func GetFileFormat(filePath string) FileFormat {
 		return FileFormat(filePath)
 	}
 }
-
-// handleListObject handle a List manifest
-func handleListObject(list *listworkloads.ListWorkloads) []workloadinterface.IMetadata {
-
-	yamlObjs := []workloadinterface.IMetadata{}
-	if items := list.GetItems(); items != nil {
-		for item := range items {
-			if m, ok := items[item].(map[string]interface{}); ok && m != nil {
-				if o := objectsenvelopes.NewObject(m); o != nil {
-					yamlObjs = append(yamlObjs, o)
-				}
-			}
-		}
-	}
-	return yamlObjs
-}

core/cautils/reportv2tov1.go

@@ -56,7 +56,9 @@ func controlReportV2ToV1(opaSessionObj *OPASessionObj, frameworkName string, con
 
 		rulesv1 := map[string]reporthandling.RuleReport{}
 
-		for _, resourceID := range crv2.ListResourcesIDs().All() {
+		iter := crv2.ListResourcesIDs().All()
+		for iter.HasNext() {
+			resourceID := iter.Next()
 			if result, ok := opaSessionObj.ResourcesResult[resourceID]; ok {
 				for _, rulev2 := range result.ListRulesOfControl(crv2.GetID(), "") {
 

core/pkg/opaprocessor/processorhandler_test.go

@@ -42,7 +42,7 @@ func TestProcessResourcesResult(t *testing.T) {
 
 	assert.Equal(t, 1, len(opaSessionObj.ResourcesResult))
 	res := opaSessionObj.ResourcesResult[deployment.GetID()]
-	assert.Equal(t, 2, len(res.ListControlsIDs(nil).All()))
+	assert.Equal(t, 2, res.ListControlsIDs(nil).All().Len())
 	assert.Equal(t, 1, len(res.ListControlsIDs(nil).Failed()))
 	assert.Equal(t, 1, len(res.ListControlsIDs(nil).Passed()))
 	assert.True(t, res.GetStatus(nil).IsFailed())
@@ -51,8 +51,8 @@ func TestProcessResourcesResult(t *testing.T) {
 
 	opap.updateResults()
 	res = opaSessionObj.ResourcesResult[deployment.GetID()]
-	assert.Equal(t, 2, len(res.ListControlsIDs(nil).All()))
-	assert.Equal(t, 2, len(res.ListControlsIDs(nil).All()))
+	assert.Equal(t, 2, res.ListControlsIDs(nil).All().Len())
+	assert.Equal(t, 2, res.ListControlsIDs(nil).All().Len())
 	assert.Equal(t, 1, len(res.ListControlsIDs(nil).Failed()))
 	assert.Equal(t, 1, len(res.ListControlsIDs(nil).Passed()))
 	assert.True(t, res.GetStatus(nil).IsFailed())
@@ -67,13 +67,13 @@ func TestProcessResourcesResult(t *testing.T) {
 	assert.Equal(t, 0, summaryDetails.NumberOfResources().Passed())
 
 	// test resource listing
-	assert.Equal(t, 1, len(summaryDetails.ListResourcesIDs().All()))
+	assert.Equal(t, 1, summaryDetails.ListResourcesIDs().All().Len())
 	assert.Equal(t, 1, len(summaryDetails.ListResourcesIDs().Failed()))
 	assert.Equal(t, 0, len(summaryDetails.ListResourcesIDs().Excluded()))
 	assert.Equal(t, 0, len(summaryDetails.ListResourcesIDs().Passed()))
 
 	// test control listing
-	assert.Equal(t, len(res.ListControlsIDs(nil).All()), summaryDetails.NumberOfControls().All())
+	assert.Equal(t, res.ListControlsIDs(nil).All().Len(), summaryDetails.NumberOfControls().All())
 	assert.Equal(t, len(res.ListControlsIDs(nil).Passed()), summaryDetails.NumberOfControls().Passed())
 	assert.Equal(t, len(res.ListControlsIDs(nil).Failed()), summaryDetails.NumberOfControls().Failed())
 	assert.Equal(t, len(res.ListControlsIDs(nil).Excluded()), summaryDetails.NumberOfControls().Excluded())
@@ -83,7 +83,7 @@ func TestProcessResourcesResult(t *testing.T) {
 	opap.updateResults()
 
 	res = opaSessionObj.ResourcesResult[deployment.GetID()]
-	assert.Equal(t, 2, len(res.ListControlsIDs(nil).All()))
+	assert.Equal(t, 2, res.ListControlsIDs(nil).All().Len())
 	assert.Equal(t, 1, len(res.ListControlsIDs(nil).Excluded()))
 	assert.Equal(t, 1, len(res.ListControlsIDs(nil).Passed()))
 	assert.True(t, res.GetStatus(nil).IsExcluded())
@@ -93,7 +93,7 @@ func TestProcessResourcesResult(t *testing.T) {
 
 	// test resource listing
 	summaryDetails = opaSessionObj.Report.SummaryDetails
-	assert.Equal(t, 1, len(summaryDetails.ListResourcesIDs().All()))
+	assert.Equal(t, 1, summaryDetails.ListResourcesIDs().All().Len())
 	assert.Equal(t, 1, len(summaryDetails.ListResourcesIDs().Failed()))
 	assert.Equal(t, 0, len(summaryDetails.ListResourcesIDs().Excluded()))
 	assert.Equal(t, 0, len(summaryDetails.ListResourcesIDs().Passed()))

core/pkg/resourcehandler/filesloader.go

@@ -37,19 +37,62 @@ func (fileHandler *FileResourceHandler) GetResources(sessionObj *cautils.OPASess
 	// map resources based on framework required resources: map["/group/version/kind"][]<k8s workloads ids>
 	k8sResources := setK8sResourceMap(sessionObj.Policies)
 	allResources := map[string]workloadinterface.IMetadata{}
-	workloadIDToSource := make(map[string]reporthandling.Source, 0)
 	ksResources := &cautils.KSResources{}
 
-	workloads := []workloadinterface.IMetadata{}
-
 	if len(fileHandler.inputPatterns) == 0 {
 		return nil, nil, nil, fmt.Errorf("missing input")
 	}
-	path := fileHandler.inputPatterns[0]
+
+	logger.L().Info("Accessing local objects")
+	cautils.StartSpinner()
+
+	for path := range fileHandler.inputPatterns {
+		workloadIDToSource, workloads, err := getResourcesFromPath(fileHandler.inputPatterns[path])
+		if err != nil {
+			return nil, allResources, nil, err
+		}
+		if len(workloads) == 0 {
+			logger.L().Debug("path ignored because contains only a non-kubernetes file", helpers.String("path", fileHandler.inputPatterns[path]))
+		}
+
+		for k, v := range workloadIDToSource {
+			sessionObj.ResourceSource[k] = v
+		}
+
+		// map all resources: map["/apiVersion/version/kind"][]<k8s workloads>
+		mappedResources := mapResources(workloads)
+
+		// save only relevant resources
+		for i := range mappedResources {
+			if _, ok := (*k8sResources)[i]; ok {
+				ids := []string{}
+				for j := range mappedResources[i] {
+					ids = append(ids, mappedResources[i][j].GetID())
+					allResources[mappedResources[i][j].GetID()] = mappedResources[i][j]
+				}
+				(*k8sResources)[i] = append((*k8sResources)[i], ids...)
+			}
+		}
+
+	}
+
+	if err := fileHandler.registryAdaptors.collectImagesVulnerabilities(k8sResources, allResources, ksResources); err != nil {
+		logger.L().Warning("failed to collect images vulnerabilities", helpers.Error(err))
+	}
+
+	cautils.StopSpinner()
+	logger.L().Success("Done accessing local objects")
+
+	return k8sResources, allResources, ksResources, nil
+}
+
+func getResourcesFromPath(path string) (map[string]reporthandling.Source, []workloadinterface.IMetadata, error) {
+	workloadIDToSource := make(map[string]reporthandling.Source, 0)
+	workloads := []workloadinterface.IMetadata{}
 
 	clonedRepo, err := cloneGitRepo(&path)
 	if err != nil {
-		return nil, allResources, nil, err
+		return nil, nil, err
 	}
 	if clonedRepo != "" {
 		defer os.RemoveAll(clonedRepo)
@@ -63,9 +106,6 @@ func (fileHandler *FileResourceHandler) GetResources(sessionObj *cautils.OPASess
 	}
 
 	// load resource from local file system
-	logger.L().Info("Accessing local objects")
-	cautils.StartSpinner()
-
 	sourceToWorkloads := cautils.LoadResourcesFromFiles(path, repoRoot)
 
 	// update workloads and workloadIDToSource
@@ -156,37 +196,7 @@ func (fileHandler *FileResourceHandler) GetResources(sessionObj *cautils.OPASess
 		logger.L().Debug("helm templates found in local storage", helpers.Int("helmTemplates", len(helmSourceToWorkloads)), helpers.Int("workloads", len(workloads)))
 	}
 
-	// addCommitData(fileHandler.inputPatterns[0], workloadIDToSource)
-
-	if len(workloads) == 0 {
-		return nil, allResources, nil, fmt.Errorf("empty list of workloads - no workloads found")
-	}
-
-	sessionObj.ResourceSource = workloadIDToSource
-
-	// map all resources: map["/apiVersion/version/kind"][]<k8s workloads>
-	mappedResources := mapResources(workloads)
-
-	// save only relevant resources
-	for i := range mappedResources {
-		if _, ok := (*k8sResources)[i]; ok {
-			ids := []string{}
-			for j := range mappedResources[i] {
-				ids = append(ids, mappedResources[i][j].GetID())
-				allResources[mappedResources[i][j].GetID()] = mappedResources[i][j]
-			}
-			(*k8sResources)[i] = ids
-		}
-	}
-
-	if err := fileHandler.registryAdaptors.collectImagesVulnerabilities(k8sResources, allResources, ksResources); err != nil {
-		logger.L().Warning("failed to collect images vulnerabilities", helpers.Error(err))
-	}
-
-	cautils.StopSpinner()
-	logger.L().Success("Accessed to local objects")
-
-	return k8sResources, allResources, ksResources, nil
+	return workloadIDToSource, workloads, nil
 }
 
 func (fileHandler *FileResourceHandler) GetClusterAPIServerInfo() *version.Info {

core/pkg/resultshandling/printer/v2/junit.go

@@ -155,7 +155,9 @@ func listTestsSuite(results *cautils.OPASessionObj) []JUnitTestSuite {
 func testsCases(results *cautils.OPASessionObj, controls reportsummary.IControlsSummaries, classname string) []JUnitTestCase {
 	var testCases []JUnitTestCase
 
-	for _, cID := range controls.ListControlsIDs().All() {
+	iter := controls.ListControlsIDs().All()
+	for iter.HasNext() {
+		cID := iter.Next()
 		testCase := JUnitTestCase{}
 		control := results.Report.SummaryDetails.Controls.GetControl(reportsummary.EControlCriteriaID, cID)
 

core/pkg/resultshandling/printer/v2/prettyprinter.go

@@ -261,7 +261,7 @@ func controlCountersForSummary(counters reportsummary.ICounters) string {
 }
 
 func controlCountersForResource(l *helpersv1.AllLists) string {
-	return fmt.Sprintf("Controls: %d (Failed: %d, Excluded: %d)", len(l.All()), len(l.Failed()), len(l.Excluded()))
+	return fmt.Sprintf("Controls: %d (Failed: %d, Excluded: %d)", l.All().Len(), len(l.Failed()), len(l.Excluded()))
 }
 func getSeparator(sep string) string {
 	s := ""

go.mod

@@ -15,8 +15,8 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/johnfercher/maroto v0.37.0
 	github.com/kubescape/go-logger v0.0.5
-	github.com/kubescape/k8s-interface v0.0.82
-	github.com/kubescape/opa-utils v0.0.173
+	github.com/kubescape/k8s-interface v0.0.83
+	github.com/kubescape/opa-utils v0.0.176
 	github.com/kubescape/rbac-utils v0.0.17
 	github.com/libgit2/git2go/v33 v33.0.9
 	github.com/mattn/go-isatty v0.0.14

go.sum

@@ -837,10 +837,10 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kubescape/go-logger v0.0.5 h1:lA25XvI4LT4JqwnNkIDUtwdYS4KVmnbWrVB7SRte2Es=
 github.com/kubescape/go-logger v0.0.5/go.mod h1:DnVWEvC90LFY1nNMaNo6nBVOcqkLMK3S0qzXP1fzRvI=
-github.com/kubescape/k8s-interface v0.0.82 h1:Gu09lQDG+Mw81OxA9MK1BSaqxV/FBRaMYX9jzVqOzzw=
-github.com/kubescape/k8s-interface v0.0.82/go.mod h1:ihX96yqar+xogHl45mFE8zT9DLI06iy7XQPAP+j5KJE=
-github.com/kubescape/opa-utils v0.0.173 h1:iL9unxmraS+UFqxd4rqNwEYsrthKD9GW4DLDIP0iGbw=
-github.com/kubescape/opa-utils v0.0.173/go.mod h1:aU4s1y+BWWxNwYw447TmHmNaKHxOWtYsFhZe5GyvOuE=
+github.com/kubescape/k8s-interface v0.0.83 h1:yQ1kWNZmKfBim/+NmxpPI/j7L9ASDq2h3mCNdmYgzqY=
+github.com/kubescape/k8s-interface v0.0.83/go.mod h1:ihX96yqar+xogHl45mFE8zT9DLI06iy7XQPAP+j5KJE=
+github.com/kubescape/opa-utils v0.0.176 h1:8i7fBpigWdIEN/0SLJr4YQP4Q3pwdyJnsQa3+WxcIa0=
+github.com/kubescape/opa-utils v0.0.176/go.mod h1:t3Orz7lBAsjacXtBKB3mwxs6CCnM4+v92JiY74YiPoI=
 github.com/kubescape/rbac-utils v0.0.17 h1:B78kjlTKqjYK/PXwmi4GPysHsFxIwVz1KFb4+IGT29w=
 github.com/kubescape/rbac-utils v0.0.17/go.mod h1:pBwjpcrVeuH/no+DiCZWvlhYtCDzd3U0o/hEZKi+eM8=
 github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo=

httphandler/go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/gorilla/schema v1.2.0
 	github.com/kubescape/go-logger v0.0.5
 	github.com/kubescape/kubescape/v2 v2.0.0-00010101000000-000000000000
-	github.com/kubescape/opa-utils v0.0.173
+	github.com/kubescape/opa-utils v0.0.176
 	github.com/stretchr/testify v1.8.0
 	k8s.io/utils v0.0.0-20220706174534-f6158b442e7c
 )
@@ -108,7 +108,7 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/jung-kurt/gofpdf v1.16.2 // indirect
 	github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
-	github.com/kubescape/k8s-interface v0.0.82 // indirect
+	github.com/kubescape/k8s-interface v0.0.83 // indirect
 	github.com/kubescape/rbac-utils v0.0.17 // indirect
 	github.com/libgit2/git2go/v33 v33.0.9 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect

httphandler/go.sum

@@ -893,10 +893,10 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kubescape/go-logger v0.0.5 h1:lA25XvI4LT4JqwnNkIDUtwdYS4KVmnbWrVB7SRte2Es=
 github.com/kubescape/go-logger v0.0.5/go.mod h1:DnVWEvC90LFY1nNMaNo6nBVOcqkLMK3S0qzXP1fzRvI=
-github.com/kubescape/k8s-interface v0.0.82 h1:Gu09lQDG+Mw81OxA9MK1BSaqxV/FBRaMYX9jzVqOzzw=
-github.com/kubescape/k8s-interface v0.0.82/go.mod h1:ihX96yqar+xogHl45mFE8zT9DLI06iy7XQPAP+j5KJE=
-github.com/kubescape/opa-utils v0.0.173 h1:iL9unxmraS+UFqxd4rqNwEYsrthKD9GW4DLDIP0iGbw=
-github.com/kubescape/opa-utils v0.0.173/go.mod h1:aU4s1y+BWWxNwYw447TmHmNaKHxOWtYsFhZe5GyvOuE=
+github.com/kubescape/k8s-interface v0.0.83 h1:yQ1kWNZmKfBim/+NmxpPI/j7L9ASDq2h3mCNdmYgzqY=
+github.com/kubescape/k8s-interface v0.0.83/go.mod h1:ihX96yqar+xogHl45mFE8zT9DLI06iy7XQPAP+j5KJE=
+github.com/kubescape/opa-utils v0.0.176 h1:8i7fBpigWdIEN/0SLJr4YQP4Q3pwdyJnsQa3+WxcIa0=
+github.com/kubescape/opa-utils v0.0.176/go.mod h1:t3Orz7lBAsjacXtBKB3mwxs6CCnM4+v92JiY74YiPoI=
 github.com/kubescape/rbac-utils v0.0.17 h1:B78kjlTKqjYK/PXwmi4GPysHsFxIwVz1KFb4+IGT29w=
 github.com/kubescape/rbac-utils v0.0.17/go.mod h1:pBwjpcrVeuH/no+DiCZWvlhYtCDzd3U0o/hEZKi+eM8=
 github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo=