Merge pull request #1758 from kubescape/bug_fix/repo_scan

fix the repo scan got stuck bug

.github/workflows/02-release.yaml

@@ -107,4 +107,6 @@ jobs:
       contents: write
     uses: ./.github/workflows/e-post-release.yaml
     needs: [publish-image]
+    with:
+      TAG: ${{ needs.retag.outputs.NEW_TAG }}
     secrets: inherit

.github/workflows/e-post-release.yaml

@@ -1,7 +1,12 @@
 name: e-post_release
 permissions: read-all
 on:
-  workflow_call: {}
+  workflow_call:
+    inputs:
+      TAG:
+        description: 'Tag name'
+        required: true
+        type: string
 jobs:
   post_release:
     name: Post release jobs
@@ -12,6 +17,9 @@ jobs:
           submodules: recursive
       - name: Update new version in krew-index
         uses: rajatjindal/krew-release-bot@v0.0.47
+        if: github.repository_owner == 'kubescape'
+        env:
+          GITHUB_REF: ${{ inputs.TAG }}
       - name: Invoke workflow to update packaging
         uses: benc-uk/workflow-dispatch@v1
         if: github.repository_owner == 'kubescape'

core/cautils/fileutils.go

@@ -38,7 +38,7 @@ type Chart struct {
 }
 
 // LoadResourcesFromHelmCharts scans a given path (recursively) for helm charts, renders the templates and returns a map of workloads and a map of chart names
-func LoadResourcesFromHelmCharts(ctx context.Context, basePath string) (map[string][]workloadinterface.IMetadata, map[string]Chart, map[string]MappingNodes) {
+func LoadResourcesFromHelmCharts(ctx context.Context, basePath string) (map[string][]workloadinterface.IMetadata, map[string]Chart) {
 	directories, _ := listDirs(basePath)
 	helmDirectories := make([]string, 0)
 	for _, dir := range directories {
@@ -49,19 +49,14 @@ func LoadResourcesFromHelmCharts(ctx context.Context, basePath string) (map[stri
 
 	sourceToWorkloads := map[string][]workloadinterface.IMetadata{}
 	sourceToChart := make(map[string]Chart, 0)
-	sourceToNodes := map[string]MappingNodes{}
 	for _, helmDir := range helmDirectories {
 		chart, err := NewHelmChart(helmDir)
 		if err == nil {
-			wls, templateToNodes, errs := chart.GetWorkloadsWithDefaultValues()
+			wls, errs := chart.GetWorkloadsWithDefaultValues()
 			if len(errs) > 0 {
 				logger.L().Ctx(ctx).Warning(fmt.Sprintf("Rendering of Helm chart template '%s', failed: %v", chart.GetName(), errs))
 				continue
 			}
-			for k, v := range templateToNodes {
-				sourceToNodes[k] = v
-			}
-
 			chartName := chart.GetName()
 			for k, v := range wls {
 				sourceToWorkloads[k] = v
@@ -72,7 +67,7 @@ func LoadResourcesFromHelmCharts(ctx context.Context, basePath string) (map[stri
 			}
 		}
 	}
-	return sourceToWorkloads, sourceToChart, sourceToNodes
+	return sourceToWorkloads, sourceToChart
 }
 
 // If the contents at given path is a Kustomize Directory, LoadResourcesFromKustomizeDirectory will

core/cautils/fileutils_test.go

@@ -45,7 +45,7 @@ func TestLoadResourcesFromFiles(t *testing.T) {
 }
 
 func TestLoadResourcesFromHelmCharts(t *testing.T) {
-	sourceToWorkloads, sourceToChartName, _ := LoadResourcesFromHelmCharts(context.TODO(), helmChartPath())
+	sourceToWorkloads, sourceToChartName := LoadResourcesFromHelmCharts(context.TODO(), helmChartPath())
 	assert.Equal(t, 6, len(sourceToWorkloads))
 
 	for file, workloads := range sourceToWorkloads {

core/cautils/helmchart.go

@@ -45,35 +45,24 @@ func (hc *HelmChart) GetDefaultValues() map[string]interface{} {
 	return hc.chart.Values
 }
 
-// GetWorkloads renders chart template using the default values and returns a map of source file to its workloads
-func (hc *HelmChart) GetWorkloadsWithDefaultValues() (map[string][]workloadinterface.IMetadata, map[string]MappingNodes, []error) {
+// GetWorkloadsWithDefaultValues renders chart template using the default values and returns a map of source file to its workloads
+func (hc *HelmChart) GetWorkloadsWithDefaultValues() (map[string][]workloadinterface.IMetadata, []error) {
 	return hc.GetWorkloads(hc.GetDefaultValues())
 }
 
 // GetWorkloads renders chart template using the provided values and returns a map of source (absolute) file path to its workloads
-func (hc *HelmChart) GetWorkloads(values map[string]interface{}) (map[string][]workloadinterface.IMetadata, map[string]MappingNodes, []error) {
+func (hc *HelmChart) GetWorkloads(values map[string]interface{}) (map[string][]workloadinterface.IMetadata, []error) {
 	vals, err := helmchartutil.ToRenderValues(hc.chart, values, helmchartutil.ReleaseOptions{}, nil)
 	if err != nil {
-		return nil, nil, []error{err}
+		return nil, []error{err}
 	}
-
-	// change the chart to template with comment, only is template(.yaml added otherwise no)
-	hc.AddCommentToTemplate()
-
 	sourceToFile, err := helmengine.Render(hc.chart, vals)
 	if err != nil {
-		return nil, nil, []error{err}
+		return nil, []error{err}
 	}
 
-	// get the resouse and analysis and store it to the struct
-	fileMapping := make(map[string]MappingNodes)
-	GetTemplateMapping(sourceToFile, fileMapping)
-
-	// delete the comment from chart and from sourceToFile
-	RemoveComment(sourceToFile)
-
-	workloads := make(map[string][]workloadinterface.IMetadata, 0)
-	errs := []error{}
+	workloads := make(map[string][]workloadinterface.IMetadata)
+	var errs []error
 
 	for path, renderedYaml := range sourceToFile {
 		if !IsYaml(strings.ToLower(path)) {
@@ -87,13 +76,9 @@ func (hc *HelmChart) GetWorkloads(values map[string]interface{}) (map[string][]w
 		if len(wls) == 0 {
 			continue
 		}
-		if firstPathSeparatorIndex := strings.Index(path, string("/")); firstPathSeparatorIndex != -1 {
+		if firstPathSeparatorIndex := strings.Index(path, "/"); firstPathSeparatorIndex != -1 {
 			absPath := filepath.Join(hc.path, path[firstPathSeparatorIndex:])
 
-			if nodes, ok := fileMapping[path]; ok {
-				fileMapping[absPath] = nodes
-				delete(fileMapping, path)
-			}
 			workloads[absPath] = []workloadinterface.IMetadata{}
 			for i := range wls {
 				lw := localworkload.NewLocalWorkload(wls[i].GetObject())
@@ -102,7 +87,7 @@ func (hc *HelmChart) GetWorkloads(values map[string]interface{}) (map[string][]w
 			}
 		}
 	}
-	return workloads, fileMapping, errs
+	return workloads, errs
 }
 
 func (hc *HelmChart) AddCommentToTemplate() {
@@ -121,27 +106,3 @@ func (hc *HelmChart) AddCommentToTemplate() {
 		}
 	}
 }
-
-func RemoveComment(sourceToFile map[string]string) {
-	// commentRe := regexp.MustCompile(CommentFormat)
-	for fileName, file := range sourceToFile {
-		if !IsYaml(strings.ToLower((fileName))) {
-			continue
-		}
-		sourceToFile[fileName] = commentRe.ReplaceAllLiteralString(file, "")
-	}
-}
-
-func GetTemplateMapping(sourceToFile map[string]string, fileMapping map[string]MappingNodes) {
-	for fileName, fileContent := range sourceToFile {
-		mappingNodes, err := GetMapping(fileName, fileContent)
-		if err != nil {
-			// if one file cannot get mapping nodes, generate error, then ignore it
-			logger.L().Warning("Failed to get File Mapping nodes", helpers.String("file name", fileName), helpers.Error(err))
-			continue
-		}
-		if len(mappingNodes.Nodes) != 0 {
-			fileMapping[fileName] = *mappingNodes
-		}
-	}
-}

core/cautils/helmchart_test.go

@@ -83,7 +83,7 @@ func (s *HelmChartTestSuite) TestGetWorkloadsWithOverride() {
 	// Override default value
 	values["image"].(map[string]interface{})["pullPolicy"] = "Never"
 
-	fileToWorkloads, _, errs := chart.GetWorkloads(values)
+	fileToWorkloads, errs := chart.GetWorkloads(values)
 	s.Len(errs, 0)
 
 	s.Lenf(fileToWorkloads, len(s.expectedFiles), "Expected %d files", len(s.expectedFiles))
@@ -111,7 +111,7 @@ func (s *HelmChartTestSuite) TestGetWorkloadsMissingValue() {
 	values := chart.GetDefaultValues()
 	delete(values, "image")
 
-	fileToWorkloads, _, errs := chart.GetWorkloads(values)
+	fileToWorkloads, errs := chart.GetWorkloads(values)
 	s.Nil(fileToWorkloads)
 	s.Len(errs, 1, "Expected an error due to missing value")
 

core/pkg/resourcehandler/filesloader.go

@@ -39,13 +39,12 @@ func (fileHandler *FileResourceHandler) GetResources(ctx context.Context, sessio
 	for path := range scanInfo.InputPatterns {
 		var workloadIDToSource map[string]reporthandling.Source
 		var workloads []workloadinterface.IMetadata
-		var workloadIDToMappingNodes map[string]cautils.MappingNodes
 		var err error
 
 		if scanInfo.ChartPath != "" && scanInfo.FilePath != "" {
-			workloadIDToSource, workloads, workloadIDToMappingNodes, _ = getWorkloadFromHelmChart(ctx, scanInfo.InputPatterns[path], scanInfo.ChartPath, scanInfo.FilePath)
+			workloadIDToSource, workloads, _ = getWorkloadFromHelmChart(ctx, scanInfo.InputPatterns[path], scanInfo.ChartPath, scanInfo.FilePath)
 		} else {
-			workloadIDToSource, workloads, workloadIDToMappingNodes, err = getResourcesFromPath(ctx, scanInfo.InputPatterns[path])
+			workloadIDToSource, workloads, err = getResourcesFromPath(ctx, scanInfo.InputPatterns[path])
 			if err != nil {
 				return nil, allResources, nil, nil, err
 			}
@@ -56,7 +55,7 @@ func (fileHandler *FileResourceHandler) GetResources(ctx context.Context, sessio
 
 		for k, v := range workloadIDToSource {
 			sessionObj.ResourceSource[k] = v
-			sessionObj.TemplateMapping[k] = workloadIDToMappingNodes[k]
+			sessionObj.TemplateMapping[k] = cautils.MappingNodes{}
 		}
 
 		// map all resources: map["/apiVersion/version/kind"][]<k8s workloads>
@@ -102,7 +101,7 @@ func (fileHandler *FileResourceHandler) GetResources(ctx context.Context, sessio
 func (fileHandler *FileResourceHandler) GetCloudProvider() string {
 	return ""
 }
-func getWorkloadFromHelmChart(ctx context.Context, path, helmPath, workloadPath string) (map[string]reporthandling.Source, []workloadinterface.IMetadata, map[string]cautils.MappingNodes, error) {
+func getWorkloadFromHelmChart(ctx context.Context, path, helmPath, workloadPath string) (map[string]reporthandling.Source, []workloadinterface.IMetadata, error) {
 	clonedRepo := cautils.GetClonedPath(path)
 
 	if clonedRepo != "" {
@@ -116,38 +115,31 @@ func getWorkloadFromHelmChart(ctx context.Context, path, helmPath, workloadPath
 	// Get repo root
 	repoRoot, gitRepo := extractGitRepo(clonedRepo)
 
-	helmSourceToWorkloads, helmSourceToChart, helmSourceToNodes := cautils.LoadResourcesFromHelmCharts(ctx, helmPath)
+	helmSourceToWorkloads, helmSourceToChart := cautils.LoadResourcesFromHelmCharts(ctx, helmPath)
 
 	wlSource, ok := helmSourceToWorkloads[workloadPath]
 	if !ok {
-		return nil, nil, nil, fmt.Errorf("workload %s not found in chart %s", workloadPath, helmPath)
+		return nil, nil, fmt.Errorf("workload %s not found in chart %s", workloadPath, helmPath)
 	}
 
 	if len(wlSource) != 1 {
-		return nil, nil, nil, fmt.Errorf("workload %s found multiple times in chart %s", workloadPath, helmPath)
+		return nil, nil, fmt.Errorf("workload %s found multiple times in chart %s", workloadPath, helmPath)
 	}
 
 	helmChart, ok := helmSourceToChart[workloadPath]
 	if !ok {
-		return nil, nil, nil, fmt.Errorf("helmChart not found for workload %s", workloadPath)
-	}
-
-	templatesNodes, ok := helmSourceToNodes[workloadPath]
-	if !ok {
-		return nil, nil, nil, fmt.Errorf("templatesNodes not found for workload %s", workloadPath)
+		return nil, nil, fmt.Errorf("helmChart not found for workload %s", workloadPath)
 	}
 
 	workloadSource := getWorkloadSourceHelmChart(repoRoot, helmPath, gitRepo, helmChart)
 
 	workloadIDToSource := make(map[string]reporthandling.Source, 1)
-	workloadIDToNodes := make(map[string]cautils.MappingNodes, 1)
 	workloadIDToSource[wlSource[0].GetID()] = workloadSource
-	workloadIDToNodes[wlSource[0].GetID()] = templatesNodes
 
 	var workloads []workloadinterface.IMetadata
 	workloads = append(workloads, wlSource...)
 
-	return workloadIDToSource, workloads, workloadIDToNodes, nil
+	return workloadIDToSource, workloads, nil
 
 }
 
@@ -181,9 +173,8 @@ func getWorkloadSourceHelmChart(repoRoot string, source string, gitRepo *cautils
 	}
 }
 
-func getResourcesFromPath(ctx context.Context, path string) (map[string]reporthandling.Source, []workloadinterface.IMetadata, map[string]cautils.MappingNodes, error) {
+func getResourcesFromPath(ctx context.Context, path string) (map[string]reporthandling.Source, []workloadinterface.IMetadata, error) {
 	workloadIDToSource := make(map[string]reporthandling.Source)
-	workloadIDToNodes := make(map[string]cautils.MappingNodes)
 	var workloads []workloadinterface.IMetadata
 
 	clonedRepo := cautils.GetClonedPath(path)
@@ -270,14 +261,10 @@ func getResourcesFromPath(ctx context.Context, path string) (map[string]reportha
 	}
 
 	// load resources from helm charts
-	helmSourceToWorkloads, helmSourceToChart, helmSourceToNodes := cautils.LoadResourcesFromHelmCharts(ctx, path)
+	helmSourceToWorkloads, helmSourceToChart := cautils.LoadResourcesFromHelmCharts(ctx, path)
 	for source, ws := range helmSourceToWorkloads {
 		workloads = append(workloads, ws...)
 		helmChart := helmSourceToChart[source]
-		var templatesNodes cautils.MappingNodes
-		if nodes, ok := helmSourceToNodes[source]; ok {
-			templatesNodes = nodes
-		}
 
 		if clonedRepo != "" && gitRepo != nil {
 			url, err := gitRepo.GetRemoteUrl()
@@ -288,14 +275,12 @@ func getResourcesFromPath(ctx context.Context, path string) (map[string]reportha
 			helmChart.Path = strings.TrimSuffix(url, ".git")
 			repoRoot = ""
 			source = strings.TrimPrefix(source, fmt.Sprintf("%s/", clonedRepo))
-			templatesNodes.TemplateFileName = source
 		}
 
 		workloadSource := getWorkloadSourceHelmChart(repoRoot, source, gitRepo, helmChart)
 
 		for i := range ws {
 			workloadIDToSource[ws[i].GetID()] = workloadSource
-			workloadIDToNodes[ws[i].GetID()] = templatesNodes
 		}
 	}
 
@@ -342,7 +327,7 @@ func getResourcesFromPath(ctx context.Context, path string) (map[string]reportha
 		}
 	}
 
-	return workloadIDToSource, workloads, workloadIDToNodes, nil
+	return workloadIDToSource, workloads, nil
 }
 
 func extractGitRepo(path string) (string, *cautils.LocalGitRepository) {

core/pkg/resultshandling/reporter/v2/reporteventreceiver.go

@@ -162,7 +162,11 @@ func (report *ReportEventReceiver) setResults(reportObj *reporthandlingv2.Postur
 
 		r, err := json.Marshal(v)
 		if err != nil {
-			return fmt.Errorf("failed to unmarshal resource '%s', reason: %v", v.GetResourceID(), err)
+			logger.L().Error("failed to marshal resource to JSON - skipping",
+				helpers.Error(err),
+				helpers.String("file", resource.GetSource().RelativePath),
+			)
+			continue
 		}
 
 		if *counter+len(r) >= MAX_REPORT_SIZE && len(reportObj.Results) > 0 {
@@ -204,7 +208,11 @@ func (report *ReportEventReceiver) setResources(reportObj *reporthandlingv2.Post
 		}
 		r, err := json.Marshal(resource)
 		if err != nil {
-			return fmt.Errorf("failed to unmarshal resource '%s', reason: %v", resourceID, err)
+			logger.L().Error("failed to marshal resource to JSON - skipping",
+				helpers.Error(err),
+				helpers.String("file", resource.GetSource().RelativePath),
+			)
+			continue
 		}
 
 		if *counter+len(r) >= MAX_REPORT_SIZE && len(reportObj.Resources) > 0 {

httphandler/go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/kubescape/go-logger v0.0.23
 	github.com/kubescape/k8s-interface v0.0.174
 	github.com/kubescape/kubescape/v3 v3.0.4
-	github.com/kubescape/opa-utils v0.0.281
+	github.com/kubescape/opa-utils v0.0.282
 	github.com/kubescape/storage v0.0.111
 	github.com/spf13/viper v1.18.2
 	github.com/stretchr/testify v1.9.0

httphandler/go.sum

@@ -1179,8 +1179,8 @@ github.com/kubescape/go-logger v0.0.23 h1:5xh+Nm8eGImhFbtippRKLaFgsvlKE1ufvQhNM2
 github.com/kubescape/go-logger v0.0.23/go.mod h1:Ayg7g769c7sXVB+P3fkJmbsJpoEmMmaUf9jeo+XuC3U=
 github.com/kubescape/k8s-interface v0.0.174 h1:I62Dc0RZmdESJrZLhtv2ZDqvf35XQ8g19T1LQPLARsk=
 github.com/kubescape/k8s-interface v0.0.174/go.mod h1:PAJHWlN7v+VI3jx4v2xH2VqvbN+4bcy7paqXd5/tQ4I=
-github.com/kubescape/opa-utils v0.0.281 h1:NwAjvsriu+xm5L9C+XQwOqb5ttVJz6OVmFCv8mlB1tc=
-github.com/kubescape/opa-utils v0.0.281/go.mod h1:N/UnbZHpoiHQH7O50yadhIXZvVl0IVtTGBmePPrSQSg=
+github.com/kubescape/opa-utils v0.0.282 h1:cE6EwRqQOLPWYlr1/simUmlr3tWA8UMzhj1nn3y8Dqs=
+github.com/kubescape/opa-utils v0.0.282/go.mod h1:N/UnbZHpoiHQH7O50yadhIXZvVl0IVtTGBmePPrSQSg=
 github.com/kubescape/rbac-utils v0.0.21-0.20230806101615-07e36f555520 h1:SqlwF8G+oFazeYmZQKoPczLEflBQpwpHCU8DoLLyfj8=
 github.com/kubescape/rbac-utils v0.0.21-0.20230806101615-07e36f555520/go.mod h1:wuxMUSDzGUyWd25IJfBzEJ/Udmw2Vy7npj+MV3u3GrU=
 github.com/kubescape/regolibrary/v2 v2.0.1 h1:7lKj171gslgTbbcmmGVHk34AZNqxForOXZIINoQfdzQ=