github/kubescape
1
0
Fork 0
mirror of https://github.com/kubescape/kubescape.git synced 2026-02-14 18:09:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix scanning repo

Browse Source 
Signed-off-by: David Wertenteil <dwertent@armosec.io>
This commit is contained in:
David Wertenteil
2024-05-06 16:22:38 +03:00
parent c937ed16f4
commit 3cbd2c458d
12 changed files with 176 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
core/cautils/remotegitutils.go
View File

@@ -1,11 +1,11 @@
package cautils
import (
"crypto/sha256"
"errors"
"fmt"
nethttp "net/http"
"os"
"path/filepath"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
@@ -17,6 +17,40 @@ import (
"github.com/kubescape/go-logger/helpers"
)
var tmpDirPaths map[string]string
func hashRepoURL(repoURL string) string {
h := sha256.New()
h.Write([]byte(repoURL))
return string(h.Sum(nil))
}
func getDirPath(repoURL string) string {
if tmpDirPaths == nil {
return ""
}
return tmpDirPaths[hashRepoURL(repoURL)]
}
// Create a temporary directory this function is called once
func createTempDir(repoURL string) (string, error) {
tmpDirPath := getDirPath(repoURL)
if tmpDirPath != "" {
return tmpDirPath, nil
}
// create temp directory
tmpDir, err := os.MkdirTemp("", "")
if err != nil {
return "", fmt.Errorf("failed to create temporary directory: %w", err)
}
if tmpDirPaths == nil {
tmpDirPaths = make(map[string]string)
}
tmpDirPaths[hashRepoURL(repoURL)] = tmpDir
return tmpDir, nil
}
// To Check if the given repository is Public(No Authentication needed), send a HTTP GET request to the URL
// If response code is 200, the repository is Public.
func isGitRepoPublic(u string) bool {
@@ -58,16 +92,21 @@ func getProviderError(gitURL giturl.IGitAPI) error {
// cloneRepo clones a repository to a local temporary directory and returns the directory
func cloneRepo(gitURL giturl.IGitAPI) (string, error) {
cloneURL := gitURL.GetHttpCloneURL()
// Check if directory exists
if p := getDirPath(cloneURL); p != "" {
// directory exists, meaning this repo was cloned
return p, nil
}
// Get the URL to clone
// Create temp directory
tmpDir, err := os.MkdirTemp("", "")
tmpDir, err := createTempDir(cloneURL)
if err != nil {
return "", fmt.Errorf("failed to create temporary directory: %w", err)
return "", err
}
// Get the URL to clone
cloneURL := gitURL.GetHttpCloneURL()
isGitTokenPresent := isGitTokenPresent(gitURL)
// Declare the authentication variable required for cloneOptions
@@ -104,6 +143,8 @@ func cloneRepo(gitURL giturl.IGitAPI) (string, error) {
if err != nil {
return "", fmt.Errorf("failed to clone %s. %w", gitURL.GetRepoName(), err)
}
// tmpDir = filepath.Join(tmpDir, gitURL.GetRepoName())
tmpDirPaths[hashRepoURL(cloneURL)] = tmpDir
return tmpDir, nil
}
@@ -125,9 +166,19 @@ func CloneGitRepo(path *string) (string, error) {
logger.L().StopError("failed to clone git repo", helpers.String("url", gitURL.GetURL().String()), helpers.Error(err))
return "", fmt.Errorf("failed to clone git repo '%s', %w", gitURL.GetURL().String(), err)
}
*path = clonedDir
*path = filepath.Join(clonedDir, gitURL.GetPath())
logger.L().StopSuccess("Done accessing local objects")
logger.L().StopSuccess("Done accessing remote repo")
return clonedDir, nil
}
func GetClonedPath(path string) string {
gitURL, err := giturl.NewGitAPI(path)
if err != nil {
return ""
}
return getDirPath(gitURL.GetHttpCloneURL())
}

60
core/cautils/remotegitutils_test.go
View File

@@ -93,3 +93,63 @@ func TestCloneRepo(t *testing.T) {
})
}
}
func TestGetClonedPath(t *testing.T) {
testCases := []struct {
name string
path string
expected string
}{
{
name: "Valid Git URL",
path: "https://github.com/kubescape/kubescape.git",
expected: "/path/to/cloned/repo", // replace with the expected path
},
{
name: "Invalid Git URL",
path: "invalid",
expected: "",
},
}
tmpDirPaths = make(map[string]string)
tmpDirPaths[hashRepoURL("https://github.com/kubescape/kubescape.git")] = "/path/to/cloned/repo" // replace with the actual path
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
result := GetClonedPath(tc.path)
if result != tc.expected {
t.Errorf("Expected %q, got %q", tc.expected, result)
}
})
}
}
func TestGetDirPath(t *testing.T) {
testCases := []struct {
name string
repoURL string
expected string
}{
{
name: "Existing Repo URL",
repoURL: "https://github.com/user/repo.git",
expected: "/path/to/cloned/repo", // replace with the expected path
},
{
name: "Non-Existing Repo URL",
repoURL: "https://github.com/user/nonexistentrepo.git",
expected: "",
},
}
// Initialize tmpDirPaths
tmpDirPaths = make(map[string]string)
tmpDirPaths[hashRepoURL("https://github.com/user/repo.git")] = "/path/to/cloned/repo" // replace with the actual path
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
result := getDirPath(tc.repoURL)
if result != tc.expected {
t.Errorf("Expected %q, got %q", tc.expected, result)
}
})
}
}

22
core/cautils/scaninfo.go
View File

@@ -25,10 +25,11 @@ import (
type ScanningContext string
const (
ContextCluster ScanningContext = "cluster"
ContextFile ScanningContext = "single-file"
ContextDir ScanningContext = "local-dir"
ContextGitLocal ScanningContext = "git-local"
ContextCluster ScanningContext = "cluster"
ContextFile ScanningContext = "single-file"
ContextDir ScanningContext = "local-dir"
ContextGitLocal ScanningContext = "git-local"
ContextGitRemote ScanningContext = "git-remote"
)
const ( // deprecated
@@ -281,6 +282,9 @@ func scanInfoToScanMetadata(ctx context.Context, scanInfo *ScanInfo) *reporthand
case ContextGitLocal:
// local-git
metadata.ScanMetadata.ScanningTarget = reporthandlingv2.GitLocal
case ContextGitRemote:
// remote
metadata.ScanMetadata.ScanningTarget = reporthandlingv2.Repo
case ContextDir:
// directory
metadata.ScanMetadata.ScanningTarget = reporthandlingv2.Directory
@@ -308,6 +312,7 @@ func (scanInfo *ScanInfo) GetScanningContext() ScanningContext {
}
// getScanningContext get scanning context from the input param
// this function should be called only once. Call GetScanningContext() to get the scanning context
func (scanInfo *ScanInfo) getScanningContext(input string) ScanningContext {
// cluster
if input == "" {
@@ -321,7 +326,7 @@ func (scanInfo *ScanInfo) getScanningContext(input string) ScanningContext {
scanInfo.cleanups = append(scanInfo.cleanups, func() {
_ = os.RemoveAll(repo)
})
return ContextGitLocal
return ContextGitRemote
}
}
}
@@ -389,6 +394,13 @@ func (scanInfo *ScanInfo) setContextMetadata(ctx context.Context, contextMetadat
logger.L().Ctx(ctx).Warning("in setContextMetadata", helpers.Interface("case", ContextGitLocal), helpers.Error(err))
}
contextMetadata.RepoContextMetadata = repoContext
case ContextGitRemote:
// remote
repoContext, err := metadataGitLocal(GetClonedPath(input))
if err != nil {
logger.L().Ctx(ctx).Warning("in setContextMetadata", helpers.Interface("case", ContextGitRemote), helpers.Error(err))
}
contextMetadata.RepoContextMetadata = repoContext
}
}

2
core/cautils/scaninfo_test.go
View File

@@ -71,7 +71,7 @@ func TestGetScanningContext(t *testing.T) {
{
name: "git URL input",
input: "https://github.com/kubescape/http-request",
want: ContextGitLocal,
want: ContextGitRemote,
},
{
name: "local git input",

35
core/pkg/resourcehandler/filesloader.go
View File

@@ -3,7 +3,6 @@ package resourcehandler
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
@@ -45,7 +44,7 @@ func (fileHandler *FileResourceHandler) GetResources(ctx context.Context, sessio
var err error
if scanInfo.ChartPath != "" && scanInfo.FilePath != "" {
workloadIDToSource, workloads, workloadIDToMappingNodes, err = getWorkloadFromHelmChart(ctx, scanInfo.ChartPath, scanInfo.FilePath)
workloadIDToSource, workloads, workloadIDToMappingNodes, err = getWorkloadFromHelmChart(ctx, scanInfo.InputPatterns[path], scanInfo.ChartPath, scanInfo.FilePath)
if err != nil {
// We should probably ignore the error so we can continue scanning other charts
}
@@ -107,26 +106,22 @@ func (fileHandler *FileResourceHandler) GetResources(ctx context.Context, sessio
func (fileHandler *FileResourceHandler) GetCloudProvider() string {
return ""
}
func getWorkloadFromHelmChart(ctx context.Context, helmPath, workloadPath string) (map[string]reporthandling.Source, []workloadinterface.IMetadata, map[string]cautils.MappingNodes, error) {
clonedRepo, err := cautils.CloneGitRepo(&helmPath)
if err != nil {
return nil, nil, nil, err
}
func getWorkloadFromHelmChart(ctx context.Context, path, helmPath, workloadPath string) (map[string]reporthandling.Source, []workloadinterface.IMetadata, map[string]cautils.MappingNodes, error) {
clonedRepo := cautils.GetClonedPath(path)
if clonedRepo != "" {
defer func(path string) {
_ = os.RemoveAll(path)
}(clonedRepo)
// if the repo was cloned, add the workload path to the cloned repo
workloadPath = filepath.Join(clonedRepo, workloadPath)
} else {
// if the repo was not cloned
clonedRepo = path
}
// Get repo root
repoRoot, gitRepo := extractGitRepo(helmPath)
repoRoot, gitRepo := extractGitRepo(clonedRepo)
helmSourceToWorkloads, helmSourceToChart, helmSourceToNodes := cautils.LoadResourcesFromHelmCharts(ctx, helmPath)
if clonedRepo != "" {
workloadPath = clonedRepo + workloadPath
}
wlSource, ok := helmSourceToWorkloads[workloadPath]
if !ok {
return nil, nil, nil, fmt.Errorf("workload %s not found in chart %s", workloadPath, helmPath)
@@ -195,14 +190,10 @@ func getResourcesFromPath(ctx context.Context, path string) (map[string]reportha
workloadIDToNodes := make(map[string]cautils.MappingNodes)
var workloads []workloadinterface.IMetadata
clonedRepo, err := cautils.CloneGitRepo(&path)
if err != nil {
return nil, nil, nil, err
}
clonedRepo := cautils.GetClonedPath(path)
if clonedRepo != "" {
defer func(path string) {
_ = os.RemoveAll(path)
}(clonedRepo)
// if the repo was cloned, add the workload path to the cloned repo
path = clonedRepo
}
// Get repo root

6
core/pkg/resultshandling/printer/v2/prettyprinter/reposcan.go
View File

@@ -3,6 +3,7 @@ package prettyprinter
import (
"fmt"
"os"
"path/filepath"
"github.com/kubescape/kubescape/v3/core/cautils"
"github.com/kubescape/kubescape/v3/core/pkg/resultshandling/printer/v2/prettyprinter/tableprinter/configurationprinter"
@@ -77,9 +78,8 @@ func (rp *RepoPrinter) getWorkloadScanCommand(ns, kind, name string, source repo
}
if source.FileType == reporthandling.SourceTypeHelmChart {
return fmt.Sprintf("%s --chart-path=%s --file-path=%s", cmd, source.HelmPath, fmt.Sprintf("%s/%s", source.Path, source.RelativePath))
return fmt.Sprintf("%s --chart-path=%s --file-path=%s", cmd, source.HelmPath, filepath.Join(source.Path, source.RelativePath))
} else {
return fmt.Sprintf("%s --file-path=%s", cmd, fmt.Sprintf("%s/%s", source.Path, source.RelativePath))
return fmt.Sprintf("%s --file-path=%s", cmd, filepath.Join(source.Path, source.RelativePath))
}
}

20
core/pkg/resultshandling/printer/v2/prettyprinter/reposcan_test.go
View File

@@ -1,6 +1,7 @@
package prettyprinter
import (
"path/filepath"
"testing"
"github.com/kubescape/opa-utils/reporthandling"
@@ -50,7 +51,18 @@ func TestRepoScan_getWorkloadScanCommand(t *testing.T) {
Path: "path",
RelativePath: "relativePath",
},
want: "$ kubescape scan workload kind/name --namespace ns --file-path=path/relativePath",
want: "$ kubescape scan workload kind/name --namespace ns --file-path=" + filepath.Join("path", "relativePath"),
},
{
testName: "relative file path",
ns: "ns",
kind: "kind",
name: "name",
source: reporthandling.Source{
Path: "",
RelativePath: "relativePath",
},
want: "$ kubescape scan workload kind/name --namespace ns --file-path=relativePath",
},
{
testName: "helm path",
@@ -63,7 +75,7 @@ func TestRepoScan_getWorkloadScanCommand(t *testing.T) {
HelmPath: "helmPath",
FileType: "Helm Chart",
},
want: "$ kubescape scan workload kind/name --namespace ns --chart-path=helmPath --file-path=path/relativePath",
want: "$ kubescape scan workload kind/name --namespace ns --chart-path=helmPath --file-path=" + filepath.Join("path", "relativePath"),
},
{
testName: "file path - no namespace",
@@ -73,7 +85,7 @@ func TestRepoScan_getWorkloadScanCommand(t *testing.T) {
Path: "path",
RelativePath: "relativePath",
},
want: "$ kubescape scan workload kind/name --file-path=path/relativePath",
want: "$ kubescape scan workload kind/name --file-path=" + filepath.Join("path", "relativePath"),
},
{
testName: "helm path - no namespace",
@@ -85,7 +97,7 @@ func TestRepoScan_getWorkloadScanCommand(t *testing.T) {
HelmPath: "helmPath",
FileType: "Helm Chart",
},
want: "$ kubescape scan workload kind/name --chart-path=helmPath --file-path=path/relativePath",
want: "$ kubescape scan workload kind/name --chart-path=helmPath --file-path=" + filepath.Join("path", "relativePath"),
},
}

2
core/pkg/resultshandling/results.go
View File

@@ -146,7 +146,7 @@ func ValidatePrinter(scanType cautils.ScanTypes, scanContext cautils.ScanningCon
if printFormat == printer.SARIFFormat {
// supported types for SARIF
switch scanContext {
case cautils.ContextDir, cautils.ContextFile, cautils.ContextGitLocal:
case cautils.ContextDir, cautils.ContextFile, cautils.ContextGitLocal, cautils.ContextGitRemote:
return nil
default:
return fmt.Errorf("format \"%s\" is only supported when scanning local files", printFormat)

2
go.mod
View File

@@ -27,7 +27,7 @@ require (
github.com/kubescape/backend v0.0.20
github.com/kubescape/go-git-url v0.0.30
github.com/kubescape/go-logger v0.0.22
github.com/kubescape/k8s-interface v0.0.161
github.com/kubescape/k8s-interface v0.0.166
github.com/kubescape/opa-utils v0.0.281
github.com/kubescape/rbac-utils v0.0.21-0.20230806101615-07e36f555520
github.com/kubescape/regolibrary/v2 v2.0.1

4
go.sum
View File

@@ -1149,8 +1149,8 @@ github.com/kubescape/go-git-url v0.0.30 h1:PIbg86ae0ftee/p/Tu/6CA1ju6VoJ51G3sQWN
github.com/kubescape/go-git-url v0.0.30/go.mod h1:3ddc1HEflms1vMhD9owt/3FBES070UaYTUarcjx8jDk=
github.com/kubescape/go-logger v0.0.22 h1:gle7wH6emOiGv9ljdpVi82pWLQ3jGucrUucvil6JXHE=
github.com/kubescape/go-logger v0.0.22/go.mod h1:x3HBpZo3cMT/WIdy18BxvVVd5D0e/PWFVk/HiwBNu3g=
github.com/kubescape/k8s-interface v0.0.161 h1:v6b3/kmA4o/2niNrejrbXj5X9MLfH0UrpI3s+e/fdwc=
github.com/kubescape/k8s-interface v0.0.161/go.mod h1:oF+Yxug3Kpfu9Yr2j63wy7gwswrKXpiqI0mLk/7gF/s=
github.com/kubescape/k8s-interface v0.0.166 h1:n9rG8vUvHw7UMh+/PZHadfK4Mcj++WAaj+rvNc8kbJY=
github.com/kubescape/k8s-interface v0.0.166/go.mod h1:oF+Yxug3Kpfu9Yr2j63wy7gwswrKXpiqI0mLk/7gF/s=
github.com/kubescape/opa-utils v0.0.281 h1:NwAjvsriu+xm5L9C+XQwOqb5ttVJz6OVmFCv8mlB1tc=
github.com/kubescape/opa-utils v0.0.281/go.mod h1:N/UnbZHpoiHQH7O50yadhIXZvVl0IVtTGBmePPrSQSg=
github.com/kubescape/rbac-utils v0.0.21-0.20230806101615-07e36f555520 h1:SqlwF8G+oFazeYmZQKoPczLEflBQpwpHCU8DoLLyfj8=

2
httphandler/go.mod
View File

@@ -16,7 +16,7 @@ require (
github.com/gorilla/schema v1.2.0
github.com/kubescape/backend v0.0.20
github.com/kubescape/go-logger v0.0.22
github.com/kubescape/k8s-interface v0.0.161
github.com/kubescape/k8s-interface v0.0.166
github.com/kubescape/kubescape/v3 v3.0.4
github.com/kubescape/opa-utils v0.0.281
github.com/kubescape/storage v0.0.20

4
httphandler/go.sum
View File

@@ -1155,8 +1155,8 @@ github.com/kubescape/go-git-url v0.0.30 h1:PIbg86ae0ftee/p/Tu/6CA1ju6VoJ51G3sQWN
github.com/kubescape/go-git-url v0.0.30/go.mod h1:3ddc1HEflms1vMhD9owt/3FBES070UaYTUarcjx8jDk=
github.com/kubescape/go-logger v0.0.22 h1:gle7wH6emOiGv9ljdpVi82pWLQ3jGucrUucvil6JXHE=
github.com/kubescape/go-logger v0.0.22/go.mod h1:x3HBpZo3cMT/WIdy18BxvVVd5D0e/PWFVk/HiwBNu3g=
github.com/kubescape/k8s-interface v0.0.161 h1:v6b3/kmA4o/2niNrejrbXj5X9MLfH0UrpI3s+e/fdwc=
github.com/kubescape/k8s-interface v0.0.161/go.mod h1:oF+Yxug3Kpfu9Yr2j63wy7gwswrKXpiqI0mLk/7gF/s=
github.com/kubescape/k8s-interface v0.0.166 h1:n9rG8vUvHw7UMh+/PZHadfK4Mcj++WAaj+rvNc8kbJY=
github.com/kubescape/k8s-interface v0.0.166/go.mod h1:oF+Yxug3Kpfu9Yr2j63wy7gwswrKXpiqI0mLk/7gF/s=
github.com/kubescape/opa-utils v0.0.281 h1:NwAjvsriu+xm5L9C+XQwOqb5ttVJz6OVmFCv8mlB1tc=
github.com/kubescape/opa-utils v0.0.281/go.mod h1:N/UnbZHpoiHQH7O50yadhIXZvVl0IVtTGBmePPrSQSg=
github.com/kubescape/rbac-utils v0.0.21-0.20230806101615-07e36f555520 h1:SqlwF8G+oFazeYmZQKoPczLEflBQpwpHCU8DoLLyfj8=