diff --git a/core/cautils/fileutils.go b/core/cautils/fileutils.go index b3ca155c..dbed9393 100644 --- a/core/cautils/fileutils.go +++ b/core/cautils/fileutils.go @@ -15,8 +15,8 @@ import ( ) var ( - YAML_PREFIX = []string{".yaml", ".yml"} - JSON_PREFIX = []string{".json"} + YAML_PREFIX = []string{"yaml", "yml"} + JSON_PREFIX = []string{"json"} ) type FileFormat string @@ -78,7 +78,6 @@ func ReadFile(fileContent []byte, fileFromat FileFormat) ([]workloadinterface.IM default: return nil, nil // []error{fmt.Errorf("file extension %s not supported", fileFromat)} } - } func listFiles(patterns []string) ([]string, []error) { @@ -178,11 +177,11 @@ func convertYamlToJson(i interface{}) interface{} { } func IsYaml(filePath string) bool { - return StringInSlice(YAML_PREFIX, filepath.Ext(filePath)) != ValueNotFound + return StringInSlice(YAML_PREFIX, strings.ReplaceAll(filepath.Ext(filePath), ".", "")) != ValueNotFound } func IsJson(filePath string) bool { - return StringInSlice(JSON_PREFIX, filepath.Ext(filePath)) != ValueNotFound + return StringInSlice(JSON_PREFIX, strings.ReplaceAll(filepath.Ext(filePath), ".", "")) != ValueNotFound } func glob(root, pattern string) ([]string, error) { diff --git a/core/cautils/scaninfo.go b/core/cautils/scaninfo.go index 70b99bef..a89d7952 100644 --- a/core/cautils/scaninfo.go +++ b/core/cautils/scaninfo.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" + giturl "github.com/armosec/go-git-url" "github.com/armosec/k8s-interface/k8sinterface" "github.com/armosec/kubescape/v2/core/cautils/getter" "github.com/armosec/kubescape/v2/core/cautils/logger" @@ -240,7 +241,7 @@ func scanInfoToScanMetadata(scanInfo *ScanInfo) *reporthandlingv2.Metadata { } func setContextMetadata(contextMetadata *reporthandlingv2.ContextMetadata, input string) { - // if cluster + // cluster if input == "" { contextMetadata.ClusterContextMetadata = &reporthandlingv2.ClusterMetadata{ ContextName: k8sinterface.GetContextName(), @@ -248,8 +249,16 @@ func setContextMetadata(contextMetadata *reporthandlingv2.ContextMetadata, input return } - // if url - if strings.HasPrefix(input, "http") { // TODO - check if can parse + // url + if gitParser, err := giturl.NewGitURL(input); err == nil { + if gitParser.GetBranch() == "" { + gitParser.SetDefaultBranch() + } + contextMetadata.RepoContextMetadata = &reporthandlingv2.RepoContextMetadata{ + Repo: gitParser.GetRepo(), + Owner: gitParser.GetOwner(), + Branch: gitParser.GetBranch(), + } return } @@ -259,7 +268,7 @@ func setContextMetadata(contextMetadata *reporthandlingv2.ContextMetadata, input } } - // if single file + // single file if IsFile(input) { contextMetadata.FileContextMetadata = &reporthandlingv2.FileContextMetadata{ FilePath: input, @@ -268,7 +277,7 @@ func setContextMetadata(contextMetadata *reporthandlingv2.ContextMetadata, input return } - // if dir/glob + // dir/glob if !IsFile(input) { contextMetadata.DirectoryContextMetadata = &reporthandlingv2.DirectoryContextMetadata{ BasePath: input, diff --git a/core/cautils/scaninfo_test.go b/core/cautils/scaninfo_test.go index 7f629734..c1d6323d 100644 --- a/core/cautils/scaninfo_test.go +++ b/core/cautils/scaninfo_test.go @@ -7,16 +7,6 @@ import ( "github.com/stretchr/testify/assert" ) -// func TestSetInputPatterns(t *testing.T) { //Unitest -// { -// scanInfo := ScanInfo{ -// InputPatterns: []string{"file"}, -// } -// scanInfo.setInputPatterns() -// assert.Equal(t, "file", scanInfo.InputPatterns[0]) -// } -// } - func TestSetContextMetadata(t *testing.T) { { ctx := reporthandlingv2.ContextMetadata{} @@ -37,6 +27,10 @@ func TestSetContextMetadata(t *testing.T) { assert.Nil(t, ctx.FileContextMetadata) assert.Nil(t, ctx.HelmContextMetadata) assert.Nil(t, ctx.RepoContextMetadata) + + hostName := getHostname() + assert.Contains(t, ctx.DirectoryContextMetadata.BasePath, "file") + assert.Equal(t, hostName, ctx.DirectoryContextMetadata.HostName) } { ctx := reporthandlingv2.ContextMetadata{} @@ -47,6 +41,10 @@ func TestSetContextMetadata(t *testing.T) { assert.NotNil(t, ctx.FileContextMetadata) assert.Nil(t, ctx.HelmContextMetadata) assert.Nil(t, ctx.RepoContextMetadata) + + hostName := getHostname() + assert.Contains(t, ctx.FileContextMetadata.FilePath, "scaninfo_test.go") + assert.Equal(t, hostName, ctx.FileContextMetadata.HostName) } { ctx := reporthandlingv2.ContextMetadata{} @@ -56,7 +54,11 @@ func TestSetContextMetadata(t *testing.T) { assert.Nil(t, ctx.DirectoryContextMetadata) assert.Nil(t, ctx.FileContextMetadata) assert.Nil(t, ctx.HelmContextMetadata) - assert.Nil(t, ctx.RepoContextMetadata) // TODO + assert.NotNil(t, ctx.RepoContextMetadata) + + assert.Equal(t, "kubescape", ctx.RepoContextMetadata.Repo) + assert.Equal(t, "armosec", ctx.RepoContextMetadata.Owner) + assert.Equal(t, "master", ctx.RepoContextMetadata.Branch) } } diff --git a/core/pkg/resourcehandler/k8sresourcesutils.go b/core/pkg/resourcehandler/k8sresourcesutils.go index db6677fe..baaa187e 100644 --- a/core/pkg/resourcehandler/k8sresourcesutils.go +++ b/core/pkg/resourcehandler/k8sresourcesutils.go @@ -50,19 +50,6 @@ func setArmoResourceMap(frameworks []reporthandling.Framework, resourceToControl return &armoResources } -func convertComplexResourceMap(frameworks []reporthandling.Framework) map[string]map[string]map[string]interface{} { - k8sResources := make(map[string]map[string]map[string]interface{}) - for _, framework := range frameworks { - for _, control := range framework.Controls { - for _, rule := range control.Rules { - for _, match := range rule.Match { - insertResources(k8sResources, match) - } - } - } - } - return k8sResources -} func setComplexK8sResourceMap(frameworks []reporthandling.Framework) map[string]map[string]map[string]interface{} { k8sResources := make(map[string]map[string]map[string]interface{}) for _, framework := range frameworks { diff --git a/core/pkg/resourcehandler/urlloader.go b/core/pkg/resourcehandler/urlloader.go index 34a404cb..e4238de9 100644 --- a/core/pkg/resourcehandler/urlloader.go +++ b/core/pkg/resourcehandler/urlloader.go @@ -1,82 +1,48 @@ package resourcehandler import ( - "bytes" - "fmt" - "io" - "net/http" - "strings" - + giturl "github.com/armosec/go-git-url" "github.com/armosec/k8s-interface/workloadinterface" "github.com/armosec/kubescape/v2/core/cautils" "github.com/armosec/kubescape/v2/core/cautils/logger" + "github.com/armosec/kubescape/v2/core/cautils/logger/helpers" ) func loadResourcesFromUrl(inputPatterns []string) (map[string][]workloadinterface.IMetadata, error) { - urls := listUrls(inputPatterns) - if len(urls) == 0 { + if len(inputPatterns) == 0 { + return nil, nil + } + g, err := giturl.NewGitURL(inputPatterns[0]) + if err != nil { return nil, nil } - workloads, errs := downloadFiles(urls) + files, errs := g.DownloadFilesWithExtension(append(cautils.YAML_PREFIX, cautils.JSON_PREFIX...)) if len(errs) > 0 { - logger.L().Error(fmt.Sprintf("%v", errs)) - } - return workloads, nil -} - -func listUrls(patterns []string) []string { - urls := []string{} - for i := range patterns { - if strings.HasPrefix(patterns[i], "http") { - if yamls, err := ScanRepository(patterns[i], ""); err == nil { // TODO - support branch - urls = append(urls, yamls...) - } else { - logger.L().Error(err.Error()) - } + for i, j := range errs { + logger.L().Error(i, helpers.Error(j)) } } - return urls -} + if len(files) == 0 { + return nil, nil + } -func downloadFiles(urls []string) (map[string][]workloadinterface.IMetadata, []error) { + // convert files to IMetadata workloads := make(map[string][]workloadinterface.IMetadata, 0) - errs := []error{} - for i := range urls { - f, err := downloadFile(urls[i]) - if err != nil { - errs = append(errs, err) + + for i, j := range files { + w, e := cautils.ReadFile(j, cautils.GetFileFormat(i)) + if len(e) != 0 || len(w) == 0 { continue } - w, e := cautils.ReadFile(f, cautils.GetFileFormat(urls[i])) - errs = append(errs, e...) - if w != nil { - if _, ok := workloads[urls[i]]; !ok { - workloads[urls[i]] = make([]workloadinterface.IMetadata, 0) - } - wSlice := workloads[urls[i]] - wSlice = append(wSlice, w...) - workloads[urls[i]] = wSlice + if _, ok := workloads[i]; !ok { + workloads[i] = make([]workloadinterface.IMetadata, 0) } + wSlice := workloads[i] + wSlice = append(wSlice, w...) + workloads[i] = wSlice } - return workloads, errs -} -func downloadFile(url string) ([]byte, error) { - resp, err := http.Get(url) - if err != nil { - return nil, err - } - defer resp.Body.Close() - if resp.StatusCode < 200 || 301 < resp.StatusCode { - return nil, fmt.Errorf("failed to download file, url: '%s', status code: %s", url, resp.Status) - } - return streamToByte(resp.Body), nil -} - -func streamToByte(stream io.Reader) []byte { - buf := new(bytes.Buffer) - buf.ReadFrom(stream) - return buf.Bytes() + return workloads, nil } diff --git a/go.mod b/go.mod index 3536f904..a1a49d2f 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.17 require ( github.com/armosec/armoapi-go v0.0.66 + github.com/armosec/go-git-url v0.0.4 github.com/armosec/k8s-interface v0.0.69 github.com/armosec/opa-utils v0.0.130 github.com/armosec/rbac-utils v0.0.14 diff --git a/go.sum b/go.sum index be3bb68b..75056c28 100644 --- a/go.sum +++ b/go.sum @@ -116,6 +116,8 @@ github.com/armosec/armoapi-go v0.0.23/go.mod h1:iaVVGyc23QGGzAdv4n+szGQg3Rbpixn9 github.com/armosec/armoapi-go v0.0.58/go.mod h1:U/Axd+D5N00x9Ekr7t+5HXqLCMO+98NfJSVAggqJftI= github.com/armosec/armoapi-go v0.0.66 h1:SEm4nTwtexlMqYQr7sj3rWIrDYZj3BQ76FvKLA8hiLo= github.com/armosec/armoapi-go v0.0.66/go.mod h1:U/Axd+D5N00x9Ekr7t+5HXqLCMO+98NfJSVAggqJftI= +github.com/armosec/go-git-url v0.0.4 h1:emG9Yfl53rHpuX41fXLD92ehzhRoNSSnGT6Pr7ogWMY= +github.com/armosec/go-git-url v0.0.4/go.mod h1:PJqdEyJyFxTQvawBcyOM0Ies6+uezire5gpwfr1XX5M= github.com/armosec/k8s-interface v0.0.8/go.mod h1:xxS+V5QT3gVQTwZyAMMDrYLWGrfKOpiJ7Jfhfa0w9sM= github.com/armosec/k8s-interface v0.0.37/go.mod h1:vHxGWqD/uh6+GQb9Sqv7OGMs+Rvc2dsFVc0XtgRh1ZU= github.com/armosec/k8s-interface v0.0.66/go.mod h1:vwprS8qn/iowd5yf0JHpqDsLA5I8W2muqX9AxKhkb0Q=