mirror of
https://github.com/kubescape/kubescape.git
synced 2026-04-15 06:58:11 +00:00
fixed url scanning
This commit is contained in:
@@ -88,6 +88,7 @@ require (
|
||||
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
|
||||
github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/whilp/git-urls v1.0.0 // indirect
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
|
||||
github.com/yashtewari/glob-intersection v0.0.0-20180916065949-5c77d914dd0b // indirect
|
||||
|
||||
@@ -774,6 +774,8 @@ github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqri
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
|
||||
github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
|
||||
github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU=
|
||||
github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
|
||||
|
||||
@@ -26,7 +26,7 @@ const (
|
||||
JSON_FILE_FORMAT FileFormat = "json"
|
||||
)
|
||||
|
||||
func LoadResourcesFromFiles(inputPatterns []string) ([]workloadinterface.IMetadata, error) {
|
||||
func LoadResourcesFromFiles(inputPatterns []string) (map[string][]workloadinterface.IMetadata, error) {
|
||||
files, errs := listFiles(inputPatterns)
|
||||
if len(errs) > 0 {
|
||||
logger.L().Error(fmt.Sprintf("%v", errs))
|
||||
@@ -42,8 +42,8 @@ func LoadResourcesFromFiles(inputPatterns []string) ([]workloadinterface.IMetada
|
||||
return workloads, nil
|
||||
}
|
||||
|
||||
func loadFiles(filePaths []string) ([]workloadinterface.IMetadata, []error) {
|
||||
workloads := []workloadinterface.IMetadata{}
|
||||
func loadFiles(filePaths []string) (map[string][]workloadinterface.IMetadata, []error) {
|
||||
workloads := make(map[string][]workloadinterface.IMetadata, 0)
|
||||
errs := []error{}
|
||||
for i := range filePaths {
|
||||
f, err := loadFile(filePaths[i])
|
||||
@@ -54,7 +54,12 @@ func loadFiles(filePaths []string) ([]workloadinterface.IMetadata, []error) {
|
||||
w, e := ReadFile(f, GetFileFormat(filePaths[i]))
|
||||
errs = append(errs, e...)
|
||||
if w != nil {
|
||||
workloads = append(workloads, w...)
|
||||
if _, ok := workloads[filePaths[i]]; !ok {
|
||||
workloads[filePaths[i]] = []workloadinterface.IMetadata{}
|
||||
}
|
||||
wSlice := workloads[filePaths[i]]
|
||||
wSlice = append(wSlice, w...)
|
||||
workloads[filePaths[i]] = wSlice
|
||||
}
|
||||
}
|
||||
return workloads, errs
|
||||
|
||||
@@ -23,6 +23,20 @@ func TestListFiles(t *testing.T) {
|
||||
assert.Equal(t, 12, len(files))
|
||||
}
|
||||
|
||||
func TestLoadResourcesFromFiles(t *testing.T) {
|
||||
workloads, err := LoadResourcesFromFiles([]string{onlineBoutiquePath()})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 12, len(workloads))
|
||||
|
||||
for i, w := range workloads {
|
||||
switch filepath.Base(i) {
|
||||
case "adservice.yaml":
|
||||
assert.Equal(t, 2, len(w))
|
||||
assert.Equal(t, "apps/v1//Deployment/adservice", w[0].GetID())
|
||||
assert.Equal(t, "/v1//Service/adservice", w[1].GetID())
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestLoadFiles(t *testing.T) {
|
||||
files, _ := listFiles([]string{onlineBoutiquePath()})
|
||||
_, err := loadFiles(files)
|
||||
|
||||
@@ -19,6 +19,7 @@ require (
|
||||
github.com/olekukonko/tablewriter v0.0.5
|
||||
github.com/open-policy-agent/opa v0.38.0
|
||||
github.com/stretchr/testify v1.7.0
|
||||
github.com/whilp/git-urls v1.0.0
|
||||
go.uber.org/zap v1.21.0
|
||||
golang.org/x/mod v0.5.1
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
|
||||
@@ -771,6 +771,8 @@ github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqri
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
|
||||
github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
|
||||
github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU=
|
||||
github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
|
||||
|
||||
@@ -32,6 +32,7 @@ func (fileHandler *FileResourceHandler) GetResources(sessionObj *cautils.OPASess
|
||||
// map resources based on framework required resources: map["/group/version/kind"][]<k8s workloads ids>
|
||||
k8sResources := setK8sResourceMap(sessionObj.Policies)
|
||||
allResources := map[string]workloadinterface.IMetadata{}
|
||||
workloadIDToSource := make(map[string]string, 0)
|
||||
armoResources := &cautils.ArmoResources{}
|
||||
|
||||
workloads := []workloadinterface.IMetadata{}
|
||||
@@ -41,8 +42,11 @@ func (fileHandler *FileResourceHandler) GetResources(sessionObj *cautils.OPASess
|
||||
if err != nil {
|
||||
return nil, allResources, nil, err
|
||||
}
|
||||
if w != nil {
|
||||
workloads = append(workloads, w...)
|
||||
for source, ws := range w {
|
||||
workloads = append(workloads, ws...)
|
||||
for i := range ws {
|
||||
workloadIDToSource[ws[i].GetID()] = source
|
||||
}
|
||||
}
|
||||
|
||||
// load resources from url
|
||||
@@ -50,8 +54,11 @@ func (fileHandler *FileResourceHandler) GetResources(sessionObj *cautils.OPASess
|
||||
if err != nil {
|
||||
return nil, allResources, nil, err
|
||||
}
|
||||
if w != nil {
|
||||
workloads = append(workloads, w...)
|
||||
for source, ws := range w {
|
||||
workloads = append(workloads, ws...)
|
||||
for i := range ws {
|
||||
workloadIDToSource[ws[i].GetID()] = source
|
||||
}
|
||||
}
|
||||
|
||||
if len(workloads) == 0 {
|
||||
|
||||
@@ -4,15 +4,24 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/armosec/kubescape/core/cautils/getter"
|
||||
giturls "github.com/whilp/git-urls"
|
||||
"k8s.io/utils/strings/slices"
|
||||
)
|
||||
|
||||
type IRepository interface {
|
||||
parse(fullURL string) error
|
||||
|
||||
setBranch(string) error
|
||||
setTree() error
|
||||
getYamlFromTree() []string
|
||||
|
||||
getBranch() string
|
||||
getTree() tree
|
||||
|
||||
getFilesFromTree([]string) []string
|
||||
}
|
||||
|
||||
type innerTree struct {
|
||||
@@ -23,19 +32,23 @@ type tree struct {
|
||||
}
|
||||
|
||||
type GitHubRepository struct {
|
||||
// name string // <org>/<repo>
|
||||
host string
|
||||
name string // <org>/<repo>
|
||||
owner string //
|
||||
repo string //
|
||||
branch string
|
||||
path string
|
||||
isFile bool
|
||||
tree tree
|
||||
}
|
||||
type githubDefaultBranchAPI struct {
|
||||
DefaultBranch string `json:"default_branch"`
|
||||
}
|
||||
|
||||
func NewGitHubRepository(rep string) *GitHubRepository {
|
||||
func NewGitHubRepository() *GitHubRepository {
|
||||
return &GitHubRepository{
|
||||
host: "github",
|
||||
name: rep,
|
||||
host: "github.com",
|
||||
// name: rep,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,75 +58,132 @@ func ScanRepository(command string, branchOptional string) ([]string, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = repo.setBranch(branchOptional)
|
||||
if err != nil {
|
||||
if err := repo.parse(command); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = repo.setTree()
|
||||
if err != nil {
|
||||
if err := repo.setBranch(branchOptional); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := repo.setTree(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// get all paths that are of the yaml type, and build them into a valid url
|
||||
return repo.getYamlFromTree(), nil
|
||||
return repo.getFilesFromTree([]string{"yaml", "yml", "json"}), nil
|
||||
}
|
||||
|
||||
func getHostAndRepoName(url string) (string, string, error) {
|
||||
splitUrl := strings.Split(url, "/")
|
||||
|
||||
if len(splitUrl) != 5 {
|
||||
return "", "", fmt.Errorf("failed to pars url: %s", url)
|
||||
func getHost(fullURL string) (string, error) {
|
||||
parsedURL, err := giturls.Parse(fullURL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
hostUrl := splitUrl[2] // github.com, gitlab.com, etc.
|
||||
repository := splitUrl[3] + "/" + strings.Split(splitUrl[4], ".")[0] // user/reposetory
|
||||
|
||||
return hostUrl, repository, nil
|
||||
return parsedURL.Host, nil
|
||||
}
|
||||
|
||||
func getRepository(url string) (IRepository, error) {
|
||||
hostUrl, repoName, err := getHostAndRepoName(url)
|
||||
// func parseHostAndRepoName(fullURL string) (string, string, error) {
|
||||
// parsedURL, err := giturls.Parse(fullURL)
|
||||
// if err != nil {
|
||||
// return "", "", err
|
||||
// }
|
||||
|
||||
// host := parsedURL.Host
|
||||
|
||||
// splittedRepo := strings.FieldsFunc(parsedURL.Path, func(c rune) bool { return c == '/' })
|
||||
// if len(splittedRepo) < 2 {
|
||||
// return "", "", fmt.Errorf("expecting <user>/<repo> in url path, received: '%s'", parsedURL.Path)
|
||||
// }
|
||||
// return host, fmt.Sprintf("%s/%s", splittedRepo[0], splittedRepo[1]), nil
|
||||
// }
|
||||
|
||||
func getRepository(fullURL string) (IRepository, error) {
|
||||
hostUrl, err := getHost(fullURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var repo IRepository
|
||||
switch repoHost := strings.Split(hostUrl, ".")[0]; repoHost {
|
||||
case "github":
|
||||
repo = NewGitHubRepository(repoName)
|
||||
switch hostUrl {
|
||||
case "github.com", "raw.githubusercontent.com":
|
||||
repo = NewGitHubRepository()
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown repository host: %s", repoHost)
|
||||
return nil, fmt.Errorf("unknown repository host: %s", hostUrl)
|
||||
}
|
||||
|
||||
// Returns the host-url, and the part of the user and repository from the url
|
||||
return repo, nil
|
||||
}
|
||||
func (g *GitHubRepository) parse(fullURL string) error {
|
||||
parsedURL, err := giturls.Parse(fullURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
splittedRepo := strings.FieldsFunc(parsedURL.Path, func(c rune) bool { return c == '/' })
|
||||
if len(splittedRepo) < 2 {
|
||||
return fmt.Errorf("expecting <user>/<repo> in url path, received: '%s'", parsedURL.Path)
|
||||
}
|
||||
g.owner = splittedRepo[0]
|
||||
g.repo = splittedRepo[1]
|
||||
|
||||
// root of repo
|
||||
if len(splittedRepo) < 4 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// is file or dir
|
||||
switch splittedRepo[2] {
|
||||
case "blob":
|
||||
g.isFile = true
|
||||
case "tree":
|
||||
g.isFile = false
|
||||
default:
|
||||
// Unknown - failed to parse
|
||||
return nil
|
||||
}
|
||||
g.branch = splittedRepo[3]
|
||||
|
||||
if len(splittedRepo) < 5 {
|
||||
return nil
|
||||
}
|
||||
g.path = strings.Join(splittedRepo[4:], "/")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *GitHubRepository) getBranch() string { return g.branch }
|
||||
func (g *GitHubRepository) getTree() tree { return g.tree }
|
||||
|
||||
func (g *GitHubRepository) setBranch(branchOptional string) error {
|
||||
// Checks whether the repository type is a master or another type.
|
||||
// By default it is "master", unless the branchOptional came with a value
|
||||
if branchOptional == "" {
|
||||
|
||||
body, err := getter.HttpGetter(&http.Client{}, g.defaultBranchAPI(), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var data githubDefaultBranchAPI
|
||||
err = json.Unmarshal([]byte(body), &data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
g.branch = data.DefaultBranch
|
||||
} else {
|
||||
if branchOptional != "" {
|
||||
g.branch = branchOptional
|
||||
}
|
||||
if g.branch != "" {
|
||||
return nil
|
||||
}
|
||||
body, err := getter.HttpGetter(&http.Client{}, g.defaultBranchAPI(), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var data githubDefaultBranchAPI
|
||||
err = json.Unmarshal([]byte(body), &data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
g.branch = data.DefaultBranch
|
||||
return nil
|
||||
}
|
||||
|
||||
func joinOwnerNRepo(owner, repo string) string {
|
||||
return fmt.Sprintf("%s/%s", owner, repo)
|
||||
}
|
||||
func (g *GitHubRepository) defaultBranchAPI() string {
|
||||
return fmt.Sprintf("https://api.github.com/repos/%s", g.name)
|
||||
return fmt.Sprintf("https://api.github.com/repos/%s", joinOwnerNRepo(g.owner, g.repo))
|
||||
}
|
||||
|
||||
func (g *GitHubRepository) setTree() error {
|
||||
@@ -135,14 +205,20 @@ func (g *GitHubRepository) setTree() error {
|
||||
}
|
||||
|
||||
func (g *GitHubRepository) treeAPI() string {
|
||||
return fmt.Sprintf("https://api.github.com/repos/%s/git/trees/%s?recursive=1", g.name, g.branch)
|
||||
return fmt.Sprintf("https://api.github.com/repos/%s/git/trees/%s?recursive=1", joinOwnerNRepo(g.owner, g.repo), g.branch)
|
||||
}
|
||||
|
||||
// return a list of yaml for a given repository tree
|
||||
func (g *GitHubRepository) getYamlFromTree() []string {
|
||||
func (g *GitHubRepository) getFilesFromTree(filesExtensions []string) []string {
|
||||
var urls []string
|
||||
if g.isFile {
|
||||
return []string{fmt.Sprintf("%s/%s", g.rowYamlUrl(), g.path)}
|
||||
}
|
||||
for _, path := range g.tree.InnerTrees {
|
||||
if strings.HasSuffix(path.Path, ".yaml") {
|
||||
if g.path != "" && !strings.HasPrefix(path.Path, g.path) {
|
||||
continue
|
||||
}
|
||||
if slices.Contains(filesExtensions, getFileExtension(path.Path)) {
|
||||
urls = append(urls, fmt.Sprintf("%s/%s", g.rowYamlUrl(), path.Path))
|
||||
}
|
||||
}
|
||||
@@ -150,5 +226,9 @@ func (g *GitHubRepository) getYamlFromTree() []string {
|
||||
}
|
||||
|
||||
func (g *GitHubRepository) rowYamlUrl() string {
|
||||
return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s", g.name, g.branch)
|
||||
return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s", joinOwnerNRepo(g.owner, g.repo), g.branch)
|
||||
}
|
||||
|
||||
func getFileExtension(path string) string {
|
||||
return strings.TrimPrefix(filepath.Ext(path), ".")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
package resourcehandler
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
var (
|
||||
urlA = "https://github.com/armosec/kubescape"
|
||||
urlB = "https://github.com/armosec/kubescape/blob/master/examples/online-boutique/adservice.yaml"
|
||||
urlC = "https://github.com/armosec/kubescape/tree/master/examples/online-boutique"
|
||||
// urlD = "https://raw.githubusercontent.com/armosec/kubescape/master/examples/online-boutique/adservice.yaml"
|
||||
)
|
||||
|
||||
func TestScanRepository(t *testing.T) {
|
||||
{
|
||||
files, err := ScanRepository(urlA, "")
|
||||
assert.NoError(t, err)
|
||||
assert.Less(t, 0, len(files))
|
||||
}
|
||||
{
|
||||
files, err := ScanRepository(urlB, "")
|
||||
assert.NoError(t, err)
|
||||
assert.Less(t, 0, len(files))
|
||||
}
|
||||
{
|
||||
files, err := ScanRepository(urlC, "")
|
||||
assert.NoError(t, err)
|
||||
assert.Less(t, 0, len(files))
|
||||
}
|
||||
// {
|
||||
// files, err := ScanRepository(urlD, "")
|
||||
// assert.NoError(t, err)
|
||||
// assert.Equal(t, 1, len(files))
|
||||
// }
|
||||
|
||||
}
|
||||
|
||||
func TestGetHost(t *testing.T) {
|
||||
{
|
||||
host, err := getHost(urlA)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "github.com", host)
|
||||
}
|
||||
{
|
||||
host, err := getHost(urlB)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "github.com", host)
|
||||
}
|
||||
{
|
||||
host, err := getHost(urlC)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "github.com", host)
|
||||
}
|
||||
}
|
||||
|
||||
// repo = NewGitHubRepository(repoName)
|
||||
|
||||
func TestGithubSetBranch(t *testing.T) {
|
||||
{
|
||||
gh := NewGitHubRepository()
|
||||
assert.NoError(t, gh.parse(urlA))
|
||||
assert.NoError(t, gh.setBranch(""))
|
||||
assert.Equal(t, "master", gh.getBranch())
|
||||
}
|
||||
{
|
||||
gh := NewGitHubRepository()
|
||||
assert.NoError(t, gh.parse(urlB))
|
||||
err := gh.setBranch("dev")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "dev", gh.getBranch())
|
||||
}
|
||||
}
|
||||
|
||||
func TestGithubSetTree(t *testing.T) {
|
||||
{
|
||||
gh := NewGitHubRepository()
|
||||
assert.NoError(t, gh.parse(urlA))
|
||||
assert.NoError(t, gh.setBranch(""))
|
||||
err := gh.setTree()
|
||||
assert.NoError(t, err)
|
||||
assert.Less(t, 0, len(gh.getTree().InnerTrees))
|
||||
}
|
||||
}
|
||||
func TestGithubGetYamlFromTree(t *testing.T) {
|
||||
{
|
||||
gh := NewGitHubRepository()
|
||||
assert.NoError(t, gh.parse(urlA))
|
||||
assert.NoError(t, gh.setBranch(""))
|
||||
assert.NoError(t, gh.setTree())
|
||||
files := gh.getFilesFromTree([]string{"yaml"})
|
||||
assert.Less(t, 0, len(files))
|
||||
}
|
||||
{
|
||||
gh := NewGitHubRepository()
|
||||
assert.NoError(t, gh.parse(urlB))
|
||||
assert.NoError(t, gh.setBranch(""))
|
||||
assert.NoError(t, gh.setTree())
|
||||
files := gh.getFilesFromTree([]string{"yaml"})
|
||||
assert.Equal(t, 1, len(files))
|
||||
}
|
||||
{
|
||||
gh := NewGitHubRepository()
|
||||
assert.NoError(t, gh.parse(urlC))
|
||||
assert.NoError(t, gh.setBranch(""))
|
||||
assert.NoError(t, gh.setTree())
|
||||
files := gh.getFilesFromTree([]string{"yaml"})
|
||||
assert.Equal(t, 12, len(files))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGithubParse(t *testing.T) {
|
||||
{
|
||||
gh := NewGitHubRepository()
|
||||
assert.NoError(t, gh.parse(urlA))
|
||||
assert.Equal(t, "armosec/kubescape", joinOwnerNRepo(gh.owner, gh.repo))
|
||||
}
|
||||
{
|
||||
gh := NewGitHubRepository()
|
||||
assert.NoError(t, gh.parse(urlB))
|
||||
assert.Equal(t, "armosec/kubescape", joinOwnerNRepo(gh.owner, gh.repo))
|
||||
assert.Equal(t, "master", gh.branch)
|
||||
assert.Equal(t, "examples/online-boutique/adservice.yaml", gh.path)
|
||||
assert.True(t, gh.isFile)
|
||||
assert.Equal(t, 1, len(gh.getFilesFromTree([]string{"yaml"})))
|
||||
assert.Equal(t, 1, len(gh.getFilesFromTree([]string{"yml"})))
|
||||
}
|
||||
{
|
||||
gh := NewGitHubRepository()
|
||||
assert.NoError(t, gh.parse(urlC))
|
||||
assert.Equal(t, "armosec/kubescape", joinOwnerNRepo(gh.owner, gh.repo))
|
||||
assert.Equal(t, "master", gh.branch)
|
||||
assert.Equal(t, "examples/online-boutique", gh.path)
|
||||
assert.False(t, gh.isFile)
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
"github.com/armosec/kubescape/core/cautils/logger"
|
||||
)
|
||||
|
||||
func loadResourcesFromUrl(inputPatterns []string) ([]workloadinterface.IMetadata, error) {
|
||||
func loadResourcesFromUrl(inputPatterns []string) (map[string][]workloadinterface.IMetadata, error) {
|
||||
urls := listUrls(inputPatterns)
|
||||
if len(urls) == 0 {
|
||||
return nil, nil
|
||||
@@ -29,14 +29,10 @@ func listUrls(patterns []string) []string {
|
||||
urls := []string{}
|
||||
for i := range patterns {
|
||||
if strings.HasPrefix(patterns[i], "http") {
|
||||
if !cautils.IsYaml(patterns[i]) && !cautils.IsJson(patterns[i]) { // if url of repo
|
||||
if yamls, err := ScanRepository(patterns[i], ""); err == nil { // TODO - support branch
|
||||
urls = append(urls, yamls...)
|
||||
} else {
|
||||
logger.L().Error(err.Error())
|
||||
}
|
||||
} else { // url of single file
|
||||
urls = append(urls, patterns[i])
|
||||
if yamls, err := ScanRepository(patterns[i], ""); err == nil { // TODO - support branch
|
||||
urls = append(urls, yamls...)
|
||||
} else {
|
||||
logger.L().Error(err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -44,8 +40,8 @@ func listUrls(patterns []string) []string {
|
||||
return urls
|
||||
}
|
||||
|
||||
func downloadFiles(urls []string) ([]workloadinterface.IMetadata, []error) {
|
||||
workloads := []workloadinterface.IMetadata{}
|
||||
func downloadFiles(urls []string) (map[string][]workloadinterface.IMetadata, []error) {
|
||||
workloads := make(map[string][]workloadinterface.IMetadata, 0)
|
||||
errs := []error{}
|
||||
for i := range urls {
|
||||
f, err := downloadFile(urls[i])
|
||||
@@ -56,7 +52,12 @@ func downloadFiles(urls []string) ([]workloadinterface.IMetadata, []error) {
|
||||
w, e := cautils.ReadFile(f, cautils.GetFileFormat(urls[i]))
|
||||
errs = append(errs, e...)
|
||||
if w != nil {
|
||||
workloads = append(workloads, w...)
|
||||
if _, ok := workloads[urls[i]]; !ok {
|
||||
workloads[urls[i]] = make([]workloadinterface.IMetadata, 0)
|
||||
}
|
||||
wSlice := workloads[urls[i]]
|
||||
wSlice = append(wSlice, w...)
|
||||
workloads[urls[i]] = wSlice
|
||||
}
|
||||
}
|
||||
return workloads, errs
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package resourcehandler
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestLoadResourcesFromUrl(t *testing.T) {
|
||||
{
|
||||
workloads, err := loadResourcesFromUrl([]string{"https://github.com/armosec/kubescape/tree/master/examples/online-boutique"})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 12, len(workloads))
|
||||
|
||||
for i, w := range workloads {
|
||||
switch i {
|
||||
case "https://raw.githubusercontent.com/armosec/kubescape/master/examples/online-boutique/adservice.yaml":
|
||||
assert.Equal(t, 2, len(w))
|
||||
assert.Equal(t, "apps/v1//Deployment/adservice", w[0].GetID())
|
||||
assert.Equal(t, "/v1//Service/adservice", w[1].GetID())
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
workloads, err := loadResourcesFromUrl([]string{"https://github.com/armosec/kubescape"})
|
||||
assert.NoError(t, err)
|
||||
assert.Less(t, 12, len(workloads))
|
||||
|
||||
for i, w := range workloads {
|
||||
switch i {
|
||||
case "https://raw.githubusercontent.com/armosec/kubescape/master/examples/online-boutique/adservice.yaml":
|
||||
assert.Equal(t, 2, len(w))
|
||||
assert.Equal(t, "apps/v1//Deployment/adservice", w[0].GetID())
|
||||
assert.Equal(t, "/v1//Service/adservice", w[1].GetID())
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
workloads, err := loadResourcesFromUrl([]string{"https://github.com/armosec/kubescape/blob/master/examples/online-boutique/adservice.yaml"})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, len(workloads))
|
||||
|
||||
for i, w := range workloads {
|
||||
switch i {
|
||||
case "https://raw.githubusercontent.com/armosec/kubescape/master/examples/online-boutique/adservice.yaml":
|
||||
assert.Equal(t, 2, len(w))
|
||||
assert.Equal(t, "apps/v1//Deployment/adservice", w[0].GetID())
|
||||
assert.Equal(t, "/v1//Service/adservice", w[1].GetID())
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
workloads, err := loadResourcesFromUrl([]string{"https://raw.githubusercontent.com/armosec/kubescape/master/examples/online-boutique/adservice.yaml"})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, 1, len(workloads))
|
||||
|
||||
for i, w := range workloads {
|
||||
switch i {
|
||||
case "https://raw.githubusercontent.com/armosec/kubescape/master/examples/online-boutique/adservice.yaml":
|
||||
assert.Equal(t, 2, len(w))
|
||||
assert.Equal(t, "apps/v1//Deployment/adservice", w[0].GetID())
|
||||
assert.Equal(t, "/v1//Service/adservice", w[1].GetID())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user