fixed url scanning

This commit is contained in:
dwertent
2022-03-24 09:45:35 +02:00
parent 5a335d4f1c
commit 8487a031ee
11 changed files with 380 additions and 64 deletions
+1
View File
@@ -88,6 +88,7 @@ require (
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/whilp/git-urls v1.0.0 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/yashtewari/glob-intersection v0.0.0-20180916065949-5c77d914dd0b // indirect
+2
View File
@@ -774,6 +774,8 @@ github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqri
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU=
github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE=
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+9 -4
View File
@@ -26,7 +26,7 @@ const (
JSON_FILE_FORMAT FileFormat = "json"
)
func LoadResourcesFromFiles(inputPatterns []string) ([]workloadinterface.IMetadata, error) {
func LoadResourcesFromFiles(inputPatterns []string) (map[string][]workloadinterface.IMetadata, error) {
files, errs := listFiles(inputPatterns)
if len(errs) > 0 {
logger.L().Error(fmt.Sprintf("%v", errs))
@@ -42,8 +42,8 @@ func LoadResourcesFromFiles(inputPatterns []string) ([]workloadinterface.IMetada
return workloads, nil
}
func loadFiles(filePaths []string) ([]workloadinterface.IMetadata, []error) {
workloads := []workloadinterface.IMetadata{}
func loadFiles(filePaths []string) (map[string][]workloadinterface.IMetadata, []error) {
workloads := make(map[string][]workloadinterface.IMetadata, 0)
errs := []error{}
for i := range filePaths {
f, err := loadFile(filePaths[i])
@@ -54,7 +54,12 @@ func loadFiles(filePaths []string) ([]workloadinterface.IMetadata, []error) {
w, e := ReadFile(f, GetFileFormat(filePaths[i]))
errs = append(errs, e...)
if w != nil {
workloads = append(workloads, w...)
if _, ok := workloads[filePaths[i]]; !ok {
workloads[filePaths[i]] = []workloadinterface.IMetadata{}
}
wSlice := workloads[filePaths[i]]
wSlice = append(wSlice, w...)
workloads[filePaths[i]] = wSlice
}
}
return workloads, errs
+14
View File
@@ -23,6 +23,20 @@ func TestListFiles(t *testing.T) {
assert.Equal(t, 12, len(files))
}
func TestLoadResourcesFromFiles(t *testing.T) {
workloads, err := LoadResourcesFromFiles([]string{onlineBoutiquePath()})
assert.NoError(t, err)
assert.Equal(t, 12, len(workloads))
for i, w := range workloads {
switch filepath.Base(i) {
case "adservice.yaml":
assert.Equal(t, 2, len(w))
assert.Equal(t, "apps/v1//Deployment/adservice", w[0].GetID())
assert.Equal(t, "/v1//Service/adservice", w[1].GetID())
}
}
}
func TestLoadFiles(t *testing.T) {
files, _ := listFiles([]string{onlineBoutiquePath()})
_, err := loadFiles(files)
+1
View File
@@ -19,6 +19,7 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/open-policy-agent/opa v0.38.0
github.com/stretchr/testify v1.7.0
github.com/whilp/git-urls v1.0.0
go.uber.org/zap v1.21.0
golang.org/x/mod v0.5.1
gopkg.in/yaml.v2 v2.4.0
+2
View File
@@ -771,6 +771,8 @@ github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqri
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU=
github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE=
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+11 -4
View File
@@ -32,6 +32,7 @@ func (fileHandler *FileResourceHandler) GetResources(sessionObj *cautils.OPASess
// map resources based on framework required resources: map["/group/version/kind"][]<k8s workloads ids>
k8sResources := setK8sResourceMap(sessionObj.Policies)
allResources := map[string]workloadinterface.IMetadata{}
workloadIDToSource := make(map[string]string, 0)
armoResources := &cautils.ArmoResources{}
workloads := []workloadinterface.IMetadata{}
@@ -41,8 +42,11 @@ func (fileHandler *FileResourceHandler) GetResources(sessionObj *cautils.OPASess
if err != nil {
return nil, allResources, nil, err
}
if w != nil {
workloads = append(workloads, w...)
for source, ws := range w {
workloads = append(workloads, ws...)
for i := range ws {
workloadIDToSource[ws[i].GetID()] = source
}
}
// load resources from url
@@ -50,8 +54,11 @@ func (fileHandler *FileResourceHandler) GetResources(sessionObj *cautils.OPASess
if err != nil {
return nil, allResources, nil, err
}
if w != nil {
workloads = append(workloads, w...)
for source, ws := range w {
workloads = append(workloads, ws...)
for i := range ws {
workloadIDToSource[ws[i].GetID()] = source
}
}
if len(workloads) == 0 {
+124 -44
View File
@@ -4,15 +4,24 @@ import (
"encoding/json"
"fmt"
"net/http"
"path/filepath"
"strings"
"github.com/armosec/kubescape/core/cautils/getter"
giturls "github.com/whilp/git-urls"
"k8s.io/utils/strings/slices"
)
type IRepository interface {
parse(fullURL string) error
setBranch(string) error
setTree() error
getYamlFromTree() []string
getBranch() string
getTree() tree
getFilesFromTree([]string) []string
}
type innerTree struct {
@@ -23,19 +32,23 @@ type tree struct {
}
type GitHubRepository struct {
// name string // <org>/<repo>
host string
name string // <org>/<repo>
owner string //
repo string //
branch string
path string
isFile bool
tree tree
}
type githubDefaultBranchAPI struct {
DefaultBranch string `json:"default_branch"`
}
func NewGitHubRepository(rep string) *GitHubRepository {
func NewGitHubRepository() *GitHubRepository {
return &GitHubRepository{
host: "github",
name: rep,
host: "github.com",
// name: rep,
}
}
@@ -45,75 +58,132 @@ func ScanRepository(command string, branchOptional string) ([]string, error) {
return nil, err
}
err = repo.setBranch(branchOptional)
if err != nil {
if err := repo.parse(command); err != nil {
return nil, err
}
err = repo.setTree()
if err != nil {
if err := repo.setBranch(branchOptional); err != nil {
return nil, err
}
if err := repo.setTree(); err != nil {
return nil, err
}
// get all paths that are of the yaml type, and build them into a valid url
return repo.getYamlFromTree(), nil
return repo.getFilesFromTree([]string{"yaml", "yml", "json"}), nil
}
func getHostAndRepoName(url string) (string, string, error) {
splitUrl := strings.Split(url, "/")
if len(splitUrl) != 5 {
return "", "", fmt.Errorf("failed to pars url: %s", url)
func getHost(fullURL string) (string, error) {
parsedURL, err := giturls.Parse(fullURL)
if err != nil {
return "", err
}
hostUrl := splitUrl[2] // github.com, gitlab.com, etc.
repository := splitUrl[3] + "/" + strings.Split(splitUrl[4], ".")[0] // user/reposetory
return hostUrl, repository, nil
return parsedURL.Host, nil
}
func getRepository(url string) (IRepository, error) {
hostUrl, repoName, err := getHostAndRepoName(url)
// func parseHostAndRepoName(fullURL string) (string, string, error) {
// parsedURL, err := giturls.Parse(fullURL)
// if err != nil {
// return "", "", err
// }
// host := parsedURL.Host
// splittedRepo := strings.FieldsFunc(parsedURL.Path, func(c rune) bool { return c == '/' })
// if len(splittedRepo) < 2 {
// return "", "", fmt.Errorf("expecting <user>/<repo> in url path, received: '%s'", parsedURL.Path)
// }
// return host, fmt.Sprintf("%s/%s", splittedRepo[0], splittedRepo[1]), nil
// }
func getRepository(fullURL string) (IRepository, error) {
hostUrl, err := getHost(fullURL)
if err != nil {
return nil, err
}
var repo IRepository
switch repoHost := strings.Split(hostUrl, ".")[0]; repoHost {
case "github":
repo = NewGitHubRepository(repoName)
switch hostUrl {
case "github.com", "raw.githubusercontent.com":
repo = NewGitHubRepository()
default:
return nil, fmt.Errorf("unknown repository host: %s", repoHost)
return nil, fmt.Errorf("unknown repository host: %s", hostUrl)
}
// Returns the host-url, and the part of the user and repository from the url
return repo, nil
}
func (g *GitHubRepository) parse(fullURL string) error {
parsedURL, err := giturls.Parse(fullURL)
if err != nil {
return err
}
splittedRepo := strings.FieldsFunc(parsedURL.Path, func(c rune) bool { return c == '/' })
if len(splittedRepo) < 2 {
return fmt.Errorf("expecting <user>/<repo> in url path, received: '%s'", parsedURL.Path)
}
g.owner = splittedRepo[0]
g.repo = splittedRepo[1]
// root of repo
if len(splittedRepo) < 4 {
return nil
}
// is file or dir
switch splittedRepo[2] {
case "blob":
g.isFile = true
case "tree":
g.isFile = false
default:
// Unknown - failed to parse
return nil
}
g.branch = splittedRepo[3]
if len(splittedRepo) < 5 {
return nil
}
g.path = strings.Join(splittedRepo[4:], "/")
return nil
}
func (g *GitHubRepository) getBranch() string { return g.branch }
func (g *GitHubRepository) getTree() tree { return g.tree }
func (g *GitHubRepository) setBranch(branchOptional string) error {
// Checks whether the repository type is a master or another type.
// By default it is "master", unless the branchOptional came with a value
if branchOptional == "" {
body, err := getter.HttpGetter(&http.Client{}, g.defaultBranchAPI(), nil)
if err != nil {
return err
}
var data githubDefaultBranchAPI
err = json.Unmarshal([]byte(body), &data)
if err != nil {
return err
}
g.branch = data.DefaultBranch
} else {
if branchOptional != "" {
g.branch = branchOptional
}
if g.branch != "" {
return nil
}
body, err := getter.HttpGetter(&http.Client{}, g.defaultBranchAPI(), nil)
if err != nil {
return err
}
var data githubDefaultBranchAPI
err = json.Unmarshal([]byte(body), &data)
if err != nil {
return err
}
g.branch = data.DefaultBranch
return nil
}
func joinOwnerNRepo(owner, repo string) string {
return fmt.Sprintf("%s/%s", owner, repo)
}
func (g *GitHubRepository) defaultBranchAPI() string {
return fmt.Sprintf("https://api.github.com/repos/%s", g.name)
return fmt.Sprintf("https://api.github.com/repos/%s", joinOwnerNRepo(g.owner, g.repo))
}
func (g *GitHubRepository) setTree() error {
@@ -135,14 +205,20 @@ func (g *GitHubRepository) setTree() error {
}
func (g *GitHubRepository) treeAPI() string {
return fmt.Sprintf("https://api.github.com/repos/%s/git/trees/%s?recursive=1", g.name, g.branch)
return fmt.Sprintf("https://api.github.com/repos/%s/git/trees/%s?recursive=1", joinOwnerNRepo(g.owner, g.repo), g.branch)
}
// return a list of yaml for a given repository tree
func (g *GitHubRepository) getYamlFromTree() []string {
func (g *GitHubRepository) getFilesFromTree(filesExtensions []string) []string {
var urls []string
if g.isFile {
return []string{fmt.Sprintf("%s/%s", g.rowYamlUrl(), g.path)}
}
for _, path := range g.tree.InnerTrees {
if strings.HasSuffix(path.Path, ".yaml") {
if g.path != "" && !strings.HasPrefix(path.Path, g.path) {
continue
}
if slices.Contains(filesExtensions, getFileExtension(path.Path)) {
urls = append(urls, fmt.Sprintf("%s/%s", g.rowYamlUrl(), path.Path))
}
}
@@ -150,5 +226,9 @@ func (g *GitHubRepository) getYamlFromTree() []string {
}
func (g *GitHubRepository) rowYamlUrl() string {
return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s", g.name, g.branch)
return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s", joinOwnerNRepo(g.owner, g.repo), g.branch)
}
func getFileExtension(path string) string {
return strings.TrimPrefix(filepath.Ext(path), ".")
}
@@ -0,0 +1,137 @@
package resourcehandler
import (
"testing"
"github.com/stretchr/testify/assert"
)
var (
urlA = "https://github.com/armosec/kubescape"
urlB = "https://github.com/armosec/kubescape/blob/master/examples/online-boutique/adservice.yaml"
urlC = "https://github.com/armosec/kubescape/tree/master/examples/online-boutique"
// urlD = "https://raw.githubusercontent.com/armosec/kubescape/master/examples/online-boutique/adservice.yaml"
)
func TestScanRepository(t *testing.T) {
{
files, err := ScanRepository(urlA, "")
assert.NoError(t, err)
assert.Less(t, 0, len(files))
}
{
files, err := ScanRepository(urlB, "")
assert.NoError(t, err)
assert.Less(t, 0, len(files))
}
{
files, err := ScanRepository(urlC, "")
assert.NoError(t, err)
assert.Less(t, 0, len(files))
}
// {
// files, err := ScanRepository(urlD, "")
// assert.NoError(t, err)
// assert.Equal(t, 1, len(files))
// }
}
func TestGetHost(t *testing.T) {
{
host, err := getHost(urlA)
assert.NoError(t, err)
assert.Equal(t, "github.com", host)
}
{
host, err := getHost(urlB)
assert.NoError(t, err)
assert.Equal(t, "github.com", host)
}
{
host, err := getHost(urlC)
assert.NoError(t, err)
assert.Equal(t, "github.com", host)
}
}
// repo = NewGitHubRepository(repoName)
func TestGithubSetBranch(t *testing.T) {
{
gh := NewGitHubRepository()
assert.NoError(t, gh.parse(urlA))
assert.NoError(t, gh.setBranch(""))
assert.Equal(t, "master", gh.getBranch())
}
{
gh := NewGitHubRepository()
assert.NoError(t, gh.parse(urlB))
err := gh.setBranch("dev")
assert.NoError(t, err)
assert.Equal(t, "dev", gh.getBranch())
}
}
func TestGithubSetTree(t *testing.T) {
{
gh := NewGitHubRepository()
assert.NoError(t, gh.parse(urlA))
assert.NoError(t, gh.setBranch(""))
err := gh.setTree()
assert.NoError(t, err)
assert.Less(t, 0, len(gh.getTree().InnerTrees))
}
}
func TestGithubGetYamlFromTree(t *testing.T) {
{
gh := NewGitHubRepository()
assert.NoError(t, gh.parse(urlA))
assert.NoError(t, gh.setBranch(""))
assert.NoError(t, gh.setTree())
files := gh.getFilesFromTree([]string{"yaml"})
assert.Less(t, 0, len(files))
}
{
gh := NewGitHubRepository()
assert.NoError(t, gh.parse(urlB))
assert.NoError(t, gh.setBranch(""))
assert.NoError(t, gh.setTree())
files := gh.getFilesFromTree([]string{"yaml"})
assert.Equal(t, 1, len(files))
}
{
gh := NewGitHubRepository()
assert.NoError(t, gh.parse(urlC))
assert.NoError(t, gh.setBranch(""))
assert.NoError(t, gh.setTree())
files := gh.getFilesFromTree([]string{"yaml"})
assert.Equal(t, 12, len(files))
}
}
func TestGithubParse(t *testing.T) {
{
gh := NewGitHubRepository()
assert.NoError(t, gh.parse(urlA))
assert.Equal(t, "armosec/kubescape", joinOwnerNRepo(gh.owner, gh.repo))
}
{
gh := NewGitHubRepository()
assert.NoError(t, gh.parse(urlB))
assert.Equal(t, "armosec/kubescape", joinOwnerNRepo(gh.owner, gh.repo))
assert.Equal(t, "master", gh.branch)
assert.Equal(t, "examples/online-boutique/adservice.yaml", gh.path)
assert.True(t, gh.isFile)
assert.Equal(t, 1, len(gh.getFilesFromTree([]string{"yaml"})))
assert.Equal(t, 1, len(gh.getFilesFromTree([]string{"yml"})))
}
{
gh := NewGitHubRepository()
assert.NoError(t, gh.parse(urlC))
assert.Equal(t, "armosec/kubescape", joinOwnerNRepo(gh.owner, gh.repo))
assert.Equal(t, "master", gh.branch)
assert.Equal(t, "examples/online-boutique", gh.path)
assert.False(t, gh.isFile)
}
}
+13 -12
View File
@@ -12,7 +12,7 @@ import (
"github.com/armosec/kubescape/core/cautils/logger"
)
func loadResourcesFromUrl(inputPatterns []string) ([]workloadinterface.IMetadata, error) {
func loadResourcesFromUrl(inputPatterns []string) (map[string][]workloadinterface.IMetadata, error) {
urls := listUrls(inputPatterns)
if len(urls) == 0 {
return nil, nil
@@ -29,14 +29,10 @@ func listUrls(patterns []string) []string {
urls := []string{}
for i := range patterns {
if strings.HasPrefix(patterns[i], "http") {
if !cautils.IsYaml(patterns[i]) && !cautils.IsJson(patterns[i]) { // if url of repo
if yamls, err := ScanRepository(patterns[i], ""); err == nil { // TODO - support branch
urls = append(urls, yamls...)
} else {
logger.L().Error(err.Error())
}
} else { // url of single file
urls = append(urls, patterns[i])
if yamls, err := ScanRepository(patterns[i], ""); err == nil { // TODO - support branch
urls = append(urls, yamls...)
} else {
logger.L().Error(err.Error())
}
}
}
@@ -44,8 +40,8 @@ func listUrls(patterns []string) []string {
return urls
}
func downloadFiles(urls []string) ([]workloadinterface.IMetadata, []error) {
workloads := []workloadinterface.IMetadata{}
func downloadFiles(urls []string) (map[string][]workloadinterface.IMetadata, []error) {
workloads := make(map[string][]workloadinterface.IMetadata, 0)
errs := []error{}
for i := range urls {
f, err := downloadFile(urls[i])
@@ -56,7 +52,12 @@ func downloadFiles(urls []string) ([]workloadinterface.IMetadata, []error) {
w, e := cautils.ReadFile(f, cautils.GetFileFormat(urls[i]))
errs = append(errs, e...)
if w != nil {
workloads = append(workloads, w...)
if _, ok := workloads[urls[i]]; !ok {
workloads[urls[i]] = make([]workloadinterface.IMetadata, 0)
}
wSlice := workloads[urls[i]]
wSlice = append(wSlice, w...)
workloads[urls[i]] = wSlice
}
}
return workloads, errs
@@ -0,0 +1,66 @@
package resourcehandler
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestLoadResourcesFromUrl(t *testing.T) {
{
workloads, err := loadResourcesFromUrl([]string{"https://github.com/armosec/kubescape/tree/master/examples/online-boutique"})
assert.NoError(t, err)
assert.Equal(t, 12, len(workloads))
for i, w := range workloads {
switch i {
case "https://raw.githubusercontent.com/armosec/kubescape/master/examples/online-boutique/adservice.yaml":
assert.Equal(t, 2, len(w))
assert.Equal(t, "apps/v1//Deployment/adservice", w[0].GetID())
assert.Equal(t, "/v1//Service/adservice", w[1].GetID())
}
}
}
{
workloads, err := loadResourcesFromUrl([]string{"https://github.com/armosec/kubescape"})
assert.NoError(t, err)
assert.Less(t, 12, len(workloads))
for i, w := range workloads {
switch i {
case "https://raw.githubusercontent.com/armosec/kubescape/master/examples/online-boutique/adservice.yaml":
assert.Equal(t, 2, len(w))
assert.Equal(t, "apps/v1//Deployment/adservice", w[0].GetID())
assert.Equal(t, "/v1//Service/adservice", w[1].GetID())
}
}
}
{
workloads, err := loadResourcesFromUrl([]string{"https://github.com/armosec/kubescape/blob/master/examples/online-boutique/adservice.yaml"})
assert.NoError(t, err)
assert.Equal(t, 1, len(workloads))
for i, w := range workloads {
switch i {
case "https://raw.githubusercontent.com/armosec/kubescape/master/examples/online-boutique/adservice.yaml":
assert.Equal(t, 2, len(w))
assert.Equal(t, "apps/v1//Deployment/adservice", w[0].GetID())
assert.Equal(t, "/v1//Service/adservice", w[1].GetID())
}
}
}
{
workloads, err := loadResourcesFromUrl([]string{"https://raw.githubusercontent.com/armosec/kubescape/master/examples/online-boutique/adservice.yaml"})
assert.NoError(t, err)
assert.Equal(t, 1, len(workloads))
for i, w := range workloads {
switch i {
case "https://raw.githubusercontent.com/armosec/kubescape/master/examples/online-boutique/adservice.yaml":
assert.Equal(t, 2, len(w))
assert.Equal(t, "apps/v1//Deployment/adservice", w[0].GetID())
assert.Equal(t, "/v1//Service/adservice", w[1].GetID())
}
}
}
}