Compare commits

...

20 Commits

Author SHA1 Message Date
dwertent
3e1fda6f3b fixed test 2022-03-24 11:12:28 +02:00
dwertent
8487a031ee fixed url scanning 2022-03-24 09:45:35 +02:00
dwertent
5a335d4f1c update error display 2022-03-23 09:05:36 +02:00
dwertent
5770a823d6 update readme 2022-03-23 08:47:24 +02:00
Rotem Refael
52d7be9108 Add kubescape covarage 2022-03-22 17:42:20 +02:00
David Wertenteil
9512b9c6c4 Merge pull request #463 from Daniel-GrunbergerCA/dev
fix json printer
2022-03-22 09:14:59 +02:00
DanielGrunbergerCA
da9ab642ec rm list initializtion 2022-03-22 09:07:55 +02:00
DanielGrunbergerCA
718ca1c7ab fix json printer 2022-03-22 08:55:36 +02:00
DanielGrunbergerCA
ee3742c5a0 update opa-utils version 2022-03-21 15:36:17 +02:00
DanielGrunbergerCA
7eef843a7a fix resources in report 2022-03-21 15:11:26 +02:00
David Wertenteil
b4a8b06f07 Merge pull request #462 from dwertent/master
update get context
2022-03-21 09:09:12 +02:00
dwertent
4e13609985 update get context 2022-03-21 09:08:01 +02:00
David Wertenteil
2e5e4328f6 Merge pull request #460 from dwertent/master
update readme
2022-03-20 11:30:37 +02:00
dwertent
d98a11a8fa udpate badges 2022-03-20 11:28:56 +02:00
dwertent
bdb25cbb66 adding vs code to readme 2022-03-20 11:08:48 +02:00
David Wertenteil
369804cb6e Merge pull request #459 from shm12/dev
send scan metadata
2022-03-20 10:45:19 +02:00
shm12
1b08a92095 send scan metadata 2022-03-20 10:05:40 +02:00
David Wertenteil
e787454d53 Merge pull request #458 from dwertent/master
fixed json output
2022-03-16 16:58:54 +02:00
dwertent
31d1ba663a json output 2022-03-16 16:58:05 +02:00
David Wertenteil
c3731d8ff6 Merge pull request #457 from dwertent/master
support frameworks from http request
2022-03-16 16:37:40 +02:00
26 changed files with 495 additions and 131 deletions

View File

@@ -12,11 +12,19 @@ Kubescape integrates natively with other DevOps tools, including Jenkins, Circle
</br>
# CLI Interface:
<!-- # Kubescape Coverage
<img src="docs/ksfromcodetodeploy.png">
</br> -->
# Kubescape CLI:
<img src="docs/demo.gif">
# Web Interface:
<img src="docs/ARMO-header-2022.gif">
</br>
<!-- # Kubescape overview:
<img src="docs/ARMO-header-2022.gif"> -->
# TL;DR
## Install:
@@ -259,6 +267,16 @@ Now you can submit the results to the Kubescape SaaS version -
kubescape submit results path/to/results.json
```
# Integrations
## VS Code Extension
![Visual Studio Marketplace Downloads](https://img.shields.io/visual-studio-marketplace/d/kubescape.kubescape?label=VScode) ![Open VSX](https://img.shields.io/open-vsx/dt/kubescape/kubescape?label=openVSX&color=yellowgreen)
Scan the YAML files while writing them using the [vs code extension](https://github.com/armosec/vscode-kubescape/blob/master/README.md)
# Under the hood
## Technology

View File

@@ -7,7 +7,7 @@ replace github.com/armosec/kubescape/core => ../core
require (
github.com/armosec/k8s-interface v0.0.68
github.com/armosec/kubescape/core v0.0.0-00010101000000-000000000000
github.com/armosec/opa-utils v0.0.120
github.com/armosec/opa-utils v0.0.125
github.com/armosec/rbac-utils v0.0.14
github.com/google/uuid v1.3.0
github.com/mattn/go-isatty v0.0.14
@@ -88,6 +88,7 @@ require (
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/whilp/git-urls v1.0.0 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/yashtewari/glob-intersection v0.0.0-20180916065949-5c77d914dd0b // indirect

View File

@@ -109,8 +109,8 @@ github.com/armosec/k8s-interface v0.0.66/go.mod h1:vwprS8qn/iowd5yf0JHpqDsLA5I8W
github.com/armosec/k8s-interface v0.0.68 h1:6CtSakISiI47YHkxh+Va9FzZQIBkWa6g9sbiNxq1Zkk=
github.com/armosec/k8s-interface v0.0.68/go.mod h1:PeWn41C2uenZi+xfZdyFF/zG5wXACA00htQyknDUWDE=
github.com/armosec/opa-utils v0.0.64/go.mod h1:6tQP8UDq2EvEfSqh8vrUdr/9QVSCG4sJfju1SXQOn4c=
github.com/armosec/opa-utils v0.0.120 h1:WAtgm2U1o9fgA/2pjYNy+igqNC6ju3/CxQ8qRHdO+5k=
github.com/armosec/opa-utils v0.0.120/go.mod h1:gap+EaLG5rnyqvIRGxtdNDC9y7VvoGNm90zK8Ls7avQ=
github.com/armosec/opa-utils v0.0.125 h1:ZA7v3kvqITwazhFkvZuGP/bc/VChsYWQdrDcvyf2qsw=
github.com/armosec/opa-utils v0.0.125/go.mod h1:gap+EaLG5rnyqvIRGxtdNDC9y7VvoGNm90zK8Ls7avQ=
github.com/armosec/rbac-utils v0.0.1/go.mod h1:pQ8CBiij8kSKV7aeZm9FMvtZN28VgA7LZcYyTWimq40=
github.com/armosec/rbac-utils v0.0.14 h1:CKYKcgqJEXWF2Hen/B1pVGtS3nDAG1wp9dDv6oNtq90=
github.com/armosec/rbac-utils v0.0.14/go.mod h1:Ex/IdGWhGv9HZq6Hs8N/ApzCKSIvpNe/ETqDfnuyah0=
@@ -774,6 +774,8 @@ github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqri
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU=
github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE=
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=

View File

@@ -81,7 +81,7 @@ func initEnvironment() {
armoERURL := urlSlices[0] // mandatory
armoBEURL := urlSlices[1] // mandatory
armoFEURL := urlSlices[2] // mandatory
if len(urlSlices) <= 4 {
if len(urlSlices) >= 4 {
armoAUTHURL = urlSlices[3]
}
getter.SetARMOAPIConnector(getter.NewARMOAPICustomized(armoERURL, armoBEURL, armoFEURL, armoAUTHURL))

View File

@@ -26,7 +26,7 @@ const (
JSON_FILE_FORMAT FileFormat = "json"
)
func LoadResourcesFromFiles(inputPatterns []string) ([]workloadinterface.IMetadata, error) {
func LoadResourcesFromFiles(inputPatterns []string) (map[string][]workloadinterface.IMetadata, error) {
files, errs := listFiles(inputPatterns)
if len(errs) > 0 {
logger.L().Error(fmt.Sprintf("%v", errs))
@@ -42,8 +42,8 @@ func LoadResourcesFromFiles(inputPatterns []string) ([]workloadinterface.IMetada
return workloads, nil
}
func loadFiles(filePaths []string) ([]workloadinterface.IMetadata, []error) {
workloads := []workloadinterface.IMetadata{}
func loadFiles(filePaths []string) (map[string][]workloadinterface.IMetadata, []error) {
workloads := make(map[string][]workloadinterface.IMetadata, 0)
errs := []error{}
for i := range filePaths {
f, err := loadFile(filePaths[i])
@@ -54,7 +54,12 @@ func loadFiles(filePaths []string) ([]workloadinterface.IMetadata, []error) {
w, e := ReadFile(f, GetFileFormat(filePaths[i]))
errs = append(errs, e...)
if w != nil {
workloads = append(workloads, w...)
if _, ok := workloads[filePaths[i]]; !ok {
workloads[filePaths[i]] = []workloadinterface.IMetadata{}
}
wSlice := workloads[filePaths[i]]
wSlice = append(wSlice, w...)
workloads[filePaths[i]] = wSlice
}
}
return workloads, errs

View File

@@ -23,6 +23,20 @@ func TestListFiles(t *testing.T) {
assert.Equal(t, 12, len(files))
}
func TestLoadResourcesFromFiles(t *testing.T) {
workloads, err := LoadResourcesFromFiles([]string{onlineBoutiquePath()})
assert.NoError(t, err)
assert.Equal(t, 12, len(workloads))
for i, w := range workloads {
switch filepath.Base(i) {
case "adservice.yaml":
assert.Equal(t, 2, len(w))
assert.Equal(t, "apps/v1//Deployment/adservice", w[0].GetID())
assert.Equal(t, "/v1//Service/adservice", w[1].GetID())
}
}
}
func TestLoadFiles(t *testing.T) {
files, _ := listFiles([]string{onlineBoutiquePath()})
_, err := loadFiles(files)

View File

@@ -38,6 +38,12 @@ func (bpf *BoolPtrFlag) String() string {
func (bpf *BoolPtrFlag) Get() *bool {
return bpf.valPtr
}
func (bpf *BoolPtrFlag) GetBool() bool {
if bpf.valPtr == nil {
return false
}
return *bpf.valPtr
}
func (bpf *BoolPtrFlag) SetBool(val bool) {
bpf.valPtr = &val

View File

@@ -5,7 +5,7 @@ go 1.17
require (
github.com/armosec/armoapi-go v0.0.58
github.com/armosec/k8s-interface v0.0.68
github.com/armosec/opa-utils v0.0.120
github.com/armosec/opa-utils v0.0.125
github.com/armosec/rbac-utils v0.0.14
github.com/armosec/utils-go v0.0.3
github.com/armosec/utils-k8s-go v0.0.3
@@ -19,6 +19,7 @@ require (
github.com/olekukonko/tablewriter v0.0.5
github.com/open-policy-agent/opa v0.38.0
github.com/stretchr/testify v1.7.0
github.com/whilp/git-urls v1.0.0
go.uber.org/zap v1.21.0
golang.org/x/mod v0.5.1
gopkg.in/yaml.v2 v2.4.0

View File

@@ -109,8 +109,8 @@ github.com/armosec/k8s-interface v0.0.66/go.mod h1:vwprS8qn/iowd5yf0JHpqDsLA5I8W
github.com/armosec/k8s-interface v0.0.68 h1:6CtSakISiI47YHkxh+Va9FzZQIBkWa6g9sbiNxq1Zkk=
github.com/armosec/k8s-interface v0.0.68/go.mod h1:PeWn41C2uenZi+xfZdyFF/zG5wXACA00htQyknDUWDE=
github.com/armosec/opa-utils v0.0.64/go.mod h1:6tQP8UDq2EvEfSqh8vrUdr/9QVSCG4sJfju1SXQOn4c=
github.com/armosec/opa-utils v0.0.120 h1:WAtgm2U1o9fgA/2pjYNy+igqNC6ju3/CxQ8qRHdO+5k=
github.com/armosec/opa-utils v0.0.120/go.mod h1:gap+EaLG5rnyqvIRGxtdNDC9y7VvoGNm90zK8Ls7avQ=
github.com/armosec/opa-utils v0.0.125 h1:ZA7v3kvqITwazhFkvZuGP/bc/VChsYWQdrDcvyf2qsw=
github.com/armosec/opa-utils v0.0.125/go.mod h1:gap+EaLG5rnyqvIRGxtdNDC9y7VvoGNm90zK8Ls7avQ=
github.com/armosec/rbac-utils v0.0.1/go.mod h1:pQ8CBiij8kSKV7aeZm9FMvtZN28VgA7LZcYyTWimq40=
github.com/armosec/rbac-utils v0.0.14 h1:CKYKcgqJEXWF2Hen/B1pVGtS3nDAG1wp9dDv6oNtq90=
github.com/armosec/rbac-utils v0.0.14/go.mod h1:Ex/IdGWhGv9HZq6Hs8N/ApzCKSIvpNe/ETqDfnuyah0=
@@ -771,6 +771,8 @@ github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqri
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU=
github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE=
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=

View File

@@ -2,7 +2,6 @@ package policyhandler
import (
"fmt"
"strings"
"github.com/armosec/k8s-interface/k8sinterface"
"github.com/armosec/kubescape/core/cautils"
@@ -47,30 +46,9 @@ func (policyHandler *PolicyHandler) CollectResources(notification *reporthandlin
return opaSessionObj, nil
}
func scanInfoToScanMetadata(opaSessionObj *cautils.OPASessionObj, scanInfo *cautils.ScanInfo) {
opaSessionObj.Metadata.ClusterMetadata.ContextName = k8sinterface.GetCurrentContext().Cluster
opaSessionObj.Metadata.ScanMetadata.Format = scanInfo.Format
opaSessionObj.Metadata.ScanMetadata.Submit = scanInfo.Submit
if len(scanInfo.ExcludedNamespaces) > 1 {
opaSessionObj.Metadata.ScanMetadata.ExcludedNamespaces = strings.Split(scanInfo.ExcludedNamespaces[1:], ",")
}
// scan type
if len(scanInfo.PolicyIdentifier) > 0 {
opaSessionObj.Metadata.ScanMetadata.TargetType = string(scanInfo.PolicyIdentifier[0].Kind)
}
// append frameworks
for _, policy := range scanInfo.PolicyIdentifier {
opaSessionObj.Metadata.ScanMetadata.TargetNames = append(opaSessionObj.Metadata.ScanMetadata.TargetNames, policy.Name)
}
opaSessionObj.Metadata.ScanMetadata.VerboseMode = scanInfo.VerboseMode
opaSessionObj.Metadata.ScanMetadata.FailThreshold = scanInfo.FailThreshold
opaSessionObj.Metadata.ScanMetadata.HostScanner = *scanInfo.HostSensorEnabled.Get()
opaSessionObj.Metadata.ScanMetadata.VerboseMode = scanInfo.VerboseMode
opaSessionObj.Metadata.ScanMetadata.ControlsInputs = scanInfo.ControlsInputs
}
func (policyHandler *PolicyHandler) getResources(notification *reporthandling.PolicyNotification, opaSessionObj *cautils.OPASessionObj, scanInfo *cautils.ScanInfo) error {
opaSessionObj.Report.ClusterAPIServerInfo = policyHandler.resourceHandler.GetClusterAPIServerInfo()
scanInfoToScanMetadata(opaSessionObj, scanInfo)
resourcesMap, allResources, armoResources, err := policyHandler.resourceHandler.GetResources(opaSessionObj, &notification.Designators)
@@ -84,3 +62,31 @@ func (policyHandler *PolicyHandler) getResources(notification *reporthandling.Po
return nil
}
func scanInfoToScanMetadata(opaSessionObj *cautils.OPASessionObj, scanInfo *cautils.ScanInfo) {
opaSessionObj.Metadata.ClusterMetadata.ContextName = k8sinterface.GetClusterName()
opaSessionObj.Metadata.ScanMetadata.Format = scanInfo.Format
opaSessionObj.Metadata.ScanMetadata.Submit = scanInfo.Submit
// TODO - Add excluded and included namespaces
// if len(scanInfo.ExcludedNamespaces) > 1 {
// opaSessionObj.Metadata.ScanMetadata.ExcludedNamespaces = strings.Split(scanInfo.ExcludedNamespaces[1:], ",")
// }
// if len(scanInfo.IncludeNamespaces) > 1 {
// opaSessionObj.Metadata.ScanMetadata.IncludeNamespaces = strings.Split(scanInfo.IncludeNamespaces[1:], ",")
// }
// scan type
if len(scanInfo.PolicyIdentifier) > 0 {
opaSessionObj.Metadata.ScanMetadata.TargetType = string(scanInfo.PolicyIdentifier[0].Kind)
}
// append frameworks
for _, policy := range scanInfo.PolicyIdentifier {
opaSessionObj.Metadata.ScanMetadata.TargetNames = append(opaSessionObj.Metadata.ScanMetadata.TargetNames, policy.Name)
}
opaSessionObj.Metadata.ScanMetadata.VerboseMode = scanInfo.VerboseMode
opaSessionObj.Metadata.ScanMetadata.FailThreshold = scanInfo.FailThreshold
opaSessionObj.Metadata.ScanMetadata.HostScanner = scanInfo.HostSensorEnabled.GetBool()
opaSessionObj.Metadata.ScanMetadata.VerboseMode = scanInfo.VerboseMode
opaSessionObj.Metadata.ScanMetadata.ControlsInputs = scanInfo.ControlsInputs
}

View File

@@ -32,7 +32,8 @@ func (fileHandler *FileResourceHandler) GetResources(sessionObj *cautils.OPASess
// map resources based on framework required resources: map["/group/version/kind"][]<k8s workloads ids>
k8sResources := setK8sResourceMap(sessionObj.Policies)
allResources := map[string]workloadinterface.IMetadata{}
var armoResources *cautils.ArmoResources
workloadIDToSource := make(map[string]string, 0)
armoResources := &cautils.ArmoResources{}
workloads := []workloadinterface.IMetadata{}
@@ -41,8 +42,11 @@ func (fileHandler *FileResourceHandler) GetResources(sessionObj *cautils.OPASess
if err != nil {
return nil, allResources, nil, err
}
if w != nil {
workloads = append(workloads, w...)
for source, ws := range w {
workloads = append(workloads, ws...)
for i := range ws {
workloadIDToSource[ws[i].GetID()] = source
}
}
// load resources from url
@@ -50,8 +54,11 @@ func (fileHandler *FileResourceHandler) GetResources(sessionObj *cautils.OPASess
if err != nil {
return nil, allResources, nil, err
}
if w != nil {
workloads = append(workloads, w...)
for source, ws := range w {
workloads = append(workloads, ws...)
for i := range ws {
workloadIDToSource[ws[i].GetID()] = source
}
}
if len(workloads) == 0 {

View File

@@ -4,15 +4,26 @@ import (
"encoding/json"
"fmt"
"net/http"
"path/filepath"
"strings"
"github.com/armosec/kubescape/core/cautils/getter"
giturls "github.com/whilp/git-urls"
"k8s.io/utils/strings/slices"
)
type IRepository interface {
parse(fullURL string) error
setBranch(string) error
setTree() error
getYamlFromTree() []string
setIsFile(bool)
getIsFile() bool
getBranch() string
getTree() tree
getFilesFromTree([]string) []string
}
type innerTree struct {
@@ -23,19 +34,23 @@ type tree struct {
}
type GitHubRepository struct {
// name string // <org>/<repo>
host string
name string // <org>/<repo>
owner string //
repo string //
branch string
path string
isFile bool
tree tree
}
type githubDefaultBranchAPI struct {
DefaultBranch string `json:"default_branch"`
}
func NewGitHubRepository(rep string) *GitHubRepository {
func NewGitHubRepository() *GitHubRepository {
return &GitHubRepository{
host: "github",
name: rep,
host: "github.com",
// name: rep,
}
}
@@ -45,75 +60,130 @@ func ScanRepository(command string, branchOptional string) ([]string, error) {
return nil, err
}
err = repo.setBranch(branchOptional)
if err != nil {
if err := repo.parse(command); err != nil {
return nil, err
}
err = repo.setTree()
if err != nil {
if err := repo.setBranch(branchOptional); err != nil {
return nil, err
}
if err := repo.setTree(); err != nil {
return nil, err
}
// get all paths that are of the yaml type, and build them into a valid url
return repo.getYamlFromTree(), nil
return repo.getFilesFromTree([]string{"yaml", "yml", "json"}), nil
}
func getHostAndRepoName(url string) (string, string, error) {
splitUrl := strings.Split(url, "/")
if len(splitUrl) != 5 {
return "", "", fmt.Errorf("failed to pars url: %s", url)
func getHost(fullURL string) (string, error) {
parsedURL, err := giturls.Parse(fullURL)
if err != nil {
return "", err
}
hostUrl := splitUrl[2] // github.com, gitlab.com, etc.
repository := splitUrl[3] + "/" + strings.Split(splitUrl[4], ".")[0] // user/reposetory
return hostUrl, repository, nil
return parsedURL.Host, nil
}
func getRepository(url string) (IRepository, error) {
hostUrl, repoName, err := getHostAndRepoName(url)
func getRepository(fullURL string) (IRepository, error) {
hostUrl, err := getHost(fullURL)
if err != nil {
return nil, err
}
var repo IRepository
switch repoHost := strings.Split(hostUrl, ".")[0]; repoHost {
case "github":
repo = NewGitHubRepository(repoName)
switch hostUrl {
case "github.com":
repo = NewGitHubRepository()
case "raw.githubusercontent.com":
repo = NewGitHubRepository()
repo.setIsFile(true)
default:
return nil, fmt.Errorf("unknown repository host: %s", repoHost)
return nil, fmt.Errorf("unknown repository host: %s", hostUrl)
}
// Returns the host-url, and the part of the user and repository from the url
return repo, nil
}
func (g *GitHubRepository) parse(fullURL string) error {
parsedURL, err := giturls.Parse(fullURL)
if err != nil {
return err
}
index := 0
splittedRepo := strings.FieldsFunc(parsedURL.Path, func(c rune) bool { return c == '/' })
if len(splittedRepo) < 2 {
return fmt.Errorf("expecting <user>/<repo> in url path, received: '%s'", parsedURL.Path)
}
g.owner = splittedRepo[index]
index += 1
g.repo = splittedRepo[index]
index += 1
// root of repo
if len(splittedRepo) < index+1 {
return nil
}
// is file or dir
switch splittedRepo[index] {
case "blob":
g.isFile = true
index += 1
case "tree":
g.isFile = false
index += 1
}
if len(splittedRepo) < index+1 {
return nil
}
g.branch = splittedRepo[index]
index += 1
if len(splittedRepo) < index+1 {
return nil
}
g.path = strings.Join(splittedRepo[index:], "/")
return nil
}
func (g *GitHubRepository) getBranch() string { return g.branch }
func (g *GitHubRepository) getTree() tree { return g.tree }
func (g *GitHubRepository) setIsFile(isFile bool) { g.isFile = isFile }
func (g *GitHubRepository) getIsFile() bool { return g.isFile }
func (g *GitHubRepository) setBranch(branchOptional string) error {
// Checks whether the repository type is a master or another type.
// By default it is "master", unless the branchOptional came with a value
if branchOptional == "" {
body, err := getter.HttpGetter(&http.Client{}, g.defaultBranchAPI(), nil)
if err != nil {
return err
}
var data githubDefaultBranchAPI
err = json.Unmarshal([]byte(body), &data)
if err != nil {
return err
}
g.branch = data.DefaultBranch
} else {
if branchOptional != "" {
g.branch = branchOptional
}
if g.branch != "" {
return nil
}
body, err := getter.HttpGetter(&http.Client{}, g.defaultBranchAPI(), nil)
if err != nil {
return err
}
var data githubDefaultBranchAPI
err = json.Unmarshal([]byte(body), &data)
if err != nil {
return err
}
g.branch = data.DefaultBranch
return nil
}
func joinOwnerNRepo(owner, repo string) string {
return fmt.Sprintf("%s/%s", owner, repo)
}
func (g *GitHubRepository) defaultBranchAPI() string {
return fmt.Sprintf("https://api.github.com/repos/%s", g.name)
return fmt.Sprintf("https://api.github.com/repos/%s", joinOwnerNRepo(g.owner, g.repo))
}
func (g *GitHubRepository) setTree() error {
@@ -135,14 +205,24 @@ func (g *GitHubRepository) setTree() error {
}
func (g *GitHubRepository) treeAPI() string {
return fmt.Sprintf("https://api.github.com/repos/%s/git/trees/%s?recursive=1", g.name, g.branch)
return fmt.Sprintf("https://api.github.com/repos/%s/git/trees/%s?recursive=1", joinOwnerNRepo(g.owner, g.repo), g.branch)
}
// return a list of yaml for a given repository tree
func (g *GitHubRepository) getYamlFromTree() []string {
func (g *GitHubRepository) getFilesFromTree(filesExtensions []string) []string {
var urls []string
if g.isFile {
if slices.Contains(filesExtensions, getFileExtension(g.path)) {
return []string{fmt.Sprintf("%s/%s", g.rowYamlUrl(), g.path)}
} else {
return []string{}
}
}
for _, path := range g.tree.InnerTrees {
if strings.HasSuffix(path.Path, ".yaml") {
if g.path != "" && !strings.HasPrefix(path.Path, g.path) {
continue
}
if slices.Contains(filesExtensions, getFileExtension(path.Path)) {
urls = append(urls, fmt.Sprintf("%s/%s", g.rowYamlUrl(), path.Path))
}
}
@@ -150,5 +230,9 @@ func (g *GitHubRepository) getYamlFromTree() []string {
}
func (g *GitHubRepository) rowYamlUrl() string {
return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s", g.name, g.branch)
return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s", joinOwnerNRepo(g.owner, g.repo), g.branch)
}
func getFileExtension(path string) string {
return strings.TrimPrefix(filepath.Ext(path), ".")
}

View File

@@ -0,0 +1,137 @@
package resourcehandler
import (
"testing"
"github.com/stretchr/testify/assert"
)
var (
urlA = "https://github.com/armosec/kubescape"
urlB = "https://github.com/armosec/kubescape/blob/master/examples/online-boutique/adservice.yaml"
urlC = "https://github.com/armosec/kubescape/tree/master/examples/online-boutique"
// urlD = "https://raw.githubusercontent.com/armosec/kubescape/master/examples/online-boutique/adservice.yaml"
)
func TestScanRepository(t *testing.T) {
{
files, err := ScanRepository(urlA, "")
assert.NoError(t, err)
assert.Less(t, 0, len(files))
}
{
files, err := ScanRepository(urlB, "")
assert.NoError(t, err)
assert.Less(t, 0, len(files))
}
{
files, err := ScanRepository(urlC, "")
assert.NoError(t, err)
assert.Less(t, 0, len(files))
}
// {
// files, err := ScanRepository(urlD, "")
// assert.NoError(t, err)
// assert.Equal(t, 1, len(files))
// }
}
func TestGetHost(t *testing.T) {
{
host, err := getHost(urlA)
assert.NoError(t, err)
assert.Equal(t, "github.com", host)
}
{
host, err := getHost(urlB)
assert.NoError(t, err)
assert.Equal(t, "github.com", host)
}
{
host, err := getHost(urlC)
assert.NoError(t, err)
assert.Equal(t, "github.com", host)
}
}
// repo = NewGitHubRepository(repoName)
func TestGithubSetBranch(t *testing.T) {
{
gh := NewGitHubRepository()
assert.NoError(t, gh.parse(urlA))
assert.NoError(t, gh.setBranch(""))
assert.Equal(t, "master", gh.getBranch())
}
{
gh := NewGitHubRepository()
assert.NoError(t, gh.parse(urlB))
err := gh.setBranch("dev")
assert.NoError(t, err)
assert.Equal(t, "dev", gh.getBranch())
}
}
func TestGithubSetTree(t *testing.T) {
{
gh := NewGitHubRepository()
assert.NoError(t, gh.parse(urlA))
assert.NoError(t, gh.setBranch(""))
err := gh.setTree()
assert.NoError(t, err)
assert.Less(t, 0, len(gh.getTree().InnerTrees))
}
}
func TestGithubGetYamlFromTree(t *testing.T) {
{
gh := NewGitHubRepository()
assert.NoError(t, gh.parse(urlA))
assert.NoError(t, gh.setBranch(""))
assert.NoError(t, gh.setTree())
files := gh.getFilesFromTree([]string{"yaml"})
assert.Less(t, 0, len(files))
}
{
gh := NewGitHubRepository()
assert.NoError(t, gh.parse(urlB))
assert.NoError(t, gh.setBranch(""))
assert.NoError(t, gh.setTree())
files := gh.getFilesFromTree([]string{"yaml"})
assert.Equal(t, 1, len(files))
}
{
gh := NewGitHubRepository()
assert.NoError(t, gh.parse(urlC))
assert.NoError(t, gh.setBranch(""))
assert.NoError(t, gh.setTree())
files := gh.getFilesFromTree([]string{"yaml"})
assert.Equal(t, 12, len(files))
}
}
func TestGithubParse(t *testing.T) {
{
gh := NewGitHubRepository()
assert.NoError(t, gh.parse(urlA))
assert.Equal(t, "armosec/kubescape", joinOwnerNRepo(gh.owner, gh.repo))
}
{
gh := NewGitHubRepository()
assert.NoError(t, gh.parse(urlB))
assert.Equal(t, "armosec/kubescape", joinOwnerNRepo(gh.owner, gh.repo))
assert.Equal(t, "master", gh.branch)
assert.Equal(t, "examples/online-boutique/adservice.yaml", gh.path)
assert.True(t, gh.isFile)
assert.Equal(t, 1, len(gh.getFilesFromTree([]string{"yaml"})))
assert.Equal(t, 1, len(gh.getFilesFromTree([]string{"yml"})))
}
{
gh := NewGitHubRepository()
assert.NoError(t, gh.parse(urlC))
assert.Equal(t, "armosec/kubescape", joinOwnerNRepo(gh.owner, gh.repo))
assert.Equal(t, "master", gh.branch)
assert.Equal(t, "examples/online-boutique", gh.path)
assert.False(t, gh.isFile)
}
}

View File

@@ -12,7 +12,7 @@ import (
"github.com/armosec/kubescape/core/cautils/logger"
)
func loadResourcesFromUrl(inputPatterns []string) ([]workloadinterface.IMetadata, error) {
func loadResourcesFromUrl(inputPatterns []string) (map[string][]workloadinterface.IMetadata, error) {
urls := listUrls(inputPatterns)
if len(urls) == 0 {
return nil, nil
@@ -29,14 +29,10 @@ func listUrls(patterns []string) []string {
urls := []string{}
for i := range patterns {
if strings.HasPrefix(patterns[i], "http") {
if !cautils.IsYaml(patterns[i]) && !cautils.IsJson(patterns[i]) { // if url of repo
if yamls, err := ScanRepository(patterns[i], ""); err == nil { // TODO - support branch
urls = append(urls, yamls...)
} else {
logger.L().Error(err.Error())
}
} else { // url of single file
urls = append(urls, patterns[i])
if yamls, err := ScanRepository(patterns[i], ""); err == nil { // TODO - support branch
urls = append(urls, yamls...)
} else {
logger.L().Error(err.Error())
}
}
}
@@ -44,8 +40,8 @@ func listUrls(patterns []string) []string {
return urls
}
func downloadFiles(urls []string) ([]workloadinterface.IMetadata, []error) {
workloads := []workloadinterface.IMetadata{}
func downloadFiles(urls []string) (map[string][]workloadinterface.IMetadata, []error) {
workloads := make(map[string][]workloadinterface.IMetadata, 0)
errs := []error{}
for i := range urls {
f, err := downloadFile(urls[i])
@@ -56,7 +52,12 @@ func downloadFiles(urls []string) ([]workloadinterface.IMetadata, []error) {
w, e := cautils.ReadFile(f, cautils.GetFileFormat(urls[i]))
errs = append(errs, e...)
if w != nil {
workloads = append(workloads, w...)
if _, ok := workloads[urls[i]]; !ok {
workloads[urls[i]] = make([]workloadinterface.IMetadata, 0)
}
wSlice := workloads[urls[i]]
wSlice = append(wSlice, w...)
workloads[urls[i]] = wSlice
}
}
return workloads, errs

View File

@@ -0,0 +1,66 @@
package resourcehandler
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestLoadResourcesFromUrl(t *testing.T) {
{
workloads, err := loadResourcesFromUrl([]string{"https://github.com/armosec/kubescape/tree/master/examples/online-boutique"})
assert.NoError(t, err)
assert.Equal(t, 12, len(workloads))
for i, w := range workloads {
switch i {
case "https://raw.githubusercontent.com/armosec/kubescape/master/examples/online-boutique/adservice.yaml":
assert.Equal(t, 2, len(w))
assert.Equal(t, "apps/v1//Deployment/adservice", w[0].GetID())
assert.Equal(t, "/v1//Service/adservice", w[1].GetID())
}
}
}
{
workloads, err := loadResourcesFromUrl([]string{"https://github.com/armosec/kubescape"})
assert.NoError(t, err)
assert.Less(t, 12, len(workloads))
for i, w := range workloads {
switch i {
case "https://raw.githubusercontent.com/armosec/kubescape/master/examples/online-boutique/adservice.yaml":
assert.Equal(t, 2, len(w))
assert.Equal(t, "apps/v1//Deployment/adservice", w[0].GetID())
assert.Equal(t, "/v1//Service/adservice", w[1].GetID())
}
}
}
{
workloads, err := loadResourcesFromUrl([]string{"https://github.com/armosec/kubescape/blob/master/examples/online-boutique/adservice.yaml"})
assert.NoError(t, err)
assert.Equal(t, 1, len(workloads))
for i, w := range workloads {
switch i {
case "https://raw.githubusercontent.com/armosec/kubescape/master/examples/online-boutique/adservice.yaml":
assert.Equal(t, 2, len(w))
assert.Equal(t, "apps/v1//Deployment/adservice", w[0].GetID())
assert.Equal(t, "/v1//Service/adservice", w[1].GetID())
}
}
}
{
workloads, err := loadResourcesFromUrl([]string{"https://raw.githubusercontent.com/armosec/kubescape/master/examples/online-boutique/adservice.yaml"})
assert.NoError(t, err)
assert.Equal(t, 1, len(workloads))
for i, w := range workloads {
switch i {
case "https://raw.githubusercontent.com/armosec/kubescape/master/examples/online-boutique/adservice.yaml":
assert.Equal(t, 2, len(w))
assert.Equal(t, "apps/v1//Deployment/adservice", w[0].GetID())
assert.Equal(t, "/v1//Service/adservice", w[1].GetID())
}
}
}
}

View File

@@ -32,7 +32,6 @@ func (jsonPrinter *JsonPrinter) ActionPrint(opaSessionObj *cautils.OPASessionObj
if err != nil {
logger.L().Fatal("failed to Marshal posture report object")
}
jsonPrinter.writer.Write(r)
logOUtputFile(jsonPrinter.writer.Name())
if _, err := jsonPrinter.writer.Write(r); err != nil {

View File

@@ -26,8 +26,7 @@ func DataToJson(data *cautils.OPASessionObj) *reporthandlingv2.PostureReport {
report.Results = make([]resourcesresults.Result, len(data.ResourcesResult))
finalizeResults(report.Results, data.ResourcesResult)
report.Resources = make([]reporthandling.Resource, 0) // do not initialize slice length
finalizeResources(report.Resources, report.Results, data.AllResources)
report.Resources = finalizeResources(report.Results, data.AllResources)
return &report
}
@@ -53,7 +52,8 @@ func mapInfoToPrintInfo(controls reportsummary.ControlSummaries) map[string]stri
return infoToPrintInfoMap
}
func finalizeResources(resources []reporthandling.Resource, results []resourcesresults.Result, allResources map[string]workloadinterface.IMetadata) {
func finalizeResources(results []resourcesresults.Result, allResources map[string]workloadinterface.IMetadata) []reporthandling.Resource {
resources := make([]reporthandling.Resource, 0)
for i := range results {
if obj, ok := allResources[results[i].ResourceID]; ok {
r := *reporthandling.NewResource(obj.GetObject())
@@ -61,6 +61,7 @@ func finalizeResources(resources []reporthandling.Resource, results []resourcesr
resources = append(resources, r)
}
}
return resources
}
func logOUtputFile(fileName string) {

View File

@@ -7,4 +7,5 @@ type IReport interface {
SetCustomerGUID(customerGUID string)
SetClusterName(clusterName string)
DisplayReportURL()
GetURL() string
}

View File

@@ -157,6 +157,9 @@ func (report *ReportEventReceiver) generateMessage() {
report.message = fmt.Sprintf("%s %s", message, u.String())
}
func (report *ReportEventReceiver) GetURL() string {
return getter.GetArmoAPIConnector().GetFrontendURL()
}
func (report *ReportEventReceiver) DisplayReportURL() {
cautils.InfoTextDisplay(os.Stderr, fmt.Sprintf("\n\n%s\n\n", report.message))
}

View File

@@ -31,6 +31,10 @@ func (reportMock *ReportMock) SetCustomerGUID(customerGUID string) {
func (reportMock *ReportMock) SetClusterName(clusterName string) {
}
func (reportMock *ReportMock) GetURL() string {
return getter.GetArmoAPIConnector().GetFrontendURL()
}
func (reportMock *ReportMock) DisplayReportURL() {
u := fmt.Sprintf("https://%s/account/login", getter.GetArmoAPIConnector().GetFrontendURL())
if reportMock.query != "" {

View File

@@ -47,7 +47,6 @@ func (report *ReportEventReceiver) ActionSendReport(opaSessionObj *cautils.OPASe
if report.customerGUID == "" {
logger.L().Warning("failed to publish results. Reason: Unknown accout ID. Run kubescape with the '--account <account ID>' flag. Contact ARMO team for more details")
return nil
}
if report.clusterName == "" {
@@ -62,14 +61,17 @@ func (report *ReportEventReceiver) ActionSendReport(opaSessionObj *cautils.OPASe
opaSessionObj.Report.ClusterName = report.clusterName
opaSessionObj.Report.Metadata = *opaSessionObj.Metadata
if err := report.prepareReport(opaSessionObj.Report); err != nil {
logger.L().Error("failed to publish results", helpers.Error(err))
} else {
err := report.prepareReport(opaSessionObj.Report)
if err == nil {
report.generateMessage()
} else {
logger.L().Debug(err.Error()) // print original error only in debug mode
err = fmt.Errorf("failed to submit scan results. url: '%s'", report.GetURL())
}
logger.L().Debug("", helpers.String("account ID", report.customerGUID))
return nil
return err
}
func (report *ReportEventReceiver) SetCustomerGUID(customerGUID string) {
@@ -95,6 +97,24 @@ func (report *ReportEventReceiver) prepareReport(postureReport *reporthandlingv2
return err
}
func (report *ReportEventReceiver) GetURL() string {
u := url.URL{}
u.Scheme = "https"
u.Host = getter.GetArmoAPIConnector().GetFrontendURL()
if report.customerAdminEMail != "" || report.token == "" { // data has been submitted
u.Path = fmt.Sprintf("configuration-scanning/%s", report.clusterName)
} else {
u.Path = "account/sign-up"
q := u.Query()
q.Add("invitationToken", report.token)
q.Add("customerGUID", report.customerGUID)
u.RawQuery = q.Encode()
}
return u.String()
}
func (report *ReportEventReceiver) sendResources(host string, postureReport *reporthandlingv2.PostureReport, reportCounter *int, isLastReport bool) error {
splittedPostureReport := setSubReport(postureReport)
counter := 0
@@ -171,26 +191,12 @@ func (report *ReportEventReceiver) sendReport(host string, postureReport *report
}
func (report *ReportEventReceiver) generateMessage() {
u := url.URL{}
u.Scheme = "https"
u.Host = getter.GetArmoAPIConnector().GetFrontendURL()
if report.customerAdminEMail != "" || report.token == "" { // data has been submitted
u.Path = fmt.Sprintf("configuration-scanning/%s", report.clusterName)
} else {
u.Path = "account/sign-up"
q := u.Query()
q.Add("invitationToken", report.token)
q.Add("customerGUID", report.customerGUID)
u.RawQuery = q.Encode()
}
report.message = ""
sep := "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"
report.message = sep
report.message += " << WOW! Now you can see the scan results on the web >>\n\n"
report.message += fmt.Sprintf(" %s\n", u.String())
report.message += fmt.Sprintf(" %s\n", report.GetURL())
report.message += sep
}

View File

@@ -44,6 +44,7 @@ func setSubReport(postureReport *reporthandlingv2.PostureReport) *reporthandling
ClusterCloudProvider: postureReport.ClusterCloudProvider,
JobID: postureReport.JobID,
ClusterAPIServerInfo: postureReport.ClusterAPIServerInfo,
Metadata: postureReport.Metadata,
}
}
func iMetaToResource(obj workloadinterface.IMetadata) *reporthandling.Resource {

View File

@@ -15,13 +15,10 @@ func finalizeReport(opaSessionObj *cautils.OPASessionObj) {
if len(opaSessionObj.Report.Results) == 0 {
opaSessionObj.Report.Results = make([]resourcesresults.Result, len(opaSessionObj.ResourcesResult))
finalizeResults(opaSessionObj.Report.Results, opaSessionObj.ResourcesResult)
opaSessionObj.ResourcesResult = nil
}
if len(opaSessionObj.Report.Resources) == 0 {
opaSessionObj.Report.Resources = make([]reporthandling.Resource, 0) // do not set slice length
finalizeResources(opaSessionObj.Report.Resources, opaSessionObj.AllResources)
opaSessionObj.AllResources = nil
opaSessionObj.Report.Resources = finalizeResources(opaSessionObj.AllResources)
}
}
@@ -33,7 +30,8 @@ func finalizeResults(results []resourcesresults.Result, resourcesResult map[stri
}
}
func finalizeResources(resources []reporthandling.Resource, allResources map[string]workloadinterface.IMetadata) {
func finalizeResources(allResources map[string]workloadinterface.IMetadata) []reporthandling.Resource {
resources := make([]reporthandling.Resource, 0)
for resourceID := range allResources {
if obj, ok := allResources[resourceID]; ok {
r := *reporthandling.NewResource(obj.GetObject())
@@ -41,6 +39,7 @@ func finalizeResources(resources []reporthandling.Resource, allResources map[str
resources = append(resources, r)
}
}
return resources
}
func maskID(id string) string {

BIN
docs/ksfromcodetodeploy.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 967 KiB

View File

@@ -6,6 +6,7 @@ replace github.com/armosec/kubescape/core => ../core
require (
github.com/armosec/kubescape/core v0.0.0-00010101000000-000000000000
github.com/armosec/opa-utils v0.0.125
github.com/armosec/utils-go v0.0.3
github.com/google/uuid v1.3.0
github.com/gorilla/mux v1.8.0
@@ -24,7 +25,6 @@ require (
github.com/OneOfOne/xxhash v1.2.8 // indirect
github.com/armosec/armoapi-go v0.0.58 // indirect
github.com/armosec/k8s-interface v0.0.68 // indirect
github.com/armosec/opa-utils v0.0.120 // indirect
github.com/armosec/rbac-utils v0.0.14 // indirect
github.com/armosec/utils-k8s-go v0.0.3 // indirect
github.com/aws/aws-sdk-go v1.41.11 // indirect

View File

@@ -109,8 +109,8 @@ github.com/armosec/k8s-interface v0.0.66/go.mod h1:vwprS8qn/iowd5yf0JHpqDsLA5I8W
github.com/armosec/k8s-interface v0.0.68 h1:6CtSakISiI47YHkxh+Va9FzZQIBkWa6g9sbiNxq1Zkk=
github.com/armosec/k8s-interface v0.0.68/go.mod h1:PeWn41C2uenZi+xfZdyFF/zG5wXACA00htQyknDUWDE=
github.com/armosec/opa-utils v0.0.64/go.mod h1:6tQP8UDq2EvEfSqh8vrUdr/9QVSCG4sJfju1SXQOn4c=
github.com/armosec/opa-utils v0.0.120 h1:WAtgm2U1o9fgA/2pjYNy+igqNC6ju3/CxQ8qRHdO+5k=
github.com/armosec/opa-utils v0.0.120/go.mod h1:gap+EaLG5rnyqvIRGxtdNDC9y7VvoGNm90zK8Ls7avQ=
github.com/armosec/opa-utils v0.0.125 h1:ZA7v3kvqITwazhFkvZuGP/bc/VChsYWQdrDcvyf2qsw=
github.com/armosec/opa-utils v0.0.125/go.mod h1:gap+EaLG5rnyqvIRGxtdNDC9y7VvoGNm90zK8Ls7avQ=
github.com/armosec/rbac-utils v0.0.1/go.mod h1:pQ8CBiij8kSKV7aeZm9FMvtZN28VgA7LZcYyTWimq40=
github.com/armosec/rbac-utils v0.0.14 h1:CKYKcgqJEXWF2Hen/B1pVGtS3nDAG1wp9dDv6oNtq90=
github.com/armosec/rbac-utils v0.0.14/go.mod h1:Ex/IdGWhGv9HZq6Hs8N/ApzCKSIvpNe/ETqDfnuyah0=