From 08f04e19efeeda85b6d3479acab1f8955ee5bd95 Mon Sep 17 00:00:00 2001 From: Bezalel Brandwine Date: Wed, 13 Oct 2021 12:24:21 +0300 Subject: [PATCH 01/17] fix download framework (create dir if necessary, lower case "download") --- cautils/getter/downloadreleasedpolicy.go | 3 +- cautils/getter/getpoliciesutils.go | 46 +++++++++--------------- cmd/download.go | 2 +- 3 files changed, 19 insertions(+), 32 deletions(-) diff --git a/cautils/getter/downloadreleasedpolicy.go b/cautils/getter/downloadreleasedpolicy.go index 1740888a..cad48259 100644 --- a/cautils/getter/downloadreleasedpolicy.go +++ b/cautils/getter/downloadreleasedpolicy.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/http" + "time" "github.com/armosec/kubescape/cautils/opapolicy" ) @@ -22,7 +23,7 @@ type DownloadReleasedPolicy struct { func NewDownloadReleasedPolicy() *DownloadReleasedPolicy { return &DownloadReleasedPolicy{ hostURL: "", - httpClient: &http.Client{}, + httpClient: &http.Client{Timeout: 61 * time.Second}, } } diff --git a/cautils/getter/getpoliciesutils.go b/cautils/getter/getpoliciesutils.go index 8eba7a87..05259f5f 100644 --- a/cautils/getter/getpoliciesutils.go +++ b/cautils/getter/getpoliciesutils.go @@ -5,8 +5,8 @@ import ( "fmt" "io" "net/http" - "net/url" "os" + "path" "path/filepath" "strings" @@ -21,14 +21,26 @@ func GetDefaultPath(name string) string { return defaultfilePath } -func SaveFrameworkInFile(framework *opapolicy.Framework, path string) error { +func SaveFrameworkInFile(framework *opapolicy.Framework, pathStr string) error { encodedData, err := json.Marshal(framework) if err != nil { return err } - err = os.WriteFile(path, []byte(fmt.Sprintf("%v", string(encodedData))), 0644) + err = os.WriteFile(pathStr, []byte(fmt.Sprintf("%v", string(encodedData))), 0644) if err != nil { - return err + if os.IsNotExist(err) { + pathDir := path.Dir(pathStr) + if err := os.Mkdir(pathDir, 0744); err != nil { + return err + } + } else { + return err + + } + err = os.WriteFile(pathStr, []byte(fmt.Sprintf("%v", string(encodedData))), 0644) + if err != nil { + return err + } } return nil } @@ -86,29 +98,3 @@ func httpRespToString(resp *http.Response) (string, error) { return respStr, err } - -// URLEncoder encode url -func urlEncoder(oldURL string) string { - fullURL := strings.Split(oldURL, "?") - baseURL, err := url.Parse(fullURL[0]) - if err != nil { - return "" - } - - // Prepare Query Parameters - if len(fullURL) > 1 { - params := url.Values{} - queryParams := strings.Split(fullURL[1], "&") - for _, i := range queryParams { - queryParam := strings.Split(i, "=") - val := "" - if len(queryParam) > 1 { - val = queryParam[1] - } - params.Add(queryParam[0], val) - } - baseURL.RawQuery = params.Encode() - } - - return baseURL.String() -} diff --git a/cmd/download.go b/cmd/download.go index a27123ac..550c42f8 100644 --- a/cmd/download.go +++ b/cmd/download.go @@ -11,7 +11,7 @@ import ( var downloadInfo cautils.DownloadInfo var downloadCmd = &cobra.Command{ - Use: fmt.Sprintf("Download framework [flags]\nSupported frameworks: %s", validFrameworks), + Use: fmt.Sprintf("download framework [flags]\nSupported frameworks: %s", validFrameworks), Short: "Download framework controls", Long: ``, Args: func(cmd *cobra.Command, args []string) error { From a922d01005fd3cf97f38888194005644d73ca336 Mon Sep 17 00:00:00 2001 From: Bezalel Brandwine Date: Wed, 13 Oct 2021 16:45:06 +0300 Subject: [PATCH 02/17] add support in --environment=dev or customized URLs --- cautils/customerloader.go | 4 +- cautils/getter/armoapi.go | 72 +++++++++++++++++-- cautils/getter/armoapiutils.go | 6 +- cautils/scaninfo.go | 2 +- cmd/cluster_get.go | 2 +- cmd/cluster_set.go | 2 +- cmd/framework.go | 6 +- cmd/root.go | 41 ++++++++++- cmd/scan.go | 2 + main.go | 3 + .../reporter/reporteventreceiverutils.go | 2 +- 11 files changed, 122 insertions(+), 20 deletions(-) diff --git a/cautils/customerloader.go b/cautils/customerloader.go index 0eef7348..c73c24a8 100644 --- a/cautils/customerloader.go +++ b/cautils/customerloader.go @@ -111,7 +111,7 @@ func (c *EmptyConfig) GetK8sAPI() *k8sinterface.KubernetesApi { return nil } func (c *EmptyConfig) GetDefaultNS() string { return k8sinterface.GetDefaultNamespace() } func (c *EmptyConfig) GetBackendAPI() getter.IBackend { return nil } // TODO: return mock obj func (c *EmptyConfig) GenerateURL() { - message := fmt.Sprintf("You can see the results in a user-friendly UI, choose your preferred compliance framework, check risk results history and trends, manage exceptions, get remediation recommendations and much more by registering here: https://%s", getter.ArmoFEURL) + message := fmt.Sprintf("You can see the results in a user-friendly UI, choose your preferred compliance framework, check risk results history and trends, manage exceptions, get remediation recommendations and much more by registering here: https://%s", getter.GetArmoAPIConnector().GetFrontendURL()) InfoTextDisplay(os.Stdout, message+"\n") } @@ -143,7 +143,7 @@ func (c *ClusterConfig) GenerateURL() { u := url.URL{} u.Scheme = "https" - u.Host = getter.ArmoFEURL + u.Host = getter.GetArmoAPIConnector().GetFrontendURL() if c.configObj == nil { return } diff --git a/cautils/getter/armoapi.go b/cautils/getter/armoapi.go index dd06be4c..3a4b1d90 100644 --- a/cautils/getter/armoapi.go +++ b/cautils/getter/armoapi.go @@ -3,9 +3,11 @@ package getter import ( "fmt" "net/http" + "time" "github.com/armosec/kubescape/cautils/armotypes" "github.com/armosec/kubescape/cautils/opapolicy" + "github.com/golang/glog" ) // ======================================================================================================================= @@ -15,22 +17,80 @@ import ( var ( // ATTENTION!!! // Changes in this URLs variable names, or in the usage is affecting the build process! BE CAREFULL - ArmoBEURL = "eggdashbe.eudev3.cyberarmorsoft.com" - ArmoERURL = "report.eudev3.cyberarmorsoft.com" - ArmoFEURL = "armoui.eudev3.cyberarmorsoft.com" - // ArmoURL = "https://dashbe.euprod1.cyberarmorsoft.com" + armoERURL = "report.armo.cloud" + armoBEURL = "api.armo.cloud" + armoFEURL = "portal.armo.cloud" + + armoDevERURL = "report.eudev3.cyberarmorsoft.com" + armoDevBEURL = "eggdashbe.eudev3.cyberarmorsoft.com" + armoDevFEURL = "armoui.eudev3.cyberarmorsoft.com" ) // Armo API for downloading policies type ArmoAPI struct { httpClient *http.Client + apiURL string + erURL string + feURL string } -func NewArmoAPI() *ArmoAPI { +var globalArmoAPIConnecctor *ArmoAPI + +func SetARMOAPIConnector(armoAPI *ArmoAPI) { + globalArmoAPIConnecctor = armoAPI +} + +func GetArmoAPIConnector() *ArmoAPI { + if globalArmoAPIConnecctor == nil { + glog.Error("returning nil API connector") + } + return globalArmoAPIConnecctor +} + +func NewARMOAPIDev() *ArmoAPI { + apiObj := newArmoAPI() + + apiObj.apiURL = armoDevBEURL + apiObj.erURL = armoDevERURL + apiObj.feURL = armoDevFEURL + + return apiObj +} + +func NewARMOAPIProd() *ArmoAPI { + apiObj := newArmoAPI() + + apiObj.apiURL = armoBEURL + apiObj.erURL = armoERURL + apiObj.feURL = armoFEURL + + return apiObj +} + +func NewARMOAPICustomized(armoERURL, armoBEURL, armoFEURL string) *ArmoAPI { + apiObj := newArmoAPI() + + apiObj.erURL = armoERURL + apiObj.apiURL = armoBEURL + apiObj.feURL = armoFEURL + + return apiObj +} + +func newArmoAPI() *ArmoAPI { return &ArmoAPI{ - httpClient: &http.Client{}, + httpClient: &http.Client{Timeout: time.Duration(61) * time.Second}, } } + +func (armoAPI *ArmoAPI) GetFrontendURL() string { + return armoAPI.feURL +} + +func (armoAPI *ArmoAPI) GetReportReceiverURL() string { + return armoAPI.erURL +} + func (armoAPI *ArmoAPI) GetFramework(name string) (*opapolicy.Framework, error) { respStr, err := HttpGetter(armoAPI.httpClient, armoAPI.getFrameworkURL(name)) if err != nil { diff --git a/cautils/getter/armoapiutils.go b/cautils/getter/armoapiutils.go index d54264fd..001a6540 100644 --- a/cautils/getter/armoapiutils.go +++ b/cautils/getter/armoapiutils.go @@ -8,7 +8,7 @@ import ( func (armoAPI *ArmoAPI) getFrameworkURL(frameworkName string) string { u := url.URL{} u.Scheme = "https" - u.Host = ArmoBEURL + u.Host = armoAPI.apiURL u.Path = "v1/armoFrameworks" q := u.Query() q.Add("customerGUID", "11111111-1111-1111-1111-111111111111") @@ -22,7 +22,7 @@ func (armoAPI *ArmoAPI) getFrameworkURL(frameworkName string) string { func (armoAPI *ArmoAPI) getExceptionsURL(customerGUID, clusterName string) string { u := url.URL{} u.Scheme = "https" - u.Host = ArmoBEURL + u.Host = armoAPI.apiURL u.Path = "api/v1/armoPostureExceptions" q := u.Query() @@ -38,7 +38,7 @@ func (armoAPI *ArmoAPI) getExceptionsURL(customerGUID, clusterName string) strin func (armoAPI *ArmoAPI) getCustomerURL() string { u := url.URL{} u.Scheme = "https" - u.Host = ArmoBEURL + u.Host = armoAPI.apiURL u.Path = "api/v1/createTenant" return u.String() } diff --git a/cautils/scaninfo.go b/cautils/scaninfo.go index 5d8607dd..53d8db42 100644 --- a/cautils/scaninfo.go +++ b/cautils/scaninfo.go @@ -43,7 +43,7 @@ func (scanInfo *ScanInfo) setUseExceptions() { // load exceptions from file scanInfo.ExceptionsGetter = getter.NewLoadPolicy(scanInfo.UseExceptions) } else { - scanInfo.ExceptionsGetter = getter.NewArmoAPI() + scanInfo.ExceptionsGetter = getter.GetArmoAPIConnector() } } diff --git a/cmd/cluster_get.go b/cmd/cluster_get.go index 83915524..649e4e4f 100644 --- a/cmd/cluster_get.go +++ b/cmd/cluster_get.go @@ -31,7 +31,7 @@ var getCmd = &cobra.Command{ key := keyValue[0] k8s := k8sinterface.NewKubernetesApi() - clusterConfig := cautils.NewClusterConfig(k8s, getter.NewArmoAPI()) + clusterConfig := cautils.NewClusterConfig(k8s, getter.GetArmoAPIConnector()) val, err := clusterConfig.GetValueByKeyFromConfigMap(key) if err != nil { if err.Error() == "value does not exist." { diff --git a/cmd/cluster_set.go b/cmd/cluster_set.go index f7c94b84..e0315a6e 100644 --- a/cmd/cluster_set.go +++ b/cmd/cluster_set.go @@ -30,7 +30,7 @@ var setCmd = &cobra.Command{ data := keyValue[1] k8s := k8sinterface.NewKubernetesApi() - clusterConfig := cautils.NewClusterConfig(k8s, getter.NewArmoAPI()) + clusterConfig := cautils.NewClusterConfig(k8s, getter.GetArmoAPIConnector()) if err := clusterConfig.SetKeyValueInConfigmap(key, data); err != nil { return err } diff --git a/cmd/framework.go b/cmd/framework.go index 6ac188a6..f60e1051 100644 --- a/cmd/framework.go +++ b/cmd/framework.go @@ -1,7 +1,6 @@ package cmd import ( - "flag" "fmt" "io" "os" @@ -32,7 +31,7 @@ type CLIHandler struct { var frameworkCmd = &cobra.Command{ - Use: fmt.Sprintf("framework [``/`-`] [flags]\nSupported frameworks: %s", validFrameworks), + Use: fmt.Sprintf("framework [``/`-`] [flags]\nSupported frameworks: %s", validFrameworks), Short: fmt.Sprintf("The framework you wish to use. Supported frameworks: %s", strings.Join(supportedFrameworks, ", ")), Long: "Execute a scan on a running Kubernetes cluster or `yaml`/`json` files (use glob) or `-` for stdin", ValidArgs: supportedFrameworks, @@ -103,7 +102,6 @@ func init() { } func CliSetup() error { - flag.Parse() flagValidation() var k8s *k8sinterface.KubernetesApi @@ -120,7 +118,7 @@ func CliSetup() error { policyHandler := policyhandler.NewPolicyHandler(&processNotification, k8s) // setup cluster config - clusterConfig := cautils.ClusterConfigSetup(&scanInfo, k8s, getter.NewArmoAPI()) + clusterConfig := cautils.ClusterConfigSetup(&scanInfo, k8s, getter.GetArmoAPIConnector()) if err := clusterConfig.SetCustomerGUID(scanInfo.Account); err != nil { fmt.Println(err) } diff --git a/cmd/root.go b/cmd/root.go index 88b9db54..c760e45a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,15 +1,22 @@ package cmd import ( + "flag" + "os" + "strings" + + "github.com/armosec/kubescape/cautils/getter" + "github.com/golang/glog" "github.com/spf13/cobra" ) var cfgFile string +var armoBEURLs = "" var rootCmd = &cobra.Command{ Use: "kubescape", Short: "Kubescape is a tool for testing Kubernetes security posture", - Long: `Kubescape is a tool for testing Kubernetes security posture based on NSA specifications.`, + Long: `Kubescape is a tool for testing Kubernetes security posture based on NSA \ MITRE ATT&CK® specifications.`, } func Execute() { @@ -17,9 +24,41 @@ func Execute() { } func init() { + flag.CommandLine.StringVar(&armoBEURLs, "environment", "", envFlagUsage) + rootCmd.PersistentFlags().StringVar(&armoBEURLs, "environment", "", envFlagUsage) + rootCmd.PersistentFlags().MarkHidden("environment") cobra.OnInitialize(initConfig) + } // initConfig reads in config file and ENV variables if set. func initConfig() { } + +func InitArmoBEConnector() { + if armoBEURLs == "" && rootCmd.Flag("environment") != nil { + armoBEURLs = rootCmd.Flag("environment").Value.String() + } + urlSlices := strings.Split(armoBEURLs, ",") + if len(urlSlices) > 3 { + glog.Errorf("Too many URLs") + os.Exit(1) + } + switch len(urlSlices) { + case 1: + switch urlSlices[0] { + case "dev": + getter.SetARMOAPIConnector(getter.NewARMOAPIDev()) + case "": + getter.SetARMOAPIConnector(getter.NewARMOAPIProd()) + default: + glog.Errorf("--environment flag usage: %s", envFlagUsage) + os.Exit(1) + } + case 2: + glog.Errorf("--environment flag usage: %s", envFlagUsage) + os.Exit(1) + case 3: + getter.SetARMOAPIConnector(getter.NewARMOAPICustomized(urlSlices[0], urlSlices[1], urlSlices[2])) + } +} diff --git a/cmd/scan.go b/cmd/scan.go index e421f852..ca14f8e3 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -4,6 +4,8 @@ import ( "github.com/spf13/cobra" ) +const envFlagUsage = "Send report results to specific URL. Format:,,.\n\t\tExample:report.armo.cloud,api.armo.cloud,portal.armo.cloud" + // scanCmd represents the scan command var scanCmd = &cobra.Command{ Use: "scan", diff --git a/main.go b/main.go index 114f855c..6565b817 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "flag" "fmt" "os" @@ -9,6 +10,8 @@ import ( func main() { CheckLatestVersion() + flag.Parse() + cmd.InitArmoBEConnector() cmd.Execute() } diff --git a/resultshandling/reporter/reporteventreceiverutils.go b/resultshandling/reporter/reporteventreceiverutils.go index ee26e203..96fa75be 100644 --- a/resultshandling/reporter/reporteventreceiverutils.go +++ b/resultshandling/reporter/reporteventreceiverutils.go @@ -37,7 +37,7 @@ func initEventReceiverURL() *url.URL { urlObj := url.URL{} urlObj.Scheme = "https" - urlObj.Host = getter.ArmoERURL + urlObj.Host = getter.GetArmoAPIConnector().GetReportReceiverURL() urlObj.Path = "/k8s/postureReport" q := urlObj.Query() q.Add("customerGUID", uuid.FromStringOrNil(cautils.CustomerGUID).String()) From 01f6a1e1c07f9647befac886d320d5607ae36097 Mon Sep 17 00:00:00 2001 From: dwertent Date: Wed, 13 Oct 2021 20:41:30 +0300 Subject: [PATCH 03/17] fixed counters --- cautils/opapolicy/datastructuresmethods.go | 6 ++---- opaprocessor/processorhandlerutils.go | 1 + resultshandling/printer/printresults.go | 11 +++-------- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/cautils/opapolicy/datastructuresmethods.go b/cautils/opapolicy/datastructuresmethods.go index 56835967..df436516 100644 --- a/cautils/opapolicy/datastructuresmethods.go +++ b/cautils/opapolicy/datastructuresmethods.go @@ -124,9 +124,7 @@ func (ruleReport *RuleReport) GetNumberOfFailedResources() int { sum := 0 for i := len(ruleReport.RuleResponses) - 1; i >= 0; i-- { if ruleReport.RuleResponses[i].GetSingleResultStatus() == "failed" { - //if !ruleReport.DeleteIfRedundantResponse(&ruleReport.RuleResponses[i], i) { sum += len(ruleReport.RuleResponses[i].AlertObject.K8SApiObjects) - //} } } return sum @@ -168,9 +166,9 @@ func removeResponse(slice []RuleResponse, index int) []RuleResponse { func (ruleReport *RuleReport) GetNumberOfWarningResources() int { sum := 0 - for i := range ruleReport.RuleResponses { + for i := len(ruleReport.RuleResponses) - 1; i >= 0; i-- { if ruleReport.RuleResponses[i].GetSingleResultStatus() == "warning" { - sum += 1 + sum += len(ruleReport.RuleResponses[i].AlertObject.K8SApiObjects) } } return sum diff --git a/opaprocessor/processorhandlerutils.go b/opaprocessor/processorhandlerutils.go index d08a8876..ddd361e3 100644 --- a/opaprocessor/processorhandlerutils.go +++ b/opaprocessor/processorhandlerutils.go @@ -98,6 +98,7 @@ func editRuleResponses(ruleResponses []opapolicy.RuleResponse) []opapolicy.RuleR // resource found -> remove from slice ruleResponses = removeFromSlice(ruleResponses, i) lenRuleResponses -= 1 + i -= 1 break } else { cleanRuleResponses(w) diff --git a/resultshandling/printer/printresults.go b/resultshandling/printer/printresults.go index e652e60e..39e89ebe 100644 --- a/resultshandling/printer/printresults.go +++ b/resultshandling/printer/printresults.go @@ -45,12 +45,7 @@ func calculatePostureScore(postureReport *opapolicy.PostureReport) float32 { totalFailed := 0 for _, frameworkReport := range postureReport.FrameworkReports { for _, controlReport := range frameworkReport.ControlReports { - for _, ruleReport := range controlReport.RuleReports { - for _, ruleResponses := range ruleReport.RuleResponses { - totalFailed += len(ruleResponses.AlertObject.K8SApiObjects) - totalFailed += len(ruleResponses.AlertObject.ExternalObjects) - } - } + totalFailed += controlReport.GetNumberOfFailedResources() totalResources += controlReport.GetNumberOfResources() } } @@ -107,7 +102,7 @@ func (printer *Printer) SummarySetup(postureReport *opapolicy.PostureReport) { printer.summary[cr.Name] = ControlSummary{ TotalResources: cr.GetNumberOfResources(), - TotalFailed: len(workloadsSummary) - cr.GetNumberOfWarningResources(), + TotalFailed: cr.GetNumberOfFailedResources(), TotalWarnign: cr.GetNumberOfWarningResources(), WorkloadSummary: mapResources, Description: cr.Description, @@ -133,7 +128,7 @@ func (printer *Printer) PrintResults() { func (printer *Printer) printSummary(controlName string, controlSummary *ControlSummary) { cautils.SimpleDisplay(printer.writer, "Summary - ") - cautils.SuccessDisplay(printer.writer, "Passed:%v ", controlSummary.TotalResources-controlSummary.TotalFailed) + cautils.SuccessDisplay(printer.writer, "Passed:%v ", controlSummary.TotalResources-controlSummary.TotalFailed-controlSummary.TotalWarnign) cautils.WarningDisplay(printer.writer, "Excluded:%v ", controlSummary.TotalWarnign) cautils.FailureDisplay(printer.writer, "Failed:%v ", controlSummary.TotalFailed) cautils.InfoDisplay(printer.writer, "Total:%v\n", controlSummary.TotalResources) From efa53bd83ca27f20a0cdba5ba36f01d79fa08cf2 Mon Sep 17 00:00:00 2001 From: dwertent Date: Thu, 14 Oct 2021 09:13:07 +0300 Subject: [PATCH 04/17] fixed loop --- cautils/opapolicy/datastructuresmethods.go | 38 +--------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/cautils/opapolicy/datastructuresmethods.go b/cautils/opapolicy/datastructuresmethods.go index df436516..a649aebd 100644 --- a/cautils/opapolicy/datastructuresmethods.go +++ b/cautils/opapolicy/datastructuresmethods.go @@ -3,8 +3,6 @@ package opapolicy import ( "bytes" "encoding/json" - - "github.com/armosec/kubescape/cautils/k8sinterface" ) func (pn *PolicyNotification) ToJSONBytesBuffer() (*bytes.Buffer, error) { @@ -130,43 +128,9 @@ func (ruleReport *RuleReport) GetNumberOfFailedResources() int { return sum } -func (ruleReport *RuleReport) DeleteIfRedundantResponse(RuleResponse *RuleResponse, index int) bool { - if b, rr := ruleReport.IsDuplicateResponseOfResource(RuleResponse, index); b { - rr.AddMessageToResponse(RuleResponse.AlertMessage) - ruleReport.RuleResponses = removeResponse(ruleReport.RuleResponses, index) - return true - } - return false -} - -func (ruleResponse *RuleResponse) AddMessageToResponse(message string) { - ruleResponse.AlertMessage += message -} - -func (ruleReport *RuleReport) IsDuplicateResponseOfResource(RuleResponse *RuleResponse, index int) (bool, *RuleResponse) { - for i := range ruleReport.RuleResponses { - if i != index { - for j := range ruleReport.RuleResponses[i].AlertObject.K8SApiObjects { - for k := range RuleResponse.AlertObject.K8SApiObjects { - w1 := k8sinterface.NewWorkloadObj(ruleReport.RuleResponses[i].AlertObject.K8SApiObjects[j]) - w2 := k8sinterface.NewWorkloadObj(RuleResponse.AlertObject.K8SApiObjects[k]) - if w1.GetName() == w2.GetName() && w1.GetNamespace() == w2.GetNamespace() && w1.GetKind() != "Role" && w1.GetKind() != "ClusterRole" { - return true, &ruleReport.RuleResponses[i] - } - } - } - } - } - return false, nil -} - -func removeResponse(slice []RuleResponse, index int) []RuleResponse { - return append(slice[:index], slice[index+1:]...) -} - func (ruleReport *RuleReport) GetNumberOfWarningResources() int { sum := 0 - for i := len(ruleReport.RuleResponses) - 1; i >= 0; i-- { + for i := range ruleReport.RuleResponses { if ruleReport.RuleResponses[i].GetSingleResultStatus() == "warning" { sum += len(ruleReport.RuleResponses[i].AlertObject.K8SApiObjects) } From 00d8660a91c3b2156c0eed13d1462b88ebaa5ad8 Mon Sep 17 00:00:00 2001 From: Bezalel Brandwine Date: Thu, 14 Oct 2021 10:46:42 +0300 Subject: [PATCH 05/17] fix --help flag --- cmd/root.go | 10 +++++++--- cmd/scan.go | 2 -- main.go | 3 --- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index c760e45a..7f51c38a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -13,10 +13,17 @@ import ( var cfgFile string var armoBEURLs = "" +const envFlagUsage = "Send report results to specific URL. Format:,,.\n\t\tExample:report.armo.cloud,api.armo.cloud,portal.armo.cloud" + var rootCmd = &cobra.Command{ Use: "kubescape", Short: "Kubescape is a tool for testing Kubernetes security posture", Long: `Kubescape is a tool for testing Kubernetes security posture based on NSA \ MITRE ATT&CK® specifications.`, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + flag.Parse() + InitArmoBEConnector() + return nil + }, } func Execute() { @@ -36,9 +43,6 @@ func initConfig() { } func InitArmoBEConnector() { - if armoBEURLs == "" && rootCmd.Flag("environment") != nil { - armoBEURLs = rootCmd.Flag("environment").Value.String() - } urlSlices := strings.Split(armoBEURLs, ",") if len(urlSlices) > 3 { glog.Errorf("Too many URLs") diff --git a/cmd/scan.go b/cmd/scan.go index ca14f8e3..e421f852 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -4,8 +4,6 @@ import ( "github.com/spf13/cobra" ) -const envFlagUsage = "Send report results to specific URL. Format:,,.\n\t\tExample:report.armo.cloud,api.armo.cloud,portal.armo.cloud" - // scanCmd represents the scan command var scanCmd = &cobra.Command{ Use: "scan", diff --git a/main.go b/main.go index 6565b817..114f855c 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,6 @@ package main import ( - "flag" "fmt" "os" @@ -10,8 +9,6 @@ import ( func main() { CheckLatestVersion() - flag.Parse() - cmd.InitArmoBEConnector() cmd.Execute() } From c90c3bbd05cb3015c62c4981e12e38c32ead95d0 Mon Sep 17 00:00:00 2001 From: Bezbran Date: Thu, 14 Oct 2021 11:32:59 +0300 Subject: [PATCH 06/17] test help message --- .github/workflows/build_dev.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build_dev.yaml b/.github/workflows/build_dev.yaml index bc5f11ff..c3b270da 100644 --- a/.github/workflows/build_dev.yaml +++ b/.github/workflows/build_dev.yaml @@ -33,6 +33,9 @@ jobs: CGO_ENABLED: 0 run: mkdir -p build/${{ matrix.os }} && go mod tidy && go build -ldflags "-w -s -X github.com/armosec/kubescape/cmd.BuildNumber=$RELEASE -X github.com/armosec/kubescape/cautils/getter.ArmoBEURL=$ArmoBEServer -X github.com/armosec/kubescape/cautils/getter.ArmoERURL=$ArmoERServer -X github.com/armosec/kubescape/cautils/getter.ArmoFEURL=$ArmoWebsite" -o build/${{ matrix.os }}/kubescape # && md5sum build/${{ matrix.os }}/kubescape > build/${{ matrix.os }}/kubescape.md5 + - name: TestHelpCMD + run: ./build/${{ matrix.os }}/kubescape -h | grep "download" + - name: Upload build artifacts uses: actions/upload-artifact@v2 with: From f9f32a10620ad87c76dd1f1490736baf50a16716 Mon Sep 17 00:00:00 2001 From: Bezalel Brandwine Date: Thu, 14 Oct 2021 11:43:11 +0300 Subject: [PATCH 07/17] comment in file loading tests --- policyhandler/filesloader_test.go | 55 ++++++++++++------------------- 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/policyhandler/filesloader_test.go b/policyhandler/filesloader_test.go index c317fcc3..26772eb6 100644 --- a/policyhandler/filesloader_test.go +++ b/policyhandler/filesloader_test.go @@ -3,56 +3,43 @@ package policyhandler import ( "fmt" "os" - "path/filepath" + "path" "strings" "testing" - - "github.com/armosec/kubescape/cautils" ) -func combineYamlFile(base, rel string) string { - finalPath := []string{} - sBase := strings.Split(base, "/") - sRel := strings.Split(rel, "/") - for i := range sBase { - if cautils.StringInSlice(sRel, sBase[i]) != cautils.ValueNotFound { - finalPath = append(finalPath, sRel...) - break - } - finalPath = append(finalPath, sBase[i]) - } - return fmt.Sprintf("/%s", filepath.Join(finalPath...)) -} func onlineBoutiquePath() string { o, _ := os.Getwd() - return combineYamlFile(o, "kubescape/examples/online-boutique/*") + return path.Join(o+"/..", "examples/online-boutique/*") } func TestListFiles(t *testing.T) { - // files, errs := listFiles([]string{onlineBoutiquePath()}) - // if len(errs) > 0 { - // t.Error(errs) - // } - // expected := 12 - // if len(files) != expected { - // t.Errorf("wrong number of files, expected: %d, found: %d", expected, len(files)) - // } + workDir, err := os.Getwd() + fmt.Printf("\n------------------\n%s,%v\n--------------\n", workDir, err) + files, errs := listFiles([]string{onlineBoutiquePath()}) + if len(errs) > 0 { + t.Error(errs) + } + expected := 12 + if len(files) != expected { + t.Errorf("wrong number of files, expected: %d, found: %d", expected, len(files)) + } } func TestLoadFiles(t *testing.T) { - // files, _ := listFiles([]string{onlineBoutiquePath()}) - // loadFiles(files) + files, _ := listFiles([]string{onlineBoutiquePath()}) + loadFiles(files) } func TestLoadFile(t *testing.T) { - // files, _ := listFiles([]string{strings.Replace(onlineBoutiquePath(), "*", "adservice.yaml", 1)}) - // _, err := loadFile(files[0]) - // if err != nil { - // t.Errorf("%v", err) - // } + files, _ := listFiles([]string{strings.Replace(onlineBoutiquePath(), "*", "adservice.yaml", 1)}) + _, err := loadFile(files[0]) + if err != nil { + t.Errorf("%v", err) + } } func TestLoadResources(t *testing.T) { - - // k8sResources, err = policyHandler.loadResources(opaSessionObj.Frameworks, scanInfo) + // policyHandler := &PolicyHandler{} + // k8sResources, err := policyHandler.loadResources(opaSessionObj.Frameworks, scanInfo) // files, _ := listFiles([]string{onlineBoutiquePath()}) // bb, err := loadFile(files[0]) // if len(err) > 0 { From 475e45b8484ac7d0d1617f8f88200402fae7db85 Mon Sep 17 00:00:00 2001 From: Bezalel Brandwine Date: Thu, 14 Oct 2021 12:07:51 +0300 Subject: [PATCH 08/17] add CLI prints checks skeleton --- .github/workflows/build_dev.yaml | 5 +---- build.py | 8 ++++++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_dev.yaml b/.github/workflows/build_dev.yaml index c3b270da..7638ac5e 100644 --- a/.github/workflows/build_dev.yaml +++ b/.github/workflows/build_dev.yaml @@ -31,10 +31,7 @@ jobs: ArmoERServer: report.euprod1.cyberarmorsoft.com ArmoWebsite: portal.armo.cloud CGO_ENABLED: 0 - run: mkdir -p build/${{ matrix.os }} && go mod tidy && go build -ldflags "-w -s -X github.com/armosec/kubescape/cmd.BuildNumber=$RELEASE -X github.com/armosec/kubescape/cautils/getter.ArmoBEURL=$ArmoBEServer -X github.com/armosec/kubescape/cautils/getter.ArmoERURL=$ArmoERServer -X github.com/armosec/kubescape/cautils/getter.ArmoFEURL=$ArmoWebsite" -o build/${{ matrix.os }}/kubescape # && md5sum build/${{ matrix.os }}/kubescape > build/${{ matrix.os }}/kubescape.md5 - - - name: TestHelpCMD - run: ./build/${{ matrix.os }}/kubescape -h | grep "download" + run: python build.py - name: Upload build artifacts uses: actions/upload-artifact@v2 diff --git a/build.py b/build.py index 2687e0ef..a8cbe021 100644 --- a/build.py +++ b/build.py @@ -59,6 +59,8 @@ def main(): ER_SERVER_CONST, ArmoERServer, WEBSITE_CONST, ArmoWebsite) status = subprocess.call(["go", "build", "-o", "%s/%s" % (buildDir, packageName), "-ldflags" ,ldflags]) checkStatus(status, "Failed to build kubescape") + + test_cli_prints(buildDir,packageName) sha1 = hashlib.sha1() @@ -69,5 +71,11 @@ def main(): print("Build Done") +def test_cli_prints(buildDir,packageName): + print("testing CLI prints") + bin_cli = os.path.join(buildDir,packageName) + status = str(subprocess.check_output([bin_cli, "-h"])) + assert "download" in status, "download is missing: " + status + if __name__ == "__main__": main() From d81984b4c6ec01ae4fcbe7a3a55837b48b317f05 Mon Sep 17 00:00:00 2001 From: Bezalel Brandwine Date: Thu, 14 Oct 2021 13:28:10 +0300 Subject: [PATCH 09/17] try to adapt path to windows --- policyhandler/filesloader_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/policyhandler/filesloader_test.go b/policyhandler/filesloader_test.go index 26772eb6..681488b2 100644 --- a/policyhandler/filesloader_test.go +++ b/policyhandler/filesloader_test.go @@ -4,18 +4,22 @@ import ( "fmt" "os" "path" + "path/filepath" "strings" "testing" ) func onlineBoutiquePath() string { o, _ := os.Getwd() - return path.Join(o+"/..", "examples/online-boutique/*") + return filepath.Join(path.Dir(o), "examples/online-boutique/*") } func TestListFiles(t *testing.T) { workDir, err := os.Getwd() fmt.Printf("\n------------------\n%s,%v\n--------------\n", workDir, err) - files, errs := listFiles([]string{onlineBoutiquePath()}) + filesPath := onlineBoutiquePath() + fmt.Printf("\n------------------\n%s\n--------------\n", filesPath) + + files, errs := listFiles([]string{filesPath}) if len(errs) > 0 { t.Error(errs) } From 46449045a6573ef29e45d049f0ff662c6fbcdeb4 Mon Sep 17 00:00:00 2001 From: Bezalel Brandwine Date: Thu, 14 Oct 2021 14:04:55 +0300 Subject: [PATCH 10/17] files tests passed locally on windows --- policyhandler/filesloader_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/policyhandler/filesloader_test.go b/policyhandler/filesloader_test.go index 681488b2..f2bea56b 100644 --- a/policyhandler/filesloader_test.go +++ b/policyhandler/filesloader_test.go @@ -3,7 +3,6 @@ package policyhandler import ( "fmt" "os" - "path" "path/filepath" "strings" "testing" @@ -11,8 +10,9 @@ import ( func onlineBoutiquePath() string { o, _ := os.Getwd() - return filepath.Join(path.Dir(o), "examples/online-boutique/*") + return filepath.Join(filepath.Dir(o), "examples/online-boutique/*") } + func TestListFiles(t *testing.T) { workDir, err := os.Getwd() fmt.Printf("\n------------------\n%s,%v\n--------------\n", workDir, err) From 9b6ad102b1d4ef4467174b944a6a7b1e59242ef7 Mon Sep 17 00:00:00 2001 From: Bezalel Brandwine Date: Thu, 14 Oct 2021 14:16:12 +0300 Subject: [PATCH 11/17] adapt test_cli_prints to windows --- build.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index a8cbe021..546b0084 100644 --- a/build.py +++ b/build.py @@ -72,8 +72,9 @@ def main(): print("Build Done") def test_cli_prints(buildDir,packageName): - print("testing CLI prints") - bin_cli = os.path.join(buildDir,packageName) + bin_cli = os.path.abspath(os.path.join(buildDir,packageName)) + + print(f"testing CLI prints on {bin_cli}") status = str(subprocess.check_output([bin_cli, "-h"])) assert "download" in status, "download is missing: " + status From d69cccf8218c5d218e5026efddf46079c7f6adf8 Mon Sep 17 00:00:00 2001 From: Bezalel Brandwine Date: Thu, 14 Oct 2021 14:24:49 +0300 Subject: [PATCH 12/17] print python version --- .github/workflows/build_dev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dev.yaml b/.github/workflows/build_dev.yaml index 7638ac5e..e22d068f 100644 --- a/.github/workflows/build_dev.yaml +++ b/.github/workflows/build_dev.yaml @@ -31,7 +31,7 @@ jobs: ArmoERServer: report.euprod1.cyberarmorsoft.com ArmoWebsite: portal.armo.cloud CGO_ENABLED: 0 - run: python build.py + run: python --version && python build.py - name: Upload build artifacts uses: actions/upload-artifact@v2 From 1dd587cd838e466c5784f90b52600b20acac0a21 Mon Sep 17 00:00:00 2001 From: Bezalel Brandwine Date: Thu, 14 Oct 2021 14:30:58 +0300 Subject: [PATCH 13/17] try python3 --- .github/workflows/build_dev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dev.yaml b/.github/workflows/build_dev.yaml index e22d068f..5ae89641 100644 --- a/.github/workflows/build_dev.yaml +++ b/.github/workflows/build_dev.yaml @@ -31,7 +31,7 @@ jobs: ArmoERServer: report.euprod1.cyberarmorsoft.com ArmoWebsite: portal.armo.cloud CGO_ENABLED: 0 - run: python --version && python build.py + run: python3 --version && python3 build.py - name: Upload build artifacts uses: actions/upload-artifact@v2 From 78670665c456df151c5617c51746d98a1daa9e74 Mon Sep 17 00:00:00 2001 From: Bezalel Brandwine Date: Thu, 14 Oct 2021 14:39:43 +0300 Subject: [PATCH 14/17] build.py support only pthon 3.7 and above --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ba65ee9c..2b82689a 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,7 @@ Kubescape is an open source project, we welcome your feedback and ideas for impr # How to build -## Build using python script +## Build using python (3.7^) script Kubescpae can be built using: From 7bebc7a8147803855deff6833710170234a0999f Mon Sep 17 00:00:00 2001 From: Bezbran Date: Thu, 14 Oct 2021 14:48:16 +0300 Subject: [PATCH 15/17] master PR python 3 for build --- .github/workflows/master_pr_checks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/master_pr_checks.yaml b/.github/workflows/master_pr_checks.yaml index ff96fd7a..53fca59f 100644 --- a/.github/workflows/master_pr_checks.yaml +++ b/.github/workflows/master_pr_checks.yaml @@ -29,7 +29,7 @@ jobs: ArmoERServer: report.armo.cloud ArmoWebsite: portal.armo.cloud CGO_ENABLED: 0 - run: python build.py + run: python3 --version && python3 build.py - name: Upload build artifacts uses: actions/upload-artifact@v2 From 7faf24cf88118e669dd1f710562ace0521866351 Mon Sep 17 00:00:00 2001 From: Bezbran Date: Thu, 14 Oct 2021 14:48:47 +0300 Subject: [PATCH 16/17] build python3 --version && python3 build.py --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f9602b81..c39f20ef 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -45,7 +45,7 @@ jobs: ArmoERServer: report.armo.cloud ArmoWebsite: portal.armo.cloud CGO_ENABLED: 0 - run: python build.py + run: python3 --version && python3 build.py - name: Upload Release binaries id: upload-release-asset From 718d549bb34afa5407989a76ab61bf8103d3faa5 Mon Sep 17 00:00:00 2001 From: Avner Tzur Date: Thu, 14 Oct 2021 15:23:35 +0300 Subject: [PATCH 17/17] update Dockerfile using Python build script --- build/Dockerfile | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index 4cf8dcb3..6385051f 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,13 +1,25 @@ FROM golang:1.17-alpine as builder -ENV GOPROXY=https://goproxy.io,direct -ENV GO111MODULE=on +#ENV GOPROXY=https://goproxy.io,direct +ENV GO111MODULE= + +ENV CGO_ENABLED=0 + +# Install required python/pip +ENV PYTHONUNBUFFERED=1 +RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python +RUN python3 -m ensurepip +RUN pip3 install --no-cache --upgrade pip setuptools WORKDIR /work ADD . . -RUN GOOS=linux CGO_ENABLED=0 go build -ldflags="-s -w " -installsuffix cgo -o kubescape . + +RUN python build.py + +RUN ls -ltr build/ubuntu-latest +RUN cat /work/build/ubuntu-latest/kubescape.sha1 FROM alpine -COPY --from=builder /work/kubescape /usr/bin/kubescape +COPY --from=builder /work/build/ubuntu-latest/kubescape /usr/bin/kubescape # # Download the frameworks. Use the "--use-default" flag when running kubescape # RUN kubescape download framework nsa && kubescape download framework mitre