Merge pull request #169 from armosec/rotemamsa-patch-1

Delete backendconnectormethods.go

.github/workflows/build.yaml

@@ -2,10 +2,7 @@ name: build
 
 on:
   push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-    types: [ closed ]
+    branches: [ master ] 
 jobs:
   once:
     name: Create release
@@ -48,7 +45,7 @@ jobs:
           ArmoERServer: report.armo.cloud
           ArmoWebsite: portal.armo.cloud
           CGO_ENABLED: 0
-        run: python build.py
+        run: python3 --version && python3 build.py
       
       - name: Upload Release binaries
         id: upload-release-asset

.github/workflows/build_dev.yaml

@@ -31,7 +31,7 @@ jobs:
           ArmoERServer: report.euprod1.cyberarmorsoft.com
           ArmoWebsite: portal.armo.cloud
           CGO_ENABLED: 0
-        run: mkdir -p build/${{ matrix.os }}  && go mod tidy && go build -ldflags "-w -s -X github.com/armosec/kubescape/cmd.BuildNumber=$RELEASE -X github.com/armosec/kubescape/cautils/getter.ArmoBEURL=$ArmoBEServer -X github.com/armosec/kubescape/cautils/getter.ArmoERURL=$ArmoERServer -X github.com/armosec/kubescape/cautils/getter.ArmoFEURL=$ArmoWebsite" -o build/${{ matrix.os }}/kubescape # && md5sum build/${{ matrix.os }}/kubescape > build/${{ matrix.os }}/kubescape.md5
+        run: python3 --version && python3 build.py
       
       - name: Upload build artifacts 
         uses: actions/upload-artifact@v2

.github/workflows/master_pr_checks.yaml

@@ -0,0 +1,38 @@
+name: master-pr
+
+on:
+  pull_request:
+    branches: [ master ]
+    types: [ edited, opened, synchronize, reopened ]
+jobs:
+  build:
+    name: Create cross-platform build
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.17
+
+      - name: Test
+        run: go test -v ./...
+
+      - name: Build
+        env:
+          RELEASE: v1.0.${{ github.run_number }} 
+          ArmoBEServer: api.armo.cloud
+          ArmoERServer: report.armo.cloud
+          ArmoWebsite: portal.armo.cloud
+          CGO_ENABLED: 0
+        run: python3 --version && python3 build.py
+
+      - name: Upload build artifacts 
+        uses: actions/upload-artifact@v2
+        with:
+          name: kubescape-${{ matrix.os }}
+          path: build/${{ matrix.os }}/kubescape 

README.md

@@ -78,8 +78,8 @@ Set-ExecutionPolicy RemoteSigned -scope CurrentUser
 | `--use-from` | | Load local framework object from specified path. If not used will download latest |
 | `--use-default` | `false` | Load local framework object from default path. If not used will download latest | `true`/`false` |
 | `--exceptions` | | Path to an [exceptions obj](examples/exceptions.json). If not set will download exceptions from Armo management portal |
-| `--submit` | `false` | If set, Kubescape will send scan results to Armo management portal to allow users to control exceptions and maintain chronological scan results. By default the results are not sent | `true`/`false`|
-| `--local` | `false` | Kubescape will not send scan results to Armo management portal. Use this flag if you ran with the `--submit` flag in the past and you do not want to submit your current scan results | `true`/`false`|
+| `--submit` | `false` | If set, Kubescape will send the scan results to Armo management portal where you can see the results in a user-friendly UI, choose your preferred compliance framework, check risk results history and trends, manage exceptions, get remediation recommendations and much more. By default the results are not sent | `true`/`false`|
+| `--keep-local` | `false` | Kubescape will not send scan results to Armo management portal. Use this flag if you ran with the `--submit` flag in the past and you do not want to submit your current scan results | `true`/`false`|
 | `--account` | | Armo portal account ID. Default will load account ID from configMap or config file | |
 
 ## Usage & Examples
@@ -97,7 +97,7 @@ kubescape scan framework nsa --exclude-namespaces kube-system,kube-public --subm
 kubescape scan framework mitre --exclude-namespaces kube-system,kube-public --submit
 ```
 
-* Scan local `yaml`/`json` files before deploying
+* Scan local `yaml`/`json` files before deploying. [Take a look at the demonstration](https://youtu.be/Ox6DaR7_4ZI)
 ```
 kubescape scan framework nsa *.yaml
 ```
@@ -154,7 +154,7 @@ Kubescape is an open source project, we welcome your feedback and ideas for impr
 
 # How to build
 
-## Build using python script
+## Build using python (3.7^) script
 
 Kubescpae can be built using:
 

build.py

@@ -59,6 +59,8 @@ def main():
            ER_SERVER_CONST, ArmoERServer, WEBSITE_CONST, ArmoWebsite)
     status = subprocess.call(["go", "build", "-o", "%s/%s" % (buildDir, packageName), "-ldflags" ,ldflags])
     checkStatus(status, "Failed to build kubescape")
+    
+    test_cli_prints(buildDir,packageName)
 
 
     sha1 = hashlib.sha1()
@@ -69,5 +71,12 @@ def main():
 
     print("Build Done")
 
+def test_cli_prints(buildDir,packageName):
+    bin_cli = os.path.abspath(os.path.join(buildDir,packageName))
+
+    print(f"testing CLI prints on {bin_cli}")
+    status = str(subprocess.check_output([bin_cli, "-h"]))
+    assert "download" in status, "download is missing: " + status    
+
 if __name__ == "__main__":
     main()

cautils/apis/backendconnectormethods.go

@@ -1,128 +0,0 @@
-package apis
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"strings"
-)
-
-func MakeBackendConnector(client *http.Client, baseURL string, loginDetails *CustomerLoginDetails) (*BackendConnector, error) {
-	if err := ValidateBEConnectorMakerInput(client, baseURL, loginDetails); err != nil {
-		return nil, err
-	}
-	conn := &BackendConnector{BaseURL: baseURL, Credentials: loginDetails, HTTPClient: client}
-	err := conn.Login()
-
-	return conn, err
-}
-
-func ValidateBEConnectorMakerInput(client *http.Client, baseURL string, loginDetails *CustomerLoginDetails) error {
-	if client == nil {
-		return fmt.Errorf("You must provide an initialized httpclient")
-	}
-	if len(baseURL) == 0 {
-		return fmt.Errorf("you must provide a valid backend url")
-	}
-
-	if loginDetails == nil || (len(loginDetails.Email) == 0 && len(loginDetails.Password) == 0) {
-		return fmt.Errorf("you must provide valid login details")
-	}
-	return nil
-
-}
-
-func (r *BackendConnector) Login() error {
-	if !r.IsExpired() {
-		return nil
-	}
-
-	loginInfoBytes, err := json.Marshal(r.Credentials)
-	if err != nil {
-		return fmt.Errorf("unable to marshal credentials properly")
-	}
-
-	beURL := fmt.Sprintf("%v/%v", r.BaseURL, "login")
-
-	req, err := http.NewRequest("POST", beURL, bytes.NewReader(loginInfoBytes))
-	if err != nil {
-		return err
-	}
-
-	req.Header.Set("Referer", strings.Replace(beURL, "dashbe", "cpanel", 1))
-	resp, err := r.HTTPClient.Do(req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return fmt.Errorf("unable to read login response")
-	}
-
-	loginS := &BELoginResponse{}
-	json.Unmarshal(body, &loginS)
-
-	loginS.Cookies = resp.Cookies()
-	r.BELoginResponse = loginS
-
-	return nil
-}
-
-func (r *BackendConnector) IsExpired() bool {
-	return r.BELoginResponse == nil || r.BELoginResponse.ToLoginObject().IsExpired()
-}
-
-func (r *BackendConnector) GetBaseURL() string {
-	return r.BaseURL
-}
-func (r *BackendConnector) GetLoginObj() *LoginObject {
-	return r.BELoginResponse.ToLoginObject()
-}
-func (r *BackendConnector) GetClient() *http.Client {
-	return r.HTTPClient
-}
-
-func (r *BackendConnector) HTTPSend(httpverb string,
-	endpoint string,
-	payload []byte,
-	f HTTPReqFunc,
-	qryData interface{}) ([]byte, error) {
-
-	beURL := fmt.Sprintf("%v/%v", r.GetBaseURL(), endpoint)
-	req, err := http.NewRequest(httpverb, beURL, bytes.NewReader(payload))
-	if err != nil {
-		return nil, err
-	}
-
-	if r.IsExpired() {
-		r.Login()
-	}
-
-	loginobj := r.GetLoginObj()
-	req.Header.Set("Authorization", loginobj.Authorization)
-	f(req, qryData)
-	q := req.URL.Query()
-	q.Set("customerGUID", loginobj.GUID)
-	req.URL.RawQuery = q.Encode()
-
-	for _, cookie := range loginobj.Cookies {
-		req.AddCookie(cookie)
-	}
-	resp, err := r.GetClient().Do(req)
-	if err != nil {
-		return nil, err
-	}
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		fmt.Printf("req:\n%v\nresp:%v\n", req, resp)
-		return nil, fmt.Errorf("Error #%v Due to: %v", resp.StatusCode, resp.Status)
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
-	}
-	return body, nil
-}

cautils/customerloader.go

@@ -65,7 +65,7 @@ func ClusterConfigSetup(scanInfo *ScanInfo, k8s *k8sinterface.KubernetesApi, beA
 			Submit - Create tenant & Submit report
 
 		If "Submitted but not signed up" -
-			Default	- Submit report (submit)
+			Default	- Delete local config & Do not send report (local)
 			Local - Delete local config & Do not send report
 			Submit - Submit report
 
@@ -76,9 +76,8 @@ func ClusterConfigSetup(scanInfo *ScanInfo, k8s *k8sinterface.KubernetesApi, beA
 
 	*/
 	clusterConfig := NewClusterConfig(k8s, beAPI)
-	if err := clusterConfig.SetCustomerGUID(scanInfo.Account); err != nil {
-		fmt.Println(err)
-	}
+	clusterConfig.LoadConfig()
+
 	if !IsSubmitted(clusterConfig) {
 		if scanInfo.Submit {
 			return clusterConfig // submit - Create tenant & Submit report
@@ -86,11 +85,11 @@ func ClusterConfigSetup(scanInfo *ScanInfo, k8s *k8sinterface.KubernetesApi, beA
 		return NewEmptyConfig() // local/default - Do not send report
 	}
 	if !IsRegistered(clusterConfig) {
-		if scanInfo.Local {
-			DeleteConfig(k8s)
-			return NewEmptyConfig() // local - Delete local config & Do not send report
+		if scanInfo.Submit {
+			return clusterConfig // submit/default - Submit report
 		}
-		return clusterConfig // submit/default -  Submit report
+		DeleteConfig(k8s)
+		return NewEmptyConfig() // local - Delete local config & Do not send report
 	}
 	if scanInfo.Local {
 		return NewEmptyConfig() // local - Do not send report
@@ -112,7 +111,7 @@ func (c *EmptyConfig) GetK8sAPI() *k8sinterface.KubernetesApi    { return nil }
 func (c *EmptyConfig) GetDefaultNS() string                      { return k8sinterface.GetDefaultNamespace() }
 func (c *EmptyConfig) GetBackendAPI() getter.IBackend            { return nil } // TODO: return mock obj
 func (c *EmptyConfig) GenerateURL() {
-	message := fmt.Sprintf("If you wish to submit your cluster so you can control exceptions and maintain chronological scan results, please run Kubescape with the `--submit` flag or sign-up here: https://%s", getter.ArmoFEURL)
+	message := fmt.Sprintf("You can see the results in a user-friendly UI, choose your preferred compliance framework, check risk results history and trends, manage exceptions, get remediation recommendations and much more by registering here: https://%s", getter.GetArmoAPIConnector().GetFrontendURL())
 	InfoTextDisplay(os.Stdout, message+"\n")
 }
 
@@ -131,6 +130,7 @@ func NewClusterConfig(k8s *k8sinterface.KubernetesApi, backendAPI getter.IBacken
 	return &ClusterConfig{
 		k8s:        k8s,
 		backendAPI: backendAPI,
+		configObj:  &ConfigObj{},
 		defaultNS:  k8sinterface.GetDefaultNamespace(),
 	}
 }
@@ -140,15 +140,16 @@ func (c *ClusterConfig) GetDefaultNS() string                   { return c.defau
 func (c *ClusterConfig) GetBackendAPI() getter.IBackend         { return c.backendAPI }
 
 func (c *ClusterConfig) GenerateURL() {
+
 	u := url.URL{}
 	u.Scheme = "https"
-	u.Host = getter.ArmoFEURL
+	u.Host = getter.GetArmoAPIConnector().GetFrontendURL()
 	if c.configObj == nil {
 		return
 	}
+	message := fmt.Sprintf("You can see the results in a user-friendly UI, choose your preferred compliance framework, check risk results history and trends, manage exceptions, get remediation recommendations and much more by registering here: %s", u.String())
 	if c.configObj.CustomerAdminEMail != "" {
-		msgStr := fmt.Sprintf("To view all controls and get remediation's ask access permissions to %s from %s", u.String(), c.configObj.CustomerAdminEMail)
-		InfoTextDisplay(os.Stdout, msgStr+"\n")
+		InfoTextDisplay(os.Stdout, message+"\n")
 		return
 	}
 	u.Path = "account/sign-up"
@@ -157,8 +158,7 @@ func (c *ClusterConfig) GenerateURL() {
 	q.Add("customerGUID", c.configObj.CustomerGUID)
 
 	u.RawQuery = q.Encode()
-	fmt.Println("To view all controls and get remediation's visit:")
-	InfoTextDisplay(os.Stdout, u.String()+"\n")
+	InfoTextDisplay(os.Stdout, message+"\n")
 
 }
 
@@ -171,21 +171,8 @@ func (c *ClusterConfig) GetCustomerGUID() string {
 
 func (c *ClusterConfig) SetCustomerGUID(customerGUID string) error {
 
-	updateConfig := false
-	createConfig := false
-
-	// get from configMap
-	if c.existsConfigMap() {
-		c.configObj, _ = c.loadConfigFromConfigMap()
-	} else if existsConfigFile() { // get from file
-		c.configObj, _ = loadConfigFromFile()
-	} else {
-		c.configObj = &ConfigObj{}
-		createConfig = true
-	}
 	if customerGUID != "" && c.GetCustomerGUID() != customerGUID {
 		c.configObj.CustomerGUID = customerGUID // override config customerGUID
-		updateConfig = true
 	}
 
 	customerGUID = c.GetCustomerGUID()
@@ -199,25 +186,38 @@ func (c *ClusterConfig) SetCustomerGUID(customerGUID string) error {
 			c.configObj.Token = tenantResponse.Token
 			c.configObj.CustomerGUID = tenantResponse.TenantID
 		}
-		updateConfig = true
 	} else {
 		if err != nil && !strings.Contains(err.Error(), "already exists") {
 			return err
 		}
 	}
-	if createConfig {
+
+	// update/create config
+	if c.existsConfigMap() {
+		c.updateConfigMap()
+	} else {
 		c.createConfigMap()
-		c.createConfigFile()
-	} else if updateConfig {
-		if c.existsConfigMap() {
-			c.updateConfigMap()
-		}
-		if existsConfigFile() {
-			c.updateConfigFile()
-		}
 	}
+	if existsConfigFile() {
+		c.updateConfigFile()
+	} else {
+		c.createConfigFile()
+	}
+
 	return nil
 }
+
+func (c *ClusterConfig) LoadConfig() {
+	// get from configMap
+	if c.existsConfigMap() {
+		c.configObj, _ = c.loadConfigFromConfigMap()
+	} else if existsConfigFile() { // get from file
+		c.configObj, _ = loadConfigFromFile()
+	} else {
+		c.configObj = &ConfigObj{}
+	}
+}
+
 func (c *ClusterConfig) ToMapString() map[string]interface{} {
 	m := map[string]interface{}{}
 	bc, _ := json.Marshal(c.configObj)

cautils/getter/armoapi.go

@@ -3,9 +3,11 @@ package getter
 import (
 	"fmt"
 	"net/http"
+	"time"
 
 	"github.com/armosec/kubescape/cautils/armotypes"
 	"github.com/armosec/kubescape/cautils/opapolicy"
+	"github.com/golang/glog"
 )
 
 // =======================================================================================================================
@@ -15,22 +17,80 @@ import (
 var (
 	// ATTENTION!!!
 	// Changes in this URLs variable names, or in the usage is affecting the build process! BE CAREFULL
-	ArmoBEURL = "eggdashbe.eudev3.cyberarmorsoft.com"
-	ArmoERURL = "report.eudev3.cyberarmorsoft.com"
-	ArmoFEURL = "armoui.eudev3.cyberarmorsoft.com"
-	// ArmoURL = "https://dashbe.euprod1.cyberarmorsoft.com"
+	armoERURL = "report.armo.cloud"
+	armoBEURL = "api.armo.cloud"
+	armoFEURL = "portal.armo.cloud"
+
+	armoDevERURL = "report.eudev3.cyberarmorsoft.com"
+	armoDevBEURL = "eggdashbe.eudev3.cyberarmorsoft.com"
+	armoDevFEURL = "armoui.eudev3.cyberarmorsoft.com"
 )
 
 // Armo API for downloading policies
 type ArmoAPI struct {
 	httpClient *http.Client
+	apiURL     string
+	erURL      string
+	feURL      string
 }
 
-func NewArmoAPI() *ArmoAPI {
+var globalArmoAPIConnecctor *ArmoAPI
+
+func SetARMOAPIConnector(armoAPI *ArmoAPI) {
+	globalArmoAPIConnecctor = armoAPI
+}
+
+func GetArmoAPIConnector() *ArmoAPI {
+	if globalArmoAPIConnecctor == nil {
+		glog.Error("returning nil API connector")
+	}
+	return globalArmoAPIConnecctor
+}
+
+func NewARMOAPIDev() *ArmoAPI {
+	apiObj := newArmoAPI()
+
+	apiObj.apiURL = armoDevBEURL
+	apiObj.erURL = armoDevERURL
+	apiObj.feURL = armoDevFEURL
+
+	return apiObj
+}
+
+func NewARMOAPIProd() *ArmoAPI {
+	apiObj := newArmoAPI()
+
+	apiObj.apiURL = armoBEURL
+	apiObj.erURL = armoERURL
+	apiObj.feURL = armoFEURL
+
+	return apiObj
+}
+
+func NewARMOAPICustomized(armoERURL, armoBEURL, armoFEURL string) *ArmoAPI {
+	apiObj := newArmoAPI()
+
+	apiObj.erURL = armoERURL
+	apiObj.apiURL = armoBEURL
+	apiObj.feURL = armoFEURL
+
+	return apiObj
+}
+
+func newArmoAPI() *ArmoAPI {
 	return &ArmoAPI{
-		httpClient: &http.Client{},
+		httpClient: &http.Client{Timeout: time.Duration(61) * time.Second},
 	}
 }
+
+func (armoAPI *ArmoAPI) GetFrontendURL() string {
+	return armoAPI.feURL
+}
+
+func (armoAPI *ArmoAPI) GetReportReceiverURL() string {
+	return armoAPI.erURL
+}
+
 func (armoAPI *ArmoAPI) GetFramework(name string) (*opapolicy.Framework, error) {
 	respStr, err := HttpGetter(armoAPI.httpClient, armoAPI.getFrameworkURL(name))
 	if err != nil {

cautils/getter/armoapiutils.go

@@ -8,7 +8,7 @@ import (
 func (armoAPI *ArmoAPI) getFrameworkURL(frameworkName string) string {
 	u := url.URL{}
 	u.Scheme = "https"
-	u.Host = ArmoBEURL
+	u.Host = armoAPI.apiURL
 	u.Path = "v1/armoFrameworks"
 	q := u.Query()
 	q.Add("customerGUID", "11111111-1111-1111-1111-111111111111")
@@ -22,7 +22,7 @@ func (armoAPI *ArmoAPI) getFrameworkURL(frameworkName string) string {
 func (armoAPI *ArmoAPI) getExceptionsURL(customerGUID, clusterName string) string {
 	u := url.URL{}
 	u.Scheme = "https"
-	u.Host = ArmoBEURL
+	u.Host = armoAPI.apiURL
 	u.Path = "api/v1/armoPostureExceptions"
 
 	q := u.Query()
@@ -38,7 +38,7 @@ func (armoAPI *ArmoAPI) getExceptionsURL(customerGUID, clusterName string) strin
 func (armoAPI *ArmoAPI) getCustomerURL() string {
 	u := url.URL{}
 	u.Scheme = "https"
-	u.Host = ArmoBEURL
+	u.Host = armoAPI.apiURL
 	u.Path = "api/v1/createTenant"
 	return u.String()
 }

cautils/getter/downloadreleasedpolicy.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"time"
 
 	"github.com/armosec/kubescape/cautils/opapolicy"
 )
@@ -22,7 +23,7 @@ type DownloadReleasedPolicy struct {
 func NewDownloadReleasedPolicy() *DownloadReleasedPolicy {
 	return &DownloadReleasedPolicy{
 		hostURL:    "",
-		httpClient: &http.Client{},
+		httpClient: &http.Client{Timeout: 61 * time.Second},
 	}
 }
 

cautils/getter/getpoliciesutils.go

@@ -5,8 +5,8 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"net/url"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 
@@ -21,14 +21,26 @@ func GetDefaultPath(name string) string {
 	return defaultfilePath
 }
 
-func SaveFrameworkInFile(framework *opapolicy.Framework, path string) error {
+func SaveFrameworkInFile(framework *opapolicy.Framework, pathStr string) error {
 	encodedData, err := json.Marshal(framework)
 	if err != nil {
 		return err
 	}
-	err = os.WriteFile(path, []byte(fmt.Sprintf("%v", string(encodedData))), 0644)
+	err = os.WriteFile(pathStr, []byte(fmt.Sprintf("%v", string(encodedData))), 0644)
 	if err != nil {
-		return err
+		if os.IsNotExist(err) {
+			pathDir := path.Dir(pathStr)
+			if err := os.Mkdir(pathDir, 0744); err != nil {
+				return err
+			}
+		} else {
+			return err
+
+		}
+		err = os.WriteFile(pathStr, []byte(fmt.Sprintf("%v", string(encodedData))), 0644)
+		if err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -86,29 +98,3 @@ func httpRespToString(resp *http.Response) (string, error) {
 
 	return respStr, err
 }
-
-// URLEncoder encode url
-func urlEncoder(oldURL string) string {
-	fullURL := strings.Split(oldURL, "?")
-	baseURL, err := url.Parse(fullURL[0])
-	if err != nil {
-		return ""
-	}
-
-	// Prepare Query Parameters
-	if len(fullURL) > 1 {
-		params := url.Values{}
-		queryParams := strings.Split(fullURL[1], "&")
-		for _, i := range queryParams {
-			queryParam := strings.Split(i, "=")
-			val := ""
-			if len(queryParam) > 1 {
-				val = queryParam[1]
-			}
-			params.Add(queryParam[0], val)
-		}
-		baseURL.RawQuery = params.Encode()
-	}
-
-	return baseURL.String()
-}

cautils/opapolicy/datastructures.go

@@ -41,7 +41,8 @@ type FrameworkReport struct {
 }
 type ControlReport struct {
 	armotypes.PortalBase `json:",inline"`
-	ControlID            string       `json:"id"`
+	Control_ID           string       `json:"id,omitempty"` // to be Deprecated
+	ControlID            string       `json:"controlID"`
 	Name                 string       `json:"name"`
 	RuleReports          []RuleReport `json:"ruleReports"`
 	Remediation          string       `json:"remediation"`
@@ -101,7 +102,8 @@ type PolicyRule struct {
 // Control represents a collection of rules which are combined together to single purpose
 type Control struct {
 	armotypes.PortalBase `json:",inline"`
-	ControlID            string       `json:"id"`
+	Control_ID           string       `json:"id,omitempty"` // to be Deprecated
+	ControlID            string       `json:"controlID"`
 	CreationTime         string       `json:"creationTime"`
 	Description          string       `json:"description"`
 	Remediation          string       `json:"remediation"`

cautils/opapolicy/datastructuresmethods.go

@@ -3,8 +3,6 @@ package opapolicy
 import (
 	"bytes"
 	"encoding/json"
-
-	"github.com/armosec/kubescape/cautils/k8sinterface"
 )
 
 func (pn *PolicyNotification) ToJSONBytesBuffer() (*bytes.Buffer, error) {
@@ -124,53 +122,17 @@ func (ruleReport *RuleReport) GetNumberOfFailedResources() int {
 	sum := 0
 	for i := len(ruleReport.RuleResponses) - 1; i >= 0; i-- {
 		if ruleReport.RuleResponses[i].GetSingleResultStatus() == "failed" {
-			if !ruleReport.DeleteIfRedundantResponse(&ruleReport.RuleResponses[i], i) {
-				sum++
-			}
+			sum += len(ruleReport.RuleResponses[i].AlertObject.K8SApiObjects)
 		}
 	}
 	return sum
 }
 
-func (ruleReport *RuleReport) DeleteIfRedundantResponse(RuleResponse *RuleResponse, index int) bool {
-	if b, rr := ruleReport.IsDuplicateResponseOfResource(RuleResponse, index); b {
-		rr.AddMessageToResponse(RuleResponse.AlertMessage)
-		ruleReport.RuleResponses = removeResponse(ruleReport.RuleResponses, index)
-		return true
-	}
-	return false
-}
-
-func (ruleResponse *RuleResponse) AddMessageToResponse(message string) {
-	ruleResponse.AlertMessage += message
-}
-
-func (ruleReport *RuleReport) IsDuplicateResponseOfResource(RuleResponse *RuleResponse, index int) (bool, *RuleResponse) {
-	for i := range ruleReport.RuleResponses {
-		if i != index {
-			for j := range ruleReport.RuleResponses[i].AlertObject.K8SApiObjects {
-				for k := range RuleResponse.AlertObject.K8SApiObjects {
-					w1 := k8sinterface.NewWorkloadObj(ruleReport.RuleResponses[i].AlertObject.K8SApiObjects[j])
-					w2 := k8sinterface.NewWorkloadObj(RuleResponse.AlertObject.K8SApiObjects[k])
-					if w1.GetName() == w2.GetName() && w1.GetNamespace() == w2.GetNamespace() && w1.GetKind() != "Role" && w1.GetKind() != "ClusterRole" {
-						return true, &ruleReport.RuleResponses[i]
-					}
-				}
-			}
-		}
-	}
-	return false, nil
-}
-
-func removeResponse(slice []RuleResponse, index int) []RuleResponse {
-	return append(slice[:index], slice[index+1:]...)
-}
-
 func (ruleReport *RuleReport) GetNumberOfWarningResources() int {
 	sum := 0
 	for i := range ruleReport.RuleResponses {
 		if ruleReport.RuleResponses[i].GetSingleResultStatus() == "warning" {
-			sum += 1
+			sum += len(ruleReport.RuleResponses[i].AlertObject.K8SApiObjects)
 		}
 	}
 	return sum

cautils/scaninfo.go

@@ -43,7 +43,7 @@ func (scanInfo *ScanInfo) setUseExceptions() {
 		// load exceptions from file
 		scanInfo.ExceptionsGetter = getter.NewLoadPolicy(scanInfo.UseExceptions)
 	} else {
-		scanInfo.ExceptionsGetter = getter.NewArmoAPI()
+		scanInfo.ExceptionsGetter = getter.GetArmoAPIConnector()
 	}
 
 }

cmd/cluster_get.go

@@ -31,7 +31,7 @@ var getCmd = &cobra.Command{
 		key := keyValue[0]
 
 		k8s := k8sinterface.NewKubernetesApi()
-		clusterConfig := cautils.NewClusterConfig(k8s, getter.NewArmoAPI())
+		clusterConfig := cautils.NewClusterConfig(k8s, getter.GetArmoAPIConnector())
 		val, err := clusterConfig.GetValueByKeyFromConfigMap(key)
 		if err != nil {
 			if err.Error() == "value does not exist." {

cmd/cluster_set.go

@@ -30,7 +30,7 @@ var setCmd = &cobra.Command{
 		data := keyValue[1]
 
 		k8s := k8sinterface.NewKubernetesApi()
-		clusterConfig := cautils.NewClusterConfig(k8s, getter.NewArmoAPI())
+		clusterConfig := cautils.NewClusterConfig(k8s, getter.GetArmoAPIConnector())
 		if err := clusterConfig.SetKeyValueInConfigmap(key, data); err != nil {
 			return err
 		}

cmd/download.go

@@ -11,7 +11,7 @@ import (
 var downloadInfo cautils.DownloadInfo
 
 var downloadCmd = &cobra.Command{
-	Use:   fmt.Sprintf("Download framework <framework-name> [flags]\nSupported frameworks: %s", validFrameworks),
+	Use:   fmt.Sprintf("download framework <framework-name> [flags]\nSupported frameworks: %s", validFrameworks),
 	Short: "Download framework controls",
 	Long:  ``,
 	Args: func(cmd *cobra.Command, args []string) error {

cmd/framework.go

@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"flag"
 	"fmt"
 	"io"
 	"os"
@@ -32,7 +31,7 @@ type CLIHandler struct {
 
 var frameworkCmd = &cobra.Command{
 
-	Use:       fmt.Sprintf("framework <framework name> [`<glob patter>`/`-`] [flags]\nSupported frameworks: %s", validFrameworks),
+	Use:       fmt.Sprintf("framework <framework name> [`<glob pattern>`/`-`] [flags]\nSupported frameworks: %s", validFrameworks),
 	Short:     fmt.Sprintf("The framework you wish to use. Supported frameworks: %s", strings.Join(supportedFrameworks, ", ")),
 	Long:      "Execute a scan on a running Kubernetes cluster or `yaml`/`json` files (use glob) or `-` for stdin",
 	ValidArgs: supportedFrameworks,
@@ -95,22 +94,25 @@ func init() {
 	frameworkCmd.Flags().StringVarP(&scanInfo.Output, "output", "o", "", "Output file. Print output to file and not stdout")
 	frameworkCmd.Flags().BoolVarP(&scanInfo.Silent, "silent", "s", false, "Silent progress messages")
 	frameworkCmd.Flags().Uint16VarP(&scanInfo.FailThreshold, "fail-threshold", "t", 0, "Failure threshold is the percent bellow which the command fails and returns exit code 1")
-	frameworkCmd.Flags().BoolVarP(&scanInfo.DoNotSendResults, "results-locally", "", false, "Deprecated. Please use `--local` instead")
-	frameworkCmd.Flags().BoolVarP(&scanInfo.Submit, "submit", "", false, "Use this flag if you wish to send your Kubescape results to Armo backend to control exceptions and maintain chronological scan results. By default the results are not submitted")
-	frameworkCmd.Flags().BoolVarP(&scanInfo.Local, "local", "", false, "If you do not want your Kubescape results reported to Armo backend. Use this flag if you ran with the `--submit` flag in the past and you do not want to submit your current scan results")
+	frameworkCmd.Flags().BoolVarP(&scanInfo.DoNotSendResults, "results-locally", "", false, "Deprecated. Please use `--keep-local` instead")
+	frameworkCmd.Flags().BoolVarP(&scanInfo.Submit, "submit", "", false, "Send the scan results to Armo management portal where you can see the results in a user-friendly UI, choose your preferred compliance framework, check risk results history and trends, manage exceptions, get remediation recommendations and much more. By default the results are not submitted")
+	frameworkCmd.Flags().BoolVarP(&scanInfo.Local, "keep-local", "", false, "If you do not want your Kubescape results reported to Armo backend. Use this flag if you ran with the `--submit` flag in the past and you do not want to submit your current scan results")
 	frameworkCmd.Flags().StringVarP(&scanInfo.Account, "account", "", "", "Armo portal account ID. Default will load account ID from configMap or config file")
 
 }
 
 func CliSetup() error {
-	flag.Parse()
 	flagValidation()
 
 	var k8s *k8sinterface.KubernetesApi
+	var clusterConfig cautils.IClusterConfig
 	if !scanInfo.ScanRunningCluster() {
 		k8sinterface.ConnectedToCluster = false
+		clusterConfig = cautils.NewEmptyConfig()
 	} else {
 		k8s = k8sinterface.NewKubernetesApi()
+		// setup cluster config
+		clusterConfig =  cautils.ClusterConfigSetup(&scanInfo, k8s, getter.GetArmoAPIConnector())
 	}
 
 	processNotification := make(chan *cautils.OPASessionObj)
@@ -119,8 +121,9 @@ func CliSetup() error {
 	// policy handler setup
 	policyHandler := policyhandler.NewPolicyHandler(&processNotification, k8s)
 
-	// setup cluster config
-	clusterConfig := cautils.ClusterConfigSetup(&scanInfo, k8s, getter.NewArmoAPI())
+	if err := clusterConfig.SetCustomerGUID(scanInfo.Account); err != nil {
+		fmt.Println(err)
+	}
 
 	cautils.CustomerGUID = clusterConfig.GetCustomerGUID()
 	cautils.ClusterName = k8sinterface.GetClusterName()
@@ -184,11 +187,11 @@ func (clihandler *CLIHandler) Scan() error {
 
 func flagValidation() {
 	if scanInfo.DoNotSendResults {
-		fmt.Println("Deprecated. Please use `--local` instead")
+		fmt.Println("Deprecated. Please use `--keep-local` instead")
 	}
 
 	if scanInfo.Submit && scanInfo.Local {
-		fmt.Println("You can use `local` or `submit`, but not both")
+		fmt.Println("You can use `keep-local` or `submit`, but not both")
 		os.Exit(1)
 	}
 	if 100 < scanInfo.FailThreshold {

cmd/root.go

@@ -1,15 +1,29 @@
 package cmd
 
 import (
+	"flag"
+	"os"
+	"strings"
+
+	"github.com/armosec/kubescape/cautils/getter"
+	"github.com/golang/glog"
 	"github.com/spf13/cobra"
 )
 
 var cfgFile string
+var armoBEURLs = ""
+
+const envFlagUsage = "Send report results to specific URL. Format:<ReportReceiver>,<Backend>,<Frontend>.\n\t\tExample:report.armo.cloud,api.armo.cloud,portal.armo.cloud"
 
 var rootCmd = &cobra.Command{
 	Use:   "kubescape",
 	Short: "Kubescape is a tool for testing Kubernetes security posture",
-	Long:  `Kubescape is a tool for testing Kubernetes security posture based on NSA specifications.`,
+	Long:  `Kubescape is a tool for testing Kubernetes security posture based on NSA \ MITRE ATT&CK® specifications.`,
+	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+		flag.Parse()
+		InitArmoBEConnector()
+		return nil
+	},
 }
 
 func Execute() {
@@ -17,9 +31,38 @@ func Execute() {
 }
 
 func init() {
+	flag.CommandLine.StringVar(&armoBEURLs, "environment", "", envFlagUsage)
+	rootCmd.PersistentFlags().StringVar(&armoBEURLs, "environment", "", envFlagUsage)
+	rootCmd.PersistentFlags().MarkHidden("environment")
 	cobra.OnInitialize(initConfig)
+
 }
 
 // initConfig reads in config file and ENV variables if set.
 func initConfig() {
 }
+
+func InitArmoBEConnector() {
+	urlSlices := strings.Split(armoBEURLs, ",")
+	if len(urlSlices) > 3 {
+		glog.Errorf("Too many URLs")
+		os.Exit(1)
+	}
+	switch len(urlSlices) {
+	case 1:
+		switch urlSlices[0] {
+		case "dev":
+			getter.SetARMOAPIConnector(getter.NewARMOAPIDev())
+		case "":
+			getter.SetARMOAPIConnector(getter.NewARMOAPIProd())
+		default:
+			glog.Errorf("--environment flag usage: %s", envFlagUsage)
+			os.Exit(1)
+		}
+	case 2:
+		glog.Errorf("--environment flag usage: %s", envFlagUsage)
+		os.Exit(1)
+	case 3:
+		getter.SetARMOAPIConnector(getter.NewARMOAPICustomized(urlSlices[0], urlSlices[1], urlSlices[2]))
+	}
+}

opaprocessor/processorhandler.go

@@ -126,6 +126,7 @@ func (opap *OPAProcessor) processControl(control *opapolicy.Control) (*opapolicy
 	controlReport := opapolicy.ControlReport{}
 	controlReport.PortalBase = control.PortalBase
 	controlReport.ControlID = control.ControlID
+	controlReport.Control_ID = control.Control_ID // TODO: delete when 'id' is deprecated
 
 	controlReport.Name = control.Name
 	controlReport.Description = control.Description

opaprocessor/processorhandlerutils.go

@@ -98,6 +98,7 @@ func editRuleResponses(ruleResponses []opapolicy.RuleResponse) []opapolicy.RuleR
 				// resource found -> remove from slice
 				ruleResponses = removeFromSlice(ruleResponses, i)
 				lenRuleResponses -= 1
+				i -= 1
 				break
 			} else {
 				cleanRuleResponses(w)

policyhandler/filesloader_test.go

@@ -6,53 +6,44 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
-
-	"github.com/armosec/kubescape/cautils"
 )
 
-func combineYamlFile(base, rel string) string {
-	finalPath := []string{}
-	sBase := strings.Split(base, "/")
-	sRel := strings.Split(rel, "/")
-	for i := range sBase {
-		if cautils.StringInSlice(sRel, sBase[i]) != cautils.ValueNotFound {
-			finalPath = append(finalPath, sRel...)
-			break
-		}
-		finalPath = append(finalPath, sBase[i])
-	}
-	return fmt.Sprintf("/%s", filepath.Join(finalPath...))
-}
 func onlineBoutiquePath() string {
 	o, _ := os.Getwd()
-	return combineYamlFile(o, "kubescape/examples/online-boutique/*")
+	return filepath.Join(filepath.Dir(o), "examples/online-boutique/*")
 }
+
 func TestListFiles(t *testing.T) {
-	// files, errs := listFiles([]string{onlineBoutiquePath()})
-	// if len(errs) > 0 {
-	// 	t.Error(errs)
-	// }
-	// expected := 12
-	// if len(files) != expected {
-	// 	t.Errorf("wrong number of files, expected: %d, found: %d", expected, len(files))
-	// }
+	workDir, err := os.Getwd()
+	fmt.Printf("\n------------------\n%s,%v\n--------------\n", workDir, err)
+	filesPath := onlineBoutiquePath()
+	fmt.Printf("\n------------------\n%s\n--------------\n", filesPath)
+
+	files, errs := listFiles([]string{filesPath})
+	if len(errs) > 0 {
+		t.Error(errs)
+	}
+	expected := 12
+	if len(files) != expected {
+		t.Errorf("wrong number of files, expected: %d, found: %d", expected, len(files))
+	}
 }
 
 func TestLoadFiles(t *testing.T) {
-	// files, _ := listFiles([]string{onlineBoutiquePath()})
-	// loadFiles(files)
+	files, _ := listFiles([]string{onlineBoutiquePath()})
+	loadFiles(files)
 }
 
 func TestLoadFile(t *testing.T) {
-	// files, _ := listFiles([]string{strings.Replace(onlineBoutiquePath(), "*", "adservice.yaml", 1)})
-	// _, err := loadFile(files[0])
-	// if err != nil {
-	// 	t.Errorf("%v", err)
-	// }
+	files, _ := listFiles([]string{strings.Replace(onlineBoutiquePath(), "*", "adservice.yaml", 1)})
+	_, err := loadFile(files[0])
+	if err != nil {
+		t.Errorf("%v", err)
+	}
 }
 func TestLoadResources(t *testing.T) {
-
-	// k8sResources, err = policyHandler.loadResources(opaSessionObj.Frameworks, scanInfo)
+	// policyHandler := &PolicyHandler{}
+	// k8sResources, err := policyHandler.loadResources(opaSessionObj.Frameworks, scanInfo)
 	// files, _ := listFiles([]string{onlineBoutiquePath()})
 	// bb, err := loadFile(files[0])
 	// if len(err) > 0 {

policyhandler/handlepullpolicies.go

@@ -2,6 +2,7 @@ package policyhandler
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/armosec/kubescape/cautils"
 	"github.com/armosec/kubescape/cautils/armotypes"
@@ -24,6 +25,9 @@ func (policyHandler *PolicyHandler) GetPoliciesFromBackend(notification *opapoli
 					exceptionPolicies = append(exceptionPolicies, recExceptionPolicies...)
 				}
 			} else if err != nil {
+				if strings.Contains(err.Error(), "unsupported protocol scheme") {
+					err = fmt.Errorf("failed to download from GitHub release, try running with `--use-default` flag")
+				}
 				return nil, nil, fmt.Errorf("kind: %v, name: %s, error: %s", rule.Kind, rule.Name, err.Error())
 			}
 

resultshandling/printer/printresults.go

@@ -45,12 +45,7 @@ func calculatePostureScore(postureReport *opapolicy.PostureReport) float32 {
 	totalFailed := 0
 	for _, frameworkReport := range postureReport.FrameworkReports {
 		for _, controlReport := range frameworkReport.ControlReports {
-			for _, ruleReport := range controlReport.RuleReports {
-				for _, ruleResponses := range ruleReport.RuleResponses {
-					totalFailed += len(ruleResponses.AlertObject.K8SApiObjects)
-					totalFailed += len(ruleResponses.AlertObject.ExternalObjects)
-				}
-			}
+			totalFailed += controlReport.GetNumberOfFailedResources()
 			totalResources += controlReport.GetNumberOfResources()
 		}
 	}
@@ -133,7 +128,7 @@ func (printer *Printer) PrintResults() {
 
 func (printer *Printer) printSummary(controlName string, controlSummary *ControlSummary) {
 	cautils.SimpleDisplay(printer.writer, "Summary - ")
-	cautils.SuccessDisplay(printer.writer, "Passed:%v   ", controlSummary.TotalResources-controlSummary.TotalFailed)
+	cautils.SuccessDisplay(printer.writer, "Passed:%v   ", controlSummary.TotalResources-controlSummary.TotalFailed-controlSummary.TotalWarnign)
 	cautils.WarningDisplay(printer.writer, "Excluded:%v   ", controlSummary.TotalWarnign)
 	cautils.FailureDisplay(printer.writer, "Failed:%v   ", controlSummary.TotalFailed)
 	cautils.InfoDisplay(printer.writer, "Total:%v\n", controlSummary.TotalResources)

resultshandling/reporter/reporteventreceiverutils.go

@@ -37,7 +37,7 @@ func initEventReceiverURL() *url.URL {
 	urlObj := url.URL{}
 
 	urlObj.Scheme = "https"
-	urlObj.Host = getter.ArmoERURL
+	urlObj.Host = getter.GetArmoAPIConnector().GetReportReceiverURL()
 	urlObj.Path = "/k8s/postureReport"
 	q := urlObj.Query()
 	q.Add("customerGUID", uuid.FromStringOrNil(cautils.CustomerGUID).String())