support url input and update readme (#40)

* split to functions

* update package name to kubescape

* support url input, update readme

.gitignore

@@ -1,3 +1,4 @@
 *.vs*
 *go.sum*
-*kubescape*
+*kubescape*
+*debug*

.gitmodules

@@ -1,4 +0,0 @@
-[submodule "vendor/github.com/armosec/capacketsgo"]
-	path = vendor/github.com/armosec/capacketsgo
-	url = git@github.com:armosec/capacketsgo.git
-	branch = master

README.md

@@ -11,29 +11,28 @@ Use Kubescape to test clusters or scan single YAML files and integrate it to you
 <img src="docs/demo.gif">
 
 # TL;DR
-## Installation
-To install the tool locally, run this:
+## Install & Run
 
+### Install:
 ```
 curl -s https://raw.githubusercontent.com/armosec/kubescape/master/install.sh | /bin/bash
 ```
 
-<img src="docs/install.jpeg">
-
-## Run
-
-### Cluster testing
-To get a fast check of the security posture of your Kubernetes cluster, run this:
-
+### Run:
 ```
 kubescape scan framework nsa --exclude-namespaces kube-system,kube-public
 ```
 
 If you wish to scan all namespaces in your cluster, remove the `--exclude-namespaces` flag.
 
-### Pre-deployment testing
-Check your YAML files before you're deploying, simply add them at the end of command line:
+<img src="docs/summary.png">
 
+
+
+## Usage & Examples
+
+### Pre-Deployment Testing
+Check your YAML files before you're deploying, simply add them at the end of command line:
 ```
 kubescape scan framework nsa *.yaml
 ```
@@ -44,16 +43,42 @@ Kubescape can produce output fitting for later processing:
 * JSON (`-o json`)
 * JUnit XML (`-o junit`)
 
-Example:
+
+* Scan a running Kubernetes cluster with [`nsa`](https://www.nsa.gov/News-Features/Feature-Stories/Article-View/Article/2716980/nsa-cisa-release-kubernetes-hardening-guidance/) framework
 ```
-kubescape scan framework nsa --silent -o -junit > results.xml
+kubescape scan framework nsa --exclude-namespaces kube-system,kube-public
 ```
 
-<img src="docs/summary.png">
+* Scan a running Kubernetes cluster with [`mitre`](https://www.microsoft.com/security/blog/2020/04/02/attack-matrix-kubernetes/) framework
+```
+kubescape scan framework mitre --exclude-namespaces kube-system,kube-public
+```
+
+
+* Scan local `yaml`/`json` files
+```
+kubescape scan framework nsa examples/online-boutique/*
+```
+
+
+* Scan `yaml`/`json` files from url
+```
+kubescape scan framework nsa https://raw.githubusercontent.com/GoogleCloudPlatform/microservices-demo/master/release/kubernetes-manifests.yaml
+```
+
+* Output in `json` format
+```
+kubescape scan framework nsa --exclude-namespaces kube-system,kube-public --silence -o json > results.json
+```
+
+* Output in `junit xml` format
+```
+kubescape scan framework nsa --exclude-namespaces kube-system,kube-public --silence -o junit > results.xml
+```
 
 # How to build 
 
-Note: development (and the release process) is done with Go 1.16 
+Note: development (and the release process) is done with Go `1.16`
 
 1. Clone Project
 ```

cautils/datastructures.go

@@ -1,7 +1,7 @@
 package cautils
 
 import (
-	"kube-escape/cautils/opapolicy"
+	"kubescape/cautils/opapolicy"
 )
 
 // K8SResources map[<api group>/<api version>/<resource>]<resource object>

cautils/k8sinterface/k8sconfig_test.go

@@ -3,7 +3,7 @@ package k8sinterface
 import (
 	"testing"
 
-	"kube-escape/cautils/cautils"
+	"kubescape/cautils/cautils"
 )
 
 func TestGetGroupVersionResource(t *testing.T) {

cautils/k8sinterface/k8sdynamic.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"strings"
 
-	"kube-escape/cautils/cautils"
+	"kubescape/cautils/cautils"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"

cautils/k8sinterface/k8sstatic.go

@@ -3,7 +3,7 @@ package k8sinterface
 import (
 	"context"
 
-	"kube-escape/cautils/cautils"
+	"kubescape/cautils/cautils"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

cautils/k8sinterface/workload.go

@@ -3,7 +3,7 @@ package k8sinterface
 import (
 	"encoding/json"
 
-	"kube-escape/cautils/apis"
+	"kubescape/cautils/apis"
 
 	corev1 "k8s.io/api/core/v1"
 

cautils/k8sinterface/workloadmethods.go

@@ -7,8 +7,8 @@ import (
 	"strings"
 	"time"
 
-	"kube-escape/cautils/apis"
-	"kube-escape/cautils/cautils"
+	"kubescape/cautils/apis"
+	"kubescape/cautils/cautils"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

cautils/opapolicy/datastructures.go

@@ -3,7 +3,7 @@ package opapolicy
 import (
 	"time"
 
-	armotypes "kube-escape/cautils/armotypes"
+	armotypes "kubescape/cautils/armotypes"
 )
 
 type AlertScore float32

cautils/opapolicy/datastructures_mock.go

@@ -3,7 +3,7 @@ package opapolicy
 import (
 	"time"
 
-	armotypes "kube-escape/cautils/armotypes"
+	armotypes "kubescape/cautils/armotypes"
 )
 
 // Mock A

cautils/opapolicy/resources/resourcesutils.go

@@ -8,7 +8,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"kube-escape/cautils/k8sinterface"
+	"kubescape/cautils/k8sinterface"
 
 	"github.com/golang/glog"
 	"github.com/open-policy-agent/opa/storage"

cmd/framework.go

@@ -4,23 +4,20 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"io/ioutil"
-	"kube-escape/cautils"
-	"kube-escape/cautils/armotypes"
-	"kube-escape/cautils/k8sinterface"
-	"kube-escape/cautils/opapolicy"
-	"kube-escape/opaprocessor"
-	"kube-escape/policyhandler"
-	"kube-escape/printer"
+	"kubescape/cautils"
+	"kubescape/cautils/armotypes"
+	"kubescape/cautils/k8sinterface"
+	"kubescape/cautils/opapolicy"
+	"kubescape/opaprocessor"
+	"kubescape/policyhandler"
+	"kubescape/printer"
 	"os"
-	"strings"
 
 	"github.com/spf13/cobra"
 )
 
 var scanInfo opapolicy.ScanInfo
 var supportedFrameworks = []string{"nsa", "mitre"}
-var isSilent bool
 
 type CLIHandler struct {
 	policyHandler *policyhandler.PolicyHandler
@@ -31,7 +28,7 @@ var frameworkCmd = &cobra.Command{
 	Use:       "framework <framework name>",
 	Short:     "The framework you wish to use. Supported frameworks: nsa, mitre",
 	Long:      ``,
-	ValidArgs: []string{"nsa", "mitre"},
+	ValidArgs: supportedFrameworks,
 	Args: func(cmd *cobra.Command, args []string) error {
 		if len(args) < 1 {
 			return errors.New("requires at least one argument")
@@ -59,23 +56,13 @@ func init() {
 	scanCmd.AddCommand(frameworkCmd)
 	scanInfo = opapolicy.ScanInfo{}
 	frameworkCmd.Flags().StringVarP(&scanInfo.ExcludedNamespaces, "exclude-namespaces", "e", "", "namespaces to exclude from check")
-	frameworkCmd.Flags().StringVarP(&scanInfo.Output, "output", "o", "pretty-printer", "output format")
-	frameworkCmd.Flags().BoolVarP(&scanInfo.Silent, "silent", "s", false, "silent output")
-}
-
-func processYamlInput(yamls string) {
-	listOfYamls := strings.Split(yamls, ",")
-	for _, yaml := range listOfYamls {
-		dat, err := ioutil.ReadFile(yaml)
-		if err != nil {
-			fmt.Printf("Could not open file: %s.", yaml)
-		}
-		fmt.Print(string(dat))
-	}
-
+	frameworkCmd.Flags().StringVarP(&scanInfo.Output, "output", "o", "pretty-printer", "output format. supported formats: 'pretty-printer'/'json'/'junit'")
+	frameworkCmd.Flags().BoolVarP(&scanInfo.Silent, "silent", "s", false, "silent progress output")
 }
 
 func CliSetup() error {
+	flag.Parse()
+
 	k8s := k8sinterface.NewKubernetesApi()
 
 	processNotification := make(chan *cautils.OPASessionObj)
@@ -117,7 +104,6 @@ func (clihandler *CLIHandler) Scan() error {
 		},
 		Designators: armotypes.PortalDesignator{},
 	}
-	flag.Parse()
 	switch policyNotification.NotificationType {
 	case opapolicy.TypeExecPostureScan:
 		go func() {

cmd/scan.go

@@ -7,7 +7,7 @@ import (
 // scanCmd represents the scan command
 var scanCmd = &cobra.Command{
 	Use:   "scan",
-	Short: "Scan command",
+	Short: "Scan the current running cluster or specified yaml files",
 	Long:  `The action you want to perform`,
 	Run: func(cmd *cobra.Command, args []string) {
 	},

examples/online-boutique/README.md

@@ -1,8 +0,0 @@
-# ./kubernetes-manifests
-
-:warning: Kubernetes manifests provided in this directory are not directly
-deployable to a cluster. They are meant to be used with `skaffold` command to
-insert the correct `image:` tags.
-
-Use the manifests in [/release](/release) directory which are configured with
-pre-built public images.

examples/online-boutique/bi-monitor.yaml

@@ -1,59 +0,0 @@
-apiVersion: v1
-data:
-  customer: Q3liZXJBcm1vclRlc3Rz
-  password: bml1ZGhmMjgzcnUyM3JrZQ==
-  username: ZHdlcnRlbnRAY3liZXJhcm1vci5pbw==
-kind: Secret
-metadata:
-  name: bi-monitor-secret
-type: Opaque
-
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: bi-monitor
-spec:
-  type: ClusterIP
-  selector:
-    app: bi-monitor
-  ports:
-    - name: http
-      port: 80
-      targetPort: 80
-
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: bi-monitor
-  labels:
-    app: bi-monitor
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: bi-monitor
-  template:
-    metadata:
-      labels:
-        app: bi-monitor
-    spec:
-      containers:
-        - name: monitor
-          image: quay.io/armosec/demoservice:v1-debian
-          env:
-            - name: THREAD_TIMEOUT
-              value: "10"
-            - name: SLEEP_DURATION
-              value: "1"
-            - name: DEMO_TARGETS
-              value: "http://frontend:80 https://cisco.com"
-            - name: CAA_SIGNATURE_DEBUG_DEEP
-          volumeMounts:
-            - name: bi-monitor-secret
-              mountPath: /etc/secrets
-      volumes:
-        - name: bi-monitor-secret
-          secret:
-            secretName: bi-monitor-secret

go.mod

@@ -1,4 +1,4 @@
-module kube-escape
+module kubescape
 
 go 1.16
 

inputhandler/clihandler/clihandler_test.go

@@ -1 +0,0 @@
-package clihandler

inputhandler/clihandler/flaghandler.go

@@ -1,97 +0,0 @@
-package clihandler
-
-import (
-	"flag"
-	"fmt"
-	"strings"
-
-	"kube-escape/cautils/opapolicy"
-)
-
-type FlagHandler struct {
-	policyIdentifier *opapolicy.PolicyIdentifier
-}
-
-func NewFlagHandler() *FlagHandler {
-	flag.Parse()
-	return &FlagHandler{}
-}
-
-func (flagHandler *FlagHandler) ExecuteScan() bool {
-	return flagHandler.policyIdentifier != nil
-}
-
-// SetupHTTPListener set up listening http servers
-func (flagHandler *FlagHandler) ParseFlag() {
-	f := "help"
-	if len(flag.Args()) >= 1 {
-		f = strings.ToLower(flag.Arg(0))
-	}
-	switch f {
-	case "scan":
-		flagHandler.Scan()
-	case "version":
-		flagHandler.Version()
-	case "help":
-		flagHandler.Help()
-	default:
-		fmt.Println("unknown input argument")
-		flagHandler.Help()
-	}
-}
-
-func (flagHandler *FlagHandler) Help() {
-	fmt.Println("Run: kubescape scan framework nsa --exclude-namespaces kube-system,kube-public")
-}
-
-func (flagHandler *FlagHandler) Version() {
-	fmt.Println("betav1")
-}
-
-func (flagHandler *FlagHandler) Scan() {
-	f := "help"
-	if len(flag.Args()) >= 2 {
-		f = strings.ToLower(flag.Arg(1))
-	}
-	switch f {
-	case "framework":
-		flagHandler.ScanFramework()
-	case "control":
-		flagHandler.ScanControl()
-	case "help":
-		flagHandler.ScanHelp()
-	default:
-		fmt.Println("unknown input argument")
-		flagHandler.ScanHelp()
-	}
-}
-func (flagHandler *FlagHandler) ScanFramework() {
-	frameworkName := strings.ToUpper(flag.Arg(2))
-	// if cautils.StringInSlice(SupportedFrameworks(), frameworkName) == cautils.ValueNotFound {
-	// 	fmt.Printf("framework %s not supported, supported frameworks: %v", frameworkName, SupportedFrameworks())
-	// 	return
-	// }
-	flagHandler.policyIdentifier = &opapolicy.PolicyIdentifier{
-		Kind: opapolicy.KindFramework,
-		Name: frameworkName,
-	}
-}
-func (flagHandler *FlagHandler) ScanControl() {
-	flagHandler.policyIdentifier = &opapolicy.PolicyIdentifier{
-		Kind: opapolicy.KindControl,
-		Name: strings.ToUpper(flag.Arg(3)),
-	}
-}
-func (flagHandler *FlagHandler) ScanHelp() {
-	fmt.Println("")
-}
-func (flagHandler *FlagHandler) ScanFrameworkHelp() {
-	fmt.Println("Run framework nsa or mitre")
-}
-func (flagHandler *FlagHandler) ScanControlHelp() {
-	fmt.Println("not supported")
-}
-
-func SupportedFrameworks() []string {
-	return []string{"nsa", "mitre"} // TODO - get from BE
-}

main.go

@@ -1,6 +1,6 @@
 package main
 
-import "kube-escape/cmd"
+import "kubescape/cmd"
 
 func main() {
 	cmd.Execute()

opaprocessor/processorhandler.go

@@ -3,13 +3,13 @@ package opaprocessor
 import (
 	"context"
 	"fmt"
-	"kube-escape/cautils"
+	"kubescape/cautils"
 	"time"
 
-	"kube-escape/cautils/k8sinterface"
+	"kubescape/cautils/k8sinterface"
 
-	"kube-escape/cautils/opapolicy"
-	"kube-escape/cautils/opapolicy/resources"
+	"kubescape/cautils/opapolicy"
+	"kubescape/cautils/opapolicy/resources"
 
 	"github.com/golang/glog"
 	"github.com/open-policy-agent/opa/ast"

opaprocessor/processorhandler_test.go

@@ -3,18 +3,18 @@ package opaprocessor
 import (
 	"context"
 	"encoding/json"
-	"kube-escape/cautils"
+	"kubescape/cautils"
 	"os"
 	"path"
 	"strings"
 	"testing"
 
-	"kube-escape/cautils/k8sinterface"
+	"kubescape/cautils/k8sinterface"
 	// _ "k8s.io/client-go/plugin/pkg/client/auth"
 	restclient "k8s.io/client-go/rest"
 
-	"kube-escape/cautils/opapolicy"
-	"kube-escape/cautils/opapolicy/resources"
+	"kubescape/cautils/opapolicy"
+	"kubescape/cautils/opapolicy/resources"
 
 	"github.com/open-policy-agent/opa/ast"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -82,10 +82,10 @@ func TestCompromisedRegistries(t *testing.T) {
 	// k8sResources["/v1/pods"] = k8sinterface.ConvertUnstructuredSliceToMap(k8sinterface.V1KubeSystemNamespaceMock().Items)
 	k8sResources["/v1/pods"] = k8sinterface.V1AllClusterWithCompromisedRegistriesMock().Items
 	wd, _ := os.Getwd()
-	baseDirName := "kube-escape"
+	baseDirName := "kubescape"
 	idx := strings.Index(wd, baseDirName)
 	wd = wd[0:idx]
-	resources.RegoDependenciesPath = path.Join(wd, "/kube-escape/vendor/asterix.cyberarmor.io/cyberarmor/capacketsgo/opapolicy/resources/rego/dependencies")
+	resources.RegoDependenciesPath = path.Join(wd, "/kubescape/vendor/asterix.cyberarmor.io/cyberarmor/capacketsgo/opapolicy/resources/rego/dependencies")
 	k8sinterface.K8SConfig = &restclient.Config{}
 
 	opaProcessor := NewOPAProcessorMock()
@@ -108,7 +108,7 @@ func TestCompromisedRegistries(t *testing.T) {
 // 	k8sResources := make(cautils.K8SResources)
 // 	// k8sResources["/v1/pods"] = k8sinterface.ConvertUnstructuredSliceToMap(k8sinterface.V1KubeSystemNamespaceMock().Items)
 // 	k8sResources["/v1/pods"] = k8sinterface.V1KubeSystemNamespaceMock().Items
-// 	resources.RegoDependenciesPath = "/home/david/go/src/kube-escape/vendor/asterix.cyberarmor.io/cyberarmor/capacketsgo/opapolicy/resources/rego/dependencies"
+// 	resources.RegoDependenciesPath = "/home/david/go/src/kubescape/vendor/asterix.cyberarmor.io/cyberarmor/capacketsgo/opapolicy/resources/rego/dependencies"
 // 	opaProcessor := NewOPAProcessorMock()
 
 // 	// set opaSessionObj

opaprocessor/processorhandlerutils.go

@@ -1,12 +1,12 @@
 package opaprocessor
 
 import (
-	"kube-escape/cautils"
+	"kubescape/cautils"
 
-	pkgcautils "kube-escape/cautils/cautils"
-	"kube-escape/cautils/k8sinterface"
-	"kube-escape/cautils/opapolicy"
-	resources "kube-escape/cautils/opapolicy/resources"
+	pkgcautils "kubescape/cautils/cautils"
+	"kubescape/cautils/k8sinterface"
+	"kubescape/cautils/opapolicy"
+	resources "kubescape/cautils/opapolicy/resources"
 
 	"github.com/golang/glog"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"

policyhandler/filesloader.go

@@ -5,10 +5,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"kube-escape/cautils"
-	"kube-escape/cautils/k8sinterface"
-	"kube-escape/cautils/opapolicy"
+	"kubescape/cautils"
+	"kubescape/cautils/k8sinterface"
+	"kubescape/cautils/opapolicy"
 	"path/filepath"
+	"strings"
 
 	"gopkg.in/yaml.v2"
 )
@@ -18,27 +19,39 @@ var (
 	JSON_PREFIX = []string{".json"}
 )
 
+type FileFormat string
+
+const (
+	YAML_FILE_FORMAT FileFormat = "yaml"
+	JSON_FILE_FORMAT FileFormat = "json"
+)
+
 func (policyHandler *PolicyHandler) loadResources(frameworks []opapolicy.Framework, scanInfo *opapolicy.ScanInfo) (*cautils.K8SResources, error) {
+	workloads := []k8sinterface.IWorkload{}
 
-	files, errs := listFiles(scanInfo.InputPatterns)
-	if len(errs) > 0 {
-		cautils.ErrorDisplay(fmt.Sprintf("%v", errs)) // TODO - print error
+	// load resource from local file system
+	w, err := loadResourcesFromFiles(scanInfo.InputPatterns)
+	if err != nil {
+		return nil, err
 	}
-	if len(files) == 0 {
-		return nil, fmt.Errorf("empty list of files - no files found")
+	if w != nil {
+		workloads = append(workloads, w...)
 	}
 
-	workloads, errs := loadFiles(files)
-	if len(errs) > 0 {
-		cautils.ErrorDisplay(fmt.Sprintf("%v", errs)) // TODO - print error
+	// load resource from url
+	w, err = loadResourcesFromUrl(scanInfo.InputPatterns)
+	if err != nil {
+		return nil, err
 	}
-	if len(workloads) == 0 {
-		return nil, fmt.Errorf("empty list of workloads - no workloads valid workloads found")
+	if w != nil {
+		workloads = append(workloads, w...)
 	}
 
+	// map all resources: map["/group/version/kind"][]<k8s workloads>
 	allResources := mapResources(workloads)
 
 	// build resources map
+	// map resources based on framework requrid resources: map["/group/version/kind"][]<k8s workloads>
 	k8sResources := setResourceMap(frameworks)
 
 	// save only relevant resources
@@ -52,6 +65,26 @@ func (policyHandler *PolicyHandler) loadResources(frameworks []opapolicy.Framewo
 
 }
 
+func loadResourcesFromFiles(inputPatterns []string) ([]k8sinterface.IWorkload, error) {
+	files, errs := listFiles(inputPatterns)
+	if len(errs) > 0 {
+		cautils.ErrorDisplay(fmt.Sprintf("%v", errs)) // TODO - print error
+	}
+	if len(files) == 0 {
+		return nil, nil
+	}
+
+	workloads, errs := loadFiles(files)
+	if len(errs) > 0 {
+		cautils.ErrorDisplay(fmt.Sprintf("%v", errs)) // TODO - print error
+	}
+	if len(workloads) == 0 {
+		return workloads, fmt.Errorf("empty list of workloads - no workloads found")
+	}
+	return workloads, nil
+}
+
+// build resources map
 func mapResources(workloads []k8sinterface.IWorkload) map[string][]map[string]interface{} {
 	allResources := map[string][]map[string]interface{}{}
 	for i := range workloads {
@@ -76,30 +109,46 @@ func mapResources(workloads []k8sinterface.IWorkload) map[string][]map[string]in
 
 }
 
-// // build resources map
 func loadFiles(filePaths []string) ([]k8sinterface.IWorkload, []error) {
 	workloads := []k8sinterface.IWorkload{}
 	errs := []error{}
 	for i := range filePaths {
-		w, e := loadFile(filePaths[i])
+		f, err := loadFile(filePaths[i])
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		w, e := readFile(f, getFileFormat(filePaths[i]))
 		errs = append(errs, e...)
-		workloads = append(workloads, w...)
+		if w != nil {
+			workloads = append(workloads, w...)
+		}
 	}
 	return workloads, errs
 }
 
-func loadFile(filePath string) ([]k8sinterface.IWorkload, []error) {
-	if isYaml(filePath) {
-		return loadYamlFile(filePath)
-	} else if isJson(filePath) {
-		return loadJsonFile(filePath)
+func loadFile(filePath string) ([]byte, error) {
+	return ioutil.ReadFile(filePath)
+}
+func readFile(fileContent []byte, fileFromat FileFormat) ([]k8sinterface.IWorkload, []error) {
+
+	switch fileFromat {
+	case YAML_FILE_FORMAT:
+		return readYamlFile(fileContent)
+	case JSON_FILE_FORMAT:
+		return readJsonFile(fileContent)
+	default:
+		return nil, []error{fmt.Errorf("file extension %s not supported", fileFromat)}
 	}
-	return nil, []error{fmt.Errorf("file extension %s not supported, file name: %s", filepath.Ext(filePath), filePath)}
+
 }
 func listFiles(patterns []string) ([]string, []error) {
 	files := []string{}
 	errs := []error{}
 	for i := range patterns {
+		if strings.HasPrefix(patterns[i], "http") {
+			continue
+		}
 		f, err := filepath.Glob(patterns[i])
 		if err != nil {
 			errs = append(errs, err)
@@ -110,12 +159,8 @@ func listFiles(patterns []string) ([]string, []error) {
 	return files, errs
 }
 
-func loadYamlFile(filePath string) ([]k8sinterface.IWorkload, []error) {
+func readYamlFile(yamlFile []byte) ([]k8sinterface.IWorkload, []error) {
 	errs := []error{}
-	yamlFile, err := ioutil.ReadFile(filePath)
-	if err != nil {
-		return nil, []error{err}
-	}
 
 	r := bytes.NewReader(yamlFile)
 	dec := yaml.NewDecoder(r)
@@ -124,24 +169,23 @@ func loadYamlFile(filePath string) ([]k8sinterface.IWorkload, []error) {
 	var t interface{}
 	for dec.Decode(&t) == nil {
 		j := convertYamlToJson(t)
+		if j == nil {
+			continue
+		}
 		if obj, ok := j.(map[string]interface{}); ok {
 			yamlObjs = append(yamlObjs, k8sinterface.NewWorkloadObj(obj))
 		} else {
-			errs = append(errs, fmt.Errorf("failed to convert yaml file %s file to map[string]interface", filePath))
+			errs = append(errs, fmt.Errorf("failed to convert yaml file to map[string]interface, file content: %v", j))
 		}
 	}
 
 	return yamlObjs, errs
 }
 
-func loadJsonFile(filePath string) ([]k8sinterface.IWorkload, []error) {
+func readJsonFile(jsonFile []byte) ([]k8sinterface.IWorkload, []error) {
 	workloads := []k8sinterface.IWorkload{}
-	jsonFile, err := ioutil.ReadFile(filePath)
-	if err != nil {
-		return workloads, []error{err}
-	}
 	var jsonObj interface{}
-	if err = json.Unmarshal(jsonFile, &jsonObj); err != nil {
+	if err := json.Unmarshal(jsonFile, &jsonObj); err != nil {
 		return workloads, []error{err}
 	}
 
@@ -183,3 +227,13 @@ func isYaml(filePath string) bool {
 func isJson(filePath string) bool {
 	return cautils.StringInSlice(YAML_PREFIX, filepath.Ext(filePath)) != cautils.ValueNotFound
 }
+
+func getFileFormat(filePath string) FileFormat {
+	if isYaml(filePath) {
+		return YAML_FILE_FORMAT
+	} else if isJson(filePath) {
+		return JSON_FILE_FORMAT
+	} else {
+		return FileFormat(filePath)
+	}
+}

policyhandler/filesloader_test.go

@@ -2,7 +2,7 @@ package policyhandler
 
 import (
 	"fmt"
-	"kube-escape/cautils"
+	"kubescape/cautils"
 	"os"
 	"path/filepath"
 	"strings"
@@ -44,13 +44,10 @@ func TestLoadFiles(t *testing.T) {
 
 func TestLoadFile(t *testing.T) {
 	files, _ := listFiles([]string{strings.Replace(onlineBoutiquePath(), "*", "bi-monitor.yaml", 1)})
-	bb, err := loadFile(files[0])
-	if len(err) > 0 {
+	_, err := loadFile(files[0])
+	if err != nil {
 		t.Errorf("%v", err)
 	}
-	for i := range bb {
-		t.Errorf("%s", bb[i].ToString())
-	}
 }
 func TestLoadResources(t *testing.T) {
 

policyhandler/handlenotification.go

@@ -2,11 +2,11 @@ package policyhandler
 
 import (
 	"fmt"
-	"kube-escape/cautils"
+	"kubescape/cautils"
 
-	"kube-escape/cautils/k8sinterface"
+	"kubescape/cautils/k8sinterface"
 
-	"kube-escape/cautils/opapolicy"
+	"kubescape/cautils/opapolicy"
 )
 
 // PolicyHandler -

policyhandler/handlepullpolicies.go

@@ -8,7 +8,7 @@ import (
 	"net/url"
 	"strings"
 
-	"kube-escape/cautils/opapolicy"
+	"kubescape/cautils/opapolicy"
 )
 
 // URLEncoder encode url

policyhandler/k8sresources.go

@@ -2,13 +2,13 @@ package policyhandler
 
 import (
 	"fmt"
-	"kube-escape/cautils"
+	"kubescape/cautils"
 	"strings"
 
-	"kube-escape/cautils/k8sinterface"
+	"kubescape/cautils/k8sinterface"
 
-	"kube-escape/cautils/armotypes"
-	"kube-escape/cautils/opapolicy"
+	"kubescape/cautils/armotypes"
+	"kubescape/cautils/opapolicy"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"

policyhandler/k8sresourcesutils.go

@@ -1,10 +1,10 @@
 package policyhandler
 
 import (
-	"kube-escape/cautils"
+	"kubescape/cautils"
 
-	"kube-escape/cautils/k8sinterface"
-	"kube-escape/cautils/opapolicy"
+	"kubescape/cautils/k8sinterface"
+	"kubescape/cautils/opapolicy"
 )
 
 func setResourceMap(frameworks []opapolicy.Framework) *cautils.K8SResources {

policyhandler/k8sresourcesutils_test.go

@@ -1,8 +1,8 @@
 package policyhandler
 
 import (
-	"kube-escape/cautils/k8sinterface"
-	"kube-escape/cautils/opapolicy"
+	"kubescape/cautils/k8sinterface"
+	"kubescape/cautils/opapolicy"
 
 	"testing"
 )

policyhandler/urlloader.go

@@ -0,0 +1,73 @@
+package policyhandler
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"kubescape/cautils"
+	"kubescape/cautils/k8sinterface"
+	"net/http"
+	"strings"
+)
+
+func loadResourcesFromUrl(inputPatterns []string) ([]k8sinterface.IWorkload, error) {
+	urls := listUrls(inputPatterns)
+	if len(urls) == 0 {
+		return nil, nil
+	}
+
+	workloads, errs := downloadFiles(urls)
+	if len(errs) > 0 {
+		cautils.ErrorDisplay(fmt.Sprintf("%v", errs)) // TODO - print error
+	}
+	if len(workloads) == 0 {
+		return workloads, fmt.Errorf("empty list of workloads - no workloads valid workloads found")
+	}
+	return workloads, nil
+}
+
+func listUrls(patterns []string) []string {
+	urls := []string{}
+	for i := range patterns {
+		if strings.HasPrefix(patterns[i], "http") {
+			urls = append(urls, patterns[i])
+		}
+	}
+	return urls
+}
+
+func downloadFiles(urls []string) ([]k8sinterface.IWorkload, []error) {
+	workloads := []k8sinterface.IWorkload{}
+	errs := []error{}
+	for i := range urls {
+		f, err := downloadFile(urls[i])
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		w, e := readFile(f, getFileFormat(urls[i]))
+		errs = append(errs, e...)
+		if w != nil {
+			workloads = append(workloads, w...)
+		}
+	}
+	return workloads, errs
+}
+
+func downloadFile(url string) ([]byte, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < 200 || 301 < resp.StatusCode {
+		return nil, fmt.Errorf("failed to download file, url: '%s', status code: %s", url, resp.Status)
+	}
+	return streamToByte(resp.Body), nil
+}
+
+func streamToByte(stream io.Reader) []byte {
+	buf := new(bytes.Buffer)
+	buf.ReadFrom(stream)
+	return buf.Bytes()
+}

printer/junit.go

@@ -3,7 +3,7 @@ package printer
 import (
 	"encoding/xml"
 	"fmt"
-	"kube-escape/cautils/opapolicy"
+	"kubescape/cautils/opapolicy"
 )
 
 type JUnitTestSuites struct {

printer/printresults.go

@@ -4,12 +4,12 @@ import (
 	"encoding/json"
 	"encoding/xml"
 	"fmt"
-	"kube-escape/cautils"
+	"kubescape/cautils"
 	"os"
 	"sort"
 
-	"kube-escape/cautils/k8sinterface"
-	"kube-escape/cautils/opapolicy"
+	"kubescape/cautils/k8sinterface"
+	"kubescape/cautils/opapolicy"
 
 	"github.com/enescakir/emoji"
 	"github.com/olekukonko/tablewriter"

printer/summeryhelpers.go

@@ -3,8 +3,8 @@ package printer
 import (
 	"fmt"
 
-	"kube-escape/cautils/k8sinterface"
-	"kube-escape/cautils/opapolicy"
+	"kubescape/cautils/k8sinterface"
+	"kubescape/cautils/opapolicy"
 )
 
 // Group workloads by namespace - return {"namespace": <[]WorkloadSummary>}