From 2d5ed19d6dd417d2db8184a1827305768a2f613a Mon Sep 17 00:00:00 2001 From: David Wertenteil <64066841+dwertent@users.noreply.github.com> Date: Sun, 29 Aug 2021 10:20:12 +0300 Subject: [PATCH] support url input and update readme (#40) * split to functions * update package name to kubescape * support url input, update readme --- .gitignore | 3 +- .gitmodules | 4 - README.md | 55 +++++--- cautils/datastructures.go | 2 +- cautils/k8sinterface/k8sconfig_test.go | 2 +- cautils/k8sinterface/k8sdynamic.go | 2 +- cautils/k8sinterface/k8sstatic.go | 2 +- cautils/k8sinterface/workload.go | 2 +- cautils/k8sinterface/workloadmethods.go | 4 +- cautils/opapolicy/datastructures.go | 2 +- cautils/opapolicy/datastructures_mock.go | 2 +- cautils/opapolicy/resources/resourcesutils.go | 2 +- cmd/framework.go | 38 ++---- cmd/scan.go | 2 +- examples/{example.sh => example.md} | 0 examples/online-boutique/README.md | 8 -- examples/online-boutique/bi-monitor.yaml | 59 --------- go.mod | 2 +- inputhandler/clihandler/clihandler_test.go | 1 - inputhandler/clihandler/flaghandler.go | 97 -------------- main.go | 2 +- opaprocessor/processorhandler.go | 8 +- opaprocessor/processorhandler_test.go | 14 +- opaprocessor/processorhandlerutils.go | 10 +- policyhandler/filesloader.go | 122 +++++++++++++----- policyhandler/filesloader_test.go | 9 +- policyhandler/handlenotification.go | 6 +- policyhandler/handlepullpolicies.go | 2 +- policyhandler/k8sresources.go | 8 +- policyhandler/k8sresourcesutils.go | 6 +- policyhandler/k8sresourcesutils_test.go | 4 +- policyhandler/urlloader.go | 73 +++++++++++ printer/junit.go | 2 +- printer/printresults.go | 6 +- printer/summeryhelpers.go | 4 +- 35 files changed, 266 insertions(+), 299 deletions(-) delete mode 100644 .gitmodules rename examples/{example.sh => example.md} (100%) delete mode 100644 examples/online-boutique/README.md delete mode 100644 examples/online-boutique/bi-monitor.yaml delete mode 100644 inputhandler/clihandler/clihandler_test.go delete mode 100644 inputhandler/clihandler/flaghandler.go create mode 100644 policyhandler/urlloader.go diff --git a/.gitignore b/.gitignore index 3c88c8d0..bd035f19 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.vs* *go.sum* -*kubescape* \ No newline at end of file +*kubescape* +*debug* \ No newline at end of file diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 75b512e8..00000000 --- a/.gitmodules +++ /dev/null @@ -1,4 +0,0 @@ -[submodule "vendor/github.com/armosec/capacketsgo"] - path = vendor/github.com/armosec/capacketsgo - url = git@github.com:armosec/capacketsgo.git - branch = master diff --git a/README.md b/README.md index ae3cccc6..c651f018 100644 --- a/README.md +++ b/README.md @@ -11,29 +11,28 @@ Use Kubescape to test clusters or scan single YAML files and integrate it to you # TL;DR -## Installation -To install the tool locally, run this: +## Install & Run +### Install: ``` curl -s https://raw.githubusercontent.com/armosec/kubescape/master/install.sh | /bin/bash ``` - - -## Run - -### Cluster testing -To get a fast check of the security posture of your Kubernetes cluster, run this: - +### Run: ``` kubescape scan framework nsa --exclude-namespaces kube-system,kube-public ``` If you wish to scan all namespaces in your cluster, remove the `--exclude-namespaces` flag. -### Pre-deployment testing -Check your YAML files before you're deploying, simply add them at the end of command line: + + + +## Usage & Examples + +### Pre-Deployment Testing +Check your YAML files before you're deploying, simply add them at the end of command line: ``` kubescape scan framework nsa *.yaml ``` @@ -44,16 +43,42 @@ Kubescape can produce output fitting for later processing: * JSON (`-o json`) * JUnit XML (`-o junit`) -Example: + +* Scan a running Kubernetes cluster with [`nsa`](https://www.nsa.gov/News-Features/Feature-Stories/Article-View/Article/2716980/nsa-cisa-release-kubernetes-hardening-guidance/) framework ``` -kubescape scan framework nsa --silent -o -junit > results.xml +kubescape scan framework nsa --exclude-namespaces kube-system,kube-public ``` - +* Scan a running Kubernetes cluster with [`mitre`](https://www.microsoft.com/security/blog/2020/04/02/attack-matrix-kubernetes/) framework +``` +kubescape scan framework mitre --exclude-namespaces kube-system,kube-public +``` + + +* Scan local `yaml`/`json` files +``` +kubescape scan framework nsa examples/online-boutique/* +``` + + +* Scan `yaml`/`json` files from url +``` +kubescape scan framework nsa https://raw.githubusercontent.com/GoogleCloudPlatform/microservices-demo/master/release/kubernetes-manifests.yaml +``` + +* Output in `json` format +``` +kubescape scan framework nsa --exclude-namespaces kube-system,kube-public --silence -o json > results.json +``` + +* Output in `junit xml` format +``` +kubescape scan framework nsa --exclude-namespaces kube-system,kube-public --silence -o junit > results.xml +``` # How to build -Note: development (and the release process) is done with Go 1.16 +Note: development (and the release process) is done with Go `1.16` 1. Clone Project ``` diff --git a/cautils/datastructures.go b/cautils/datastructures.go index d31e906f..53d7b2aa 100644 --- a/cautils/datastructures.go +++ b/cautils/datastructures.go @@ -1,7 +1,7 @@ package cautils import ( - "kube-escape/cautils/opapolicy" + "kubescape/cautils/opapolicy" ) // K8SResources map[//] diff --git a/cautils/k8sinterface/k8sconfig_test.go b/cautils/k8sinterface/k8sconfig_test.go index 88ce14f8..2b29a70f 100644 --- a/cautils/k8sinterface/k8sconfig_test.go +++ b/cautils/k8sinterface/k8sconfig_test.go @@ -3,7 +3,7 @@ package k8sinterface import ( "testing" - "kube-escape/cautils/cautils" + "kubescape/cautils/cautils" ) func TestGetGroupVersionResource(t *testing.T) { diff --git a/cautils/k8sinterface/k8sdynamic.go b/cautils/k8sinterface/k8sdynamic.go index 9d0091af..ae59f0f5 100644 --- a/cautils/k8sinterface/k8sdynamic.go +++ b/cautils/k8sinterface/k8sdynamic.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - "kube-escape/cautils/cautils" + "kubescape/cautils/cautils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/cautils/k8sinterface/k8sstatic.go b/cautils/k8sinterface/k8sstatic.go index 308e8bd3..f7cedc2c 100644 --- a/cautils/k8sinterface/k8sstatic.go +++ b/cautils/k8sinterface/k8sstatic.go @@ -3,7 +3,7 @@ package k8sinterface import ( "context" - "kube-escape/cautils/cautils" + "kubescape/cautils/cautils" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/cautils/k8sinterface/workload.go b/cautils/k8sinterface/workload.go index 0f3b0205..bb347012 100644 --- a/cautils/k8sinterface/workload.go +++ b/cautils/k8sinterface/workload.go @@ -3,7 +3,7 @@ package k8sinterface import ( "encoding/json" - "kube-escape/cautils/apis" + "kubescape/cautils/apis" corev1 "k8s.io/api/core/v1" diff --git a/cautils/k8sinterface/workloadmethods.go b/cautils/k8sinterface/workloadmethods.go index 0a63070c..8ae380ea 100644 --- a/cautils/k8sinterface/workloadmethods.go +++ b/cautils/k8sinterface/workloadmethods.go @@ -7,8 +7,8 @@ import ( "strings" "time" - "kube-escape/cautils/apis" - "kube-escape/cautils/cautils" + "kubescape/cautils/apis" + "kubescape/cautils/cautils" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/cautils/opapolicy/datastructures.go b/cautils/opapolicy/datastructures.go index ac73aef6..2e791556 100644 --- a/cautils/opapolicy/datastructures.go +++ b/cautils/opapolicy/datastructures.go @@ -3,7 +3,7 @@ package opapolicy import ( "time" - armotypes "kube-escape/cautils/armotypes" + armotypes "kubescape/cautils/armotypes" ) type AlertScore float32 diff --git a/cautils/opapolicy/datastructures_mock.go b/cautils/opapolicy/datastructures_mock.go index 2c1937c7..4e2a4036 100644 --- a/cautils/opapolicy/datastructures_mock.go +++ b/cautils/opapolicy/datastructures_mock.go @@ -3,7 +3,7 @@ package opapolicy import ( "time" - armotypes "kube-escape/cautils/armotypes" + armotypes "kubescape/cautils/armotypes" ) // Mock A diff --git a/cautils/opapolicy/resources/resourcesutils.go b/cautils/opapolicy/resources/resourcesutils.go index 2ce18c29..ea89aa19 100644 --- a/cautils/opapolicy/resources/resourcesutils.go +++ b/cautils/opapolicy/resources/resourcesutils.go @@ -8,7 +8,7 @@ import ( "path/filepath" "strings" - "kube-escape/cautils/k8sinterface" + "kubescape/cautils/k8sinterface" "github.com/golang/glog" "github.com/open-policy-agent/opa/storage" diff --git a/cmd/framework.go b/cmd/framework.go index eb48af53..d21a19c0 100644 --- a/cmd/framework.go +++ b/cmd/framework.go @@ -4,23 +4,20 @@ import ( "errors" "flag" "fmt" - "io/ioutil" - "kube-escape/cautils" - "kube-escape/cautils/armotypes" - "kube-escape/cautils/k8sinterface" - "kube-escape/cautils/opapolicy" - "kube-escape/opaprocessor" - "kube-escape/policyhandler" - "kube-escape/printer" + "kubescape/cautils" + "kubescape/cautils/armotypes" + "kubescape/cautils/k8sinterface" + "kubescape/cautils/opapolicy" + "kubescape/opaprocessor" + "kubescape/policyhandler" + "kubescape/printer" "os" - "strings" "github.com/spf13/cobra" ) var scanInfo opapolicy.ScanInfo var supportedFrameworks = []string{"nsa", "mitre"} -var isSilent bool type CLIHandler struct { policyHandler *policyhandler.PolicyHandler @@ -31,7 +28,7 @@ var frameworkCmd = &cobra.Command{ Use: "framework ", Short: "The framework you wish to use. Supported frameworks: nsa, mitre", Long: ``, - ValidArgs: []string{"nsa", "mitre"}, + ValidArgs: supportedFrameworks, Args: func(cmd *cobra.Command, args []string) error { if len(args) < 1 { return errors.New("requires at least one argument") @@ -59,23 +56,13 @@ func init() { scanCmd.AddCommand(frameworkCmd) scanInfo = opapolicy.ScanInfo{} frameworkCmd.Flags().StringVarP(&scanInfo.ExcludedNamespaces, "exclude-namespaces", "e", "", "namespaces to exclude from check") - frameworkCmd.Flags().StringVarP(&scanInfo.Output, "output", "o", "pretty-printer", "output format") - frameworkCmd.Flags().BoolVarP(&scanInfo.Silent, "silent", "s", false, "silent output") -} - -func processYamlInput(yamls string) { - listOfYamls := strings.Split(yamls, ",") - for _, yaml := range listOfYamls { - dat, err := ioutil.ReadFile(yaml) - if err != nil { - fmt.Printf("Could not open file: %s.", yaml) - } - fmt.Print(string(dat)) - } - + frameworkCmd.Flags().StringVarP(&scanInfo.Output, "output", "o", "pretty-printer", "output format. supported formats: 'pretty-printer'/'json'/'junit'") + frameworkCmd.Flags().BoolVarP(&scanInfo.Silent, "silent", "s", false, "silent progress output") } func CliSetup() error { + flag.Parse() + k8s := k8sinterface.NewKubernetesApi() processNotification := make(chan *cautils.OPASessionObj) @@ -117,7 +104,6 @@ func (clihandler *CLIHandler) Scan() error { }, Designators: armotypes.PortalDesignator{}, } - flag.Parse() switch policyNotification.NotificationType { case opapolicy.TypeExecPostureScan: go func() { diff --git a/cmd/scan.go b/cmd/scan.go index 98d6bdab..50af4324 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -7,7 +7,7 @@ import ( // scanCmd represents the scan command var scanCmd = &cobra.Command{ Use: "scan", - Short: "Scan command", + Short: "Scan the current running cluster or specified yaml files", Long: `The action you want to perform`, Run: func(cmd *cobra.Command, args []string) { }, diff --git a/examples/example.sh b/examples/example.md similarity index 100% rename from examples/example.sh rename to examples/example.md diff --git a/examples/online-boutique/README.md b/examples/online-boutique/README.md deleted file mode 100644 index ed852b72..00000000 --- a/examples/online-boutique/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# ./kubernetes-manifests - -:warning: Kubernetes manifests provided in this directory are not directly -deployable to a cluster. They are meant to be used with `skaffold` command to -insert the correct `image:` tags. - -Use the manifests in [/release](/release) directory which are configured with -pre-built public images. diff --git a/examples/online-boutique/bi-monitor.yaml b/examples/online-boutique/bi-monitor.yaml deleted file mode 100644 index d29fd043..00000000 --- a/examples/online-boutique/bi-monitor.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: v1 -data: - customer: Q3liZXJBcm1vclRlc3Rz - password: bml1ZGhmMjgzcnUyM3JrZQ== - username: ZHdlcnRlbnRAY3liZXJhcm1vci5pbw== -kind: Secret -metadata: - name: bi-monitor-secret -type: Opaque - ---- -apiVersion: v1 -kind: Service -metadata: - name: bi-monitor -spec: - type: ClusterIP - selector: - app: bi-monitor - ports: - - name: http - port: 80 - targetPort: 80 - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: bi-monitor - labels: - app: bi-monitor -spec: - replicas: 1 - selector: - matchLabels: - app: bi-monitor - template: - metadata: - labels: - app: bi-monitor - spec: - containers: - - name: monitor - image: quay.io/armosec/demoservice:v1-debian - env: - - name: THREAD_TIMEOUT - value: "10" - - name: SLEEP_DURATION - value: "1" - - name: DEMO_TARGETS - value: "http://frontend:80 https://cisco.com" - - name: CAA_SIGNATURE_DEBUG_DEEP - volumeMounts: - - name: bi-monitor-secret - mountPath: /etc/secrets - volumes: - - name: bi-monitor-secret - secret: - secretName: bi-monitor-secret diff --git a/go.mod b/go.mod index 0322258f..df427248 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module kube-escape +module kubescape go 1.16 diff --git a/inputhandler/clihandler/clihandler_test.go b/inputhandler/clihandler/clihandler_test.go deleted file mode 100644 index 4ee666a0..00000000 --- a/inputhandler/clihandler/clihandler_test.go +++ /dev/null @@ -1 +0,0 @@ -package clihandler diff --git a/inputhandler/clihandler/flaghandler.go b/inputhandler/clihandler/flaghandler.go deleted file mode 100644 index 9fa1d390..00000000 --- a/inputhandler/clihandler/flaghandler.go +++ /dev/null @@ -1,97 +0,0 @@ -package clihandler - -import ( - "flag" - "fmt" - "strings" - - "kube-escape/cautils/opapolicy" -) - -type FlagHandler struct { - policyIdentifier *opapolicy.PolicyIdentifier -} - -func NewFlagHandler() *FlagHandler { - flag.Parse() - return &FlagHandler{} -} - -func (flagHandler *FlagHandler) ExecuteScan() bool { - return flagHandler.policyIdentifier != nil -} - -// SetupHTTPListener set up listening http servers -func (flagHandler *FlagHandler) ParseFlag() { - f := "help" - if len(flag.Args()) >= 1 { - f = strings.ToLower(flag.Arg(0)) - } - switch f { - case "scan": - flagHandler.Scan() - case "version": - flagHandler.Version() - case "help": - flagHandler.Help() - default: - fmt.Println("unknown input argument") - flagHandler.Help() - } -} - -func (flagHandler *FlagHandler) Help() { - fmt.Println("Run: kubescape scan framework nsa --exclude-namespaces kube-system,kube-public") -} - -func (flagHandler *FlagHandler) Version() { - fmt.Println("betav1") -} - -func (flagHandler *FlagHandler) Scan() { - f := "help" - if len(flag.Args()) >= 2 { - f = strings.ToLower(flag.Arg(1)) - } - switch f { - case "framework": - flagHandler.ScanFramework() - case "control": - flagHandler.ScanControl() - case "help": - flagHandler.ScanHelp() - default: - fmt.Println("unknown input argument") - flagHandler.ScanHelp() - } -} -func (flagHandler *FlagHandler) ScanFramework() { - frameworkName := strings.ToUpper(flag.Arg(2)) - // if cautils.StringInSlice(SupportedFrameworks(), frameworkName) == cautils.ValueNotFound { - // fmt.Printf("framework %s not supported, supported frameworks: %v", frameworkName, SupportedFrameworks()) - // return - // } - flagHandler.policyIdentifier = &opapolicy.PolicyIdentifier{ - Kind: opapolicy.KindFramework, - Name: frameworkName, - } -} -func (flagHandler *FlagHandler) ScanControl() { - flagHandler.policyIdentifier = &opapolicy.PolicyIdentifier{ - Kind: opapolicy.KindControl, - Name: strings.ToUpper(flag.Arg(3)), - } -} -func (flagHandler *FlagHandler) ScanHelp() { - fmt.Println("") -} -func (flagHandler *FlagHandler) ScanFrameworkHelp() { - fmt.Println("Run framework nsa or mitre") -} -func (flagHandler *FlagHandler) ScanControlHelp() { - fmt.Println("not supported") -} - -func SupportedFrameworks() []string { - return []string{"nsa", "mitre"} // TODO - get from BE -} diff --git a/main.go b/main.go index 4f4ce7ac..47f8524d 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,6 @@ package main -import "kube-escape/cmd" +import "kubescape/cmd" func main() { cmd.Execute() diff --git a/opaprocessor/processorhandler.go b/opaprocessor/processorhandler.go index a49118a8..52ac6f7c 100644 --- a/opaprocessor/processorhandler.go +++ b/opaprocessor/processorhandler.go @@ -3,13 +3,13 @@ package opaprocessor import ( "context" "fmt" - "kube-escape/cautils" + "kubescape/cautils" "time" - "kube-escape/cautils/k8sinterface" + "kubescape/cautils/k8sinterface" - "kube-escape/cautils/opapolicy" - "kube-escape/cautils/opapolicy/resources" + "kubescape/cautils/opapolicy" + "kubescape/cautils/opapolicy/resources" "github.com/golang/glog" "github.com/open-policy-agent/opa/ast" diff --git a/opaprocessor/processorhandler_test.go b/opaprocessor/processorhandler_test.go index 264f12f5..eae72844 100644 --- a/opaprocessor/processorhandler_test.go +++ b/opaprocessor/processorhandler_test.go @@ -3,18 +3,18 @@ package opaprocessor import ( "context" "encoding/json" - "kube-escape/cautils" + "kubescape/cautils" "os" "path" "strings" "testing" - "kube-escape/cautils/k8sinterface" + "kubescape/cautils/k8sinterface" // _ "k8s.io/client-go/plugin/pkg/client/auth" restclient "k8s.io/client-go/rest" - "kube-escape/cautils/opapolicy" - "kube-escape/cautils/opapolicy/resources" + "kubescape/cautils/opapolicy" + "kubescape/cautils/opapolicy/resources" "github.com/open-policy-agent/opa/ast" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -82,10 +82,10 @@ func TestCompromisedRegistries(t *testing.T) { // k8sResources["/v1/pods"] = k8sinterface.ConvertUnstructuredSliceToMap(k8sinterface.V1KubeSystemNamespaceMock().Items) k8sResources["/v1/pods"] = k8sinterface.V1AllClusterWithCompromisedRegistriesMock().Items wd, _ := os.Getwd() - baseDirName := "kube-escape" + baseDirName := "kubescape" idx := strings.Index(wd, baseDirName) wd = wd[0:idx] - resources.RegoDependenciesPath = path.Join(wd, "/kube-escape/vendor/asterix.cyberarmor.io/cyberarmor/capacketsgo/opapolicy/resources/rego/dependencies") + resources.RegoDependenciesPath = path.Join(wd, "/kubescape/vendor/asterix.cyberarmor.io/cyberarmor/capacketsgo/opapolicy/resources/rego/dependencies") k8sinterface.K8SConfig = &restclient.Config{} opaProcessor := NewOPAProcessorMock() @@ -108,7 +108,7 @@ func TestCompromisedRegistries(t *testing.T) { // k8sResources := make(cautils.K8SResources) // // k8sResources["/v1/pods"] = k8sinterface.ConvertUnstructuredSliceToMap(k8sinterface.V1KubeSystemNamespaceMock().Items) // k8sResources["/v1/pods"] = k8sinterface.V1KubeSystemNamespaceMock().Items -// resources.RegoDependenciesPath = "/home/david/go/src/kube-escape/vendor/asterix.cyberarmor.io/cyberarmor/capacketsgo/opapolicy/resources/rego/dependencies" +// resources.RegoDependenciesPath = "/home/david/go/src/kubescape/vendor/asterix.cyberarmor.io/cyberarmor/capacketsgo/opapolicy/resources/rego/dependencies" // opaProcessor := NewOPAProcessorMock() // // set opaSessionObj diff --git a/opaprocessor/processorhandlerutils.go b/opaprocessor/processorhandlerutils.go index 4982159f..8189f7ad 100644 --- a/opaprocessor/processorhandlerutils.go +++ b/opaprocessor/processorhandlerutils.go @@ -1,12 +1,12 @@ package opaprocessor import ( - "kube-escape/cautils" + "kubescape/cautils" - pkgcautils "kube-escape/cautils/cautils" - "kube-escape/cautils/k8sinterface" - "kube-escape/cautils/opapolicy" - resources "kube-escape/cautils/opapolicy/resources" + pkgcautils "kubescape/cautils/cautils" + "kubescape/cautils/k8sinterface" + "kubescape/cautils/opapolicy" + resources "kubescape/cautils/opapolicy/resources" "github.com/golang/glog" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/policyhandler/filesloader.go b/policyhandler/filesloader.go index cdd223b6..e65a608e 100644 --- a/policyhandler/filesloader.go +++ b/policyhandler/filesloader.go @@ -5,10 +5,11 @@ import ( "encoding/json" "fmt" "io/ioutil" - "kube-escape/cautils" - "kube-escape/cautils/k8sinterface" - "kube-escape/cautils/opapolicy" + "kubescape/cautils" + "kubescape/cautils/k8sinterface" + "kubescape/cautils/opapolicy" "path/filepath" + "strings" "gopkg.in/yaml.v2" ) @@ -18,27 +19,39 @@ var ( JSON_PREFIX = []string{".json"} ) +type FileFormat string + +const ( + YAML_FILE_FORMAT FileFormat = "yaml" + JSON_FILE_FORMAT FileFormat = "json" +) + func (policyHandler *PolicyHandler) loadResources(frameworks []opapolicy.Framework, scanInfo *opapolicy.ScanInfo) (*cautils.K8SResources, error) { + workloads := []k8sinterface.IWorkload{} - files, errs := listFiles(scanInfo.InputPatterns) - if len(errs) > 0 { - cautils.ErrorDisplay(fmt.Sprintf("%v", errs)) // TODO - print error + // load resource from local file system + w, err := loadResourcesFromFiles(scanInfo.InputPatterns) + if err != nil { + return nil, err } - if len(files) == 0 { - return nil, fmt.Errorf("empty list of files - no files found") + if w != nil { + workloads = append(workloads, w...) } - workloads, errs := loadFiles(files) - if len(errs) > 0 { - cautils.ErrorDisplay(fmt.Sprintf("%v", errs)) // TODO - print error + // load resource from url + w, err = loadResourcesFromUrl(scanInfo.InputPatterns) + if err != nil { + return nil, err } - if len(workloads) == 0 { - return nil, fmt.Errorf("empty list of workloads - no workloads valid workloads found") + if w != nil { + workloads = append(workloads, w...) } + // map all resources: map["/group/version/kind"][] allResources := mapResources(workloads) // build resources map + // map resources based on framework requrid resources: map["/group/version/kind"][] k8sResources := setResourceMap(frameworks) // save only relevant resources @@ -52,6 +65,26 @@ func (policyHandler *PolicyHandler) loadResources(frameworks []opapolicy.Framewo } +func loadResourcesFromFiles(inputPatterns []string) ([]k8sinterface.IWorkload, error) { + files, errs := listFiles(inputPatterns) + if len(errs) > 0 { + cautils.ErrorDisplay(fmt.Sprintf("%v", errs)) // TODO - print error + } + if len(files) == 0 { + return nil, nil + } + + workloads, errs := loadFiles(files) + if len(errs) > 0 { + cautils.ErrorDisplay(fmt.Sprintf("%v", errs)) // TODO - print error + } + if len(workloads) == 0 { + return workloads, fmt.Errorf("empty list of workloads - no workloads found") + } + return workloads, nil +} + +// build resources map func mapResources(workloads []k8sinterface.IWorkload) map[string][]map[string]interface{} { allResources := map[string][]map[string]interface{}{} for i := range workloads { @@ -76,30 +109,46 @@ func mapResources(workloads []k8sinterface.IWorkload) map[string][]map[string]in } -// // build resources map func loadFiles(filePaths []string) ([]k8sinterface.IWorkload, []error) { workloads := []k8sinterface.IWorkload{} errs := []error{} for i := range filePaths { - w, e := loadFile(filePaths[i]) + f, err := loadFile(filePaths[i]) + if err != nil { + errs = append(errs, err) + continue + } + w, e := readFile(f, getFileFormat(filePaths[i])) errs = append(errs, e...) - workloads = append(workloads, w...) + if w != nil { + workloads = append(workloads, w...) + } } return workloads, errs } -func loadFile(filePath string) ([]k8sinterface.IWorkload, []error) { - if isYaml(filePath) { - return loadYamlFile(filePath) - } else if isJson(filePath) { - return loadJsonFile(filePath) +func loadFile(filePath string) ([]byte, error) { + return ioutil.ReadFile(filePath) +} +func readFile(fileContent []byte, fileFromat FileFormat) ([]k8sinterface.IWorkload, []error) { + + switch fileFromat { + case YAML_FILE_FORMAT: + return readYamlFile(fileContent) + case JSON_FILE_FORMAT: + return readJsonFile(fileContent) + default: + return nil, []error{fmt.Errorf("file extension %s not supported", fileFromat)} } - return nil, []error{fmt.Errorf("file extension %s not supported, file name: %s", filepath.Ext(filePath), filePath)} + } func listFiles(patterns []string) ([]string, []error) { files := []string{} errs := []error{} for i := range patterns { + if strings.HasPrefix(patterns[i], "http") { + continue + } f, err := filepath.Glob(patterns[i]) if err != nil { errs = append(errs, err) @@ -110,12 +159,8 @@ func listFiles(patterns []string) ([]string, []error) { return files, errs } -func loadYamlFile(filePath string) ([]k8sinterface.IWorkload, []error) { +func readYamlFile(yamlFile []byte) ([]k8sinterface.IWorkload, []error) { errs := []error{} - yamlFile, err := ioutil.ReadFile(filePath) - if err != nil { - return nil, []error{err} - } r := bytes.NewReader(yamlFile) dec := yaml.NewDecoder(r) @@ -124,24 +169,23 @@ func loadYamlFile(filePath string) ([]k8sinterface.IWorkload, []error) { var t interface{} for dec.Decode(&t) == nil { j := convertYamlToJson(t) + if j == nil { + continue + } if obj, ok := j.(map[string]interface{}); ok { yamlObjs = append(yamlObjs, k8sinterface.NewWorkloadObj(obj)) } else { - errs = append(errs, fmt.Errorf("failed to convert yaml file %s file to map[string]interface", filePath)) + errs = append(errs, fmt.Errorf("failed to convert yaml file to map[string]interface, file content: %v", j)) } } return yamlObjs, errs } -func loadJsonFile(filePath string) ([]k8sinterface.IWorkload, []error) { +func readJsonFile(jsonFile []byte) ([]k8sinterface.IWorkload, []error) { workloads := []k8sinterface.IWorkload{} - jsonFile, err := ioutil.ReadFile(filePath) - if err != nil { - return workloads, []error{err} - } var jsonObj interface{} - if err = json.Unmarshal(jsonFile, &jsonObj); err != nil { + if err := json.Unmarshal(jsonFile, &jsonObj); err != nil { return workloads, []error{err} } @@ -183,3 +227,13 @@ func isYaml(filePath string) bool { func isJson(filePath string) bool { return cautils.StringInSlice(YAML_PREFIX, filepath.Ext(filePath)) != cautils.ValueNotFound } + +func getFileFormat(filePath string) FileFormat { + if isYaml(filePath) { + return YAML_FILE_FORMAT + } else if isJson(filePath) { + return JSON_FILE_FORMAT + } else { + return FileFormat(filePath) + } +} diff --git a/policyhandler/filesloader_test.go b/policyhandler/filesloader_test.go index 903badf7..c41b6993 100644 --- a/policyhandler/filesloader_test.go +++ b/policyhandler/filesloader_test.go @@ -2,7 +2,7 @@ package policyhandler import ( "fmt" - "kube-escape/cautils" + "kubescape/cautils" "os" "path/filepath" "strings" @@ -44,13 +44,10 @@ func TestLoadFiles(t *testing.T) { func TestLoadFile(t *testing.T) { files, _ := listFiles([]string{strings.Replace(onlineBoutiquePath(), "*", "bi-monitor.yaml", 1)}) - bb, err := loadFile(files[0]) - if len(err) > 0 { + _, err := loadFile(files[0]) + if err != nil { t.Errorf("%v", err) } - for i := range bb { - t.Errorf("%s", bb[i].ToString()) - } } func TestLoadResources(t *testing.T) { diff --git a/policyhandler/handlenotification.go b/policyhandler/handlenotification.go index 05fecad1..754cbdbf 100644 --- a/policyhandler/handlenotification.go +++ b/policyhandler/handlenotification.go @@ -2,11 +2,11 @@ package policyhandler import ( "fmt" - "kube-escape/cautils" + "kubescape/cautils" - "kube-escape/cautils/k8sinterface" + "kubescape/cautils/k8sinterface" - "kube-escape/cautils/opapolicy" + "kubescape/cautils/opapolicy" ) // PolicyHandler - diff --git a/policyhandler/handlepullpolicies.go b/policyhandler/handlepullpolicies.go index 926b0a51..822e4d30 100644 --- a/policyhandler/handlepullpolicies.go +++ b/policyhandler/handlepullpolicies.go @@ -8,7 +8,7 @@ import ( "net/url" "strings" - "kube-escape/cautils/opapolicy" + "kubescape/cautils/opapolicy" ) // URLEncoder encode url diff --git a/policyhandler/k8sresources.go b/policyhandler/k8sresources.go index c87ca5b7..cd94a654 100644 --- a/policyhandler/k8sresources.go +++ b/policyhandler/k8sresources.go @@ -2,13 +2,13 @@ package policyhandler import ( "fmt" - "kube-escape/cautils" + "kubescape/cautils" "strings" - "kube-escape/cautils/k8sinterface" + "kubescape/cautils/k8sinterface" - "kube-escape/cautils/armotypes" - "kube-escape/cautils/opapolicy" + "kubescape/cautils/armotypes" + "kubescape/cautils/opapolicy" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" diff --git a/policyhandler/k8sresourcesutils.go b/policyhandler/k8sresourcesutils.go index 4da5b2a5..8e6cc23f 100644 --- a/policyhandler/k8sresourcesutils.go +++ b/policyhandler/k8sresourcesutils.go @@ -1,10 +1,10 @@ package policyhandler import ( - "kube-escape/cautils" + "kubescape/cautils" - "kube-escape/cautils/k8sinterface" - "kube-escape/cautils/opapolicy" + "kubescape/cautils/k8sinterface" + "kubescape/cautils/opapolicy" ) func setResourceMap(frameworks []opapolicy.Framework) *cautils.K8SResources { diff --git a/policyhandler/k8sresourcesutils_test.go b/policyhandler/k8sresourcesutils_test.go index 73414884..7518aa74 100644 --- a/policyhandler/k8sresourcesutils_test.go +++ b/policyhandler/k8sresourcesutils_test.go @@ -1,8 +1,8 @@ package policyhandler import ( - "kube-escape/cautils/k8sinterface" - "kube-escape/cautils/opapolicy" + "kubescape/cautils/k8sinterface" + "kubescape/cautils/opapolicy" "testing" ) diff --git a/policyhandler/urlloader.go b/policyhandler/urlloader.go new file mode 100644 index 00000000..d1de20db --- /dev/null +++ b/policyhandler/urlloader.go @@ -0,0 +1,73 @@ +package policyhandler + +import ( + "bytes" + "fmt" + "io" + "kubescape/cautils" + "kubescape/cautils/k8sinterface" + "net/http" + "strings" +) + +func loadResourcesFromUrl(inputPatterns []string) ([]k8sinterface.IWorkload, error) { + urls := listUrls(inputPatterns) + if len(urls) == 0 { + return nil, nil + } + + workloads, errs := downloadFiles(urls) + if len(errs) > 0 { + cautils.ErrorDisplay(fmt.Sprintf("%v", errs)) // TODO - print error + } + if len(workloads) == 0 { + return workloads, fmt.Errorf("empty list of workloads - no workloads valid workloads found") + } + return workloads, nil +} + +func listUrls(patterns []string) []string { + urls := []string{} + for i := range patterns { + if strings.HasPrefix(patterns[i], "http") { + urls = append(urls, patterns[i]) + } + } + return urls +} + +func downloadFiles(urls []string) ([]k8sinterface.IWorkload, []error) { + workloads := []k8sinterface.IWorkload{} + errs := []error{} + for i := range urls { + f, err := downloadFile(urls[i]) + if err != nil { + errs = append(errs, err) + continue + } + w, e := readFile(f, getFileFormat(urls[i])) + errs = append(errs, e...) + if w != nil { + workloads = append(workloads, w...) + } + } + return workloads, errs +} + +func downloadFile(url string) ([]byte, error) { + resp, err := http.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || 301 < resp.StatusCode { + return nil, fmt.Errorf("failed to download file, url: '%s', status code: %s", url, resp.Status) + } + return streamToByte(resp.Body), nil +} + +func streamToByte(stream io.Reader) []byte { + buf := new(bytes.Buffer) + buf.ReadFrom(stream) + return buf.Bytes() +} diff --git a/printer/junit.go b/printer/junit.go index dd51e43d..7d2b2866 100644 --- a/printer/junit.go +++ b/printer/junit.go @@ -3,7 +3,7 @@ package printer import ( "encoding/xml" "fmt" - "kube-escape/cautils/opapolicy" + "kubescape/cautils/opapolicy" ) type JUnitTestSuites struct { diff --git a/printer/printresults.go b/printer/printresults.go index 6f273024..bf712138 100644 --- a/printer/printresults.go +++ b/printer/printresults.go @@ -4,12 +4,12 @@ import ( "encoding/json" "encoding/xml" "fmt" - "kube-escape/cautils" + "kubescape/cautils" "os" "sort" - "kube-escape/cautils/k8sinterface" - "kube-escape/cautils/opapolicy" + "kubescape/cautils/k8sinterface" + "kubescape/cautils/opapolicy" "github.com/enescakir/emoji" "github.com/olekukonko/tablewriter" diff --git a/printer/summeryhelpers.go b/printer/summeryhelpers.go index 7a02bd3a..b61ddb67 100644 --- a/printer/summeryhelpers.go +++ b/printer/summeryhelpers.go @@ -3,8 +3,8 @@ package printer import ( "fmt" - "kube-escape/cautils/k8sinterface" - "kube-escape/cautils/opapolicy" + "kubescape/cautils/k8sinterface" + "kubescape/cautils/opapolicy" ) // Group workloads by namespace - return {"namespace": <[]WorkloadSummary>}