From 8e950e0f5448f2eddb1a2e6ab1ea0f463d575250 Mon Sep 17 00:00:00 2001 From: Calvin Figuereo-Supraner Date: Sat, 9 Oct 2021 21:25:01 -0700 Subject: [PATCH 1/6] Add missing return --- cautils/apis/backendconnectormethods.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cautils/apis/backendconnectormethods.go b/cautils/apis/backendconnectormethods.go index 26c195c6..bc05456e 100644 --- a/cautils/apis/backendconnectormethods.go +++ b/cautils/apis/backendconnectormethods.go @@ -21,7 +21,7 @@ func MakeBackendConnector(client *http.Client, baseURL string, loginDetails *Cus func ValidateBEConnectorMakerInput(client *http.Client, baseURL string, loginDetails *CustomerLoginDetails) error { if client == nil { - fmt.Errorf("You must provide an initialized httpclient") + return fmt.Errorf("You must provide an initialized httpclient") } if len(baseURL) == 0 { return fmt.Errorf("you must provide a valid backend url") From 9c65aadcc70223d616e7249c6de726043c8fa7e3 Mon Sep 17 00:00:00 2001 From: Rotem Refael <48656234+rotemamsa@users.noreply.github.com> Date: Mon, 18 Oct 2021 10:26:59 +0300 Subject: [PATCH 2/6] Delete backendconnectormethods.go --- cautils/apis/backendconnectormethods.go | 128 ------------------------ 1 file changed, 128 deletions(-) delete mode 100644 cautils/apis/backendconnectormethods.go diff --git a/cautils/apis/backendconnectormethods.go b/cautils/apis/backendconnectormethods.go deleted file mode 100644 index bc05456e..00000000 --- a/cautils/apis/backendconnectormethods.go +++ /dev/null @@ -1,128 +0,0 @@ -package apis - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "strings" -) - -func MakeBackendConnector(client *http.Client, baseURL string, loginDetails *CustomerLoginDetails) (*BackendConnector, error) { - if err := ValidateBEConnectorMakerInput(client, baseURL, loginDetails); err != nil { - return nil, err - } - conn := &BackendConnector{BaseURL: baseURL, Credentials: loginDetails, HTTPClient: client} - err := conn.Login() - - return conn, err -} - -func ValidateBEConnectorMakerInput(client *http.Client, baseURL string, loginDetails *CustomerLoginDetails) error { - if client == nil { - return fmt.Errorf("You must provide an initialized httpclient") - } - if len(baseURL) == 0 { - return fmt.Errorf("you must provide a valid backend url") - } - - if loginDetails == nil || (len(loginDetails.Email) == 0 && len(loginDetails.Password) == 0) { - return fmt.Errorf("you must provide valid login details") - } - return nil - -} - -func (r *BackendConnector) Login() error { - if !r.IsExpired() { - return nil - } - - loginInfoBytes, err := json.Marshal(r.Credentials) - if err != nil { - return fmt.Errorf("unable to marshal credentials properly") - } - - beURL := fmt.Sprintf("%v/%v", r.BaseURL, "login") - - req, err := http.NewRequest("POST", beURL, bytes.NewReader(loginInfoBytes)) - if err != nil { - return err - } - - req.Header.Set("Referer", strings.Replace(beURL, "dashbe", "cpanel", 1)) - resp, err := r.HTTPClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("unable to read login response") - } - - loginS := &BELoginResponse{} - json.Unmarshal(body, &loginS) - - loginS.Cookies = resp.Cookies() - r.BELoginResponse = loginS - - return nil -} - -func (r *BackendConnector) IsExpired() bool { - return r.BELoginResponse == nil || r.BELoginResponse.ToLoginObject().IsExpired() -} - -func (r *BackendConnector) GetBaseURL() string { - return r.BaseURL -} -func (r *BackendConnector) GetLoginObj() *LoginObject { - return r.BELoginResponse.ToLoginObject() -} -func (r *BackendConnector) GetClient() *http.Client { - return r.HTTPClient -} - -func (r *BackendConnector) HTTPSend(httpverb string, - endpoint string, - payload []byte, - f HTTPReqFunc, - qryData interface{}) ([]byte, error) { - - beURL := fmt.Sprintf("%v/%v", r.GetBaseURL(), endpoint) - req, err := http.NewRequest(httpverb, beURL, bytes.NewReader(payload)) - if err != nil { - return nil, err - } - - if r.IsExpired() { - r.Login() - } - - loginobj := r.GetLoginObj() - req.Header.Set("Authorization", loginobj.Authorization) - f(req, qryData) - q := req.URL.Query() - q.Set("customerGUID", loginobj.GUID) - req.URL.RawQuery = q.Encode() - - for _, cookie := range loginobj.Cookies { - req.AddCookie(cookie) - } - resp, err := r.GetClient().Do(req) - if err != nil { - return nil, err - } - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - fmt.Printf("req:\n%v\nresp:%v\n", req, resp) - return nil, fmt.Errorf("Error #%v Due to: %v", resp.StatusCode, resp.Status) - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - return body, nil -} From d948e2068202ebc12cdb44d1daab0e5917945bec Mon Sep 17 00:00:00 2001 From: Rotem Refael Date: Mon, 18 Oct 2021 14:12:57 +0300 Subject: [PATCH 3/6] Change readme text --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2b82689a..63d26e8b 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,11 @@ [![build](https://github.com/armosec/kubescape/actions/workflows/build.yaml/badge.svg)](https://github.com/armosec/kubescape/actions/workflows/build.yaml) [![Go Report Card](https://goreportcard.com/badge/github.com/armosec/kubescape)](https://goreportcard.com/report/github.com/armosec/kubescape) -Kubescape is the first tool for testing if Kubernetes is deployed securely as defined in [Kubernetes Hardening Guidance by NSA and CISA](https://www.nsa.gov/Press-Room/News-Highlights/Article/Article/2716980/nsa-cisa-release-kubernetes-hardening-guidance/) +Kubescape is the first open-source tool for testing if Kubernetes is deployed securely according to multiple frameworks: +regulatory, customized company policies and DevSecOps best practices, such as the [NSA-CISA](https://www.armosec.io/blog/kubernetes-hardening-guidance-summary-by-armo) and the [MITRE ATT&CK®](https://www.microsoft.com/security/blog/2021/03/23/secure-containerized-environments-with-updated-threat-matrix-for-kubernetes/) . +Kubescape scans K8s clusters, YAML files, and HELM charts, and detect misconfigurations and software vulnerabilities at early stages of the CI/CD pipeline and provides a risk score instantly and risk trends over time. +Kubescape integrates natively with other DevOps tools, including Jenkins, CircleCI and Github workflows. + Use Kubescape to test clusters or scan single YAML files and integrate it to your processes. From 6ed8287b01be05dff03431d5bb48618df1ef8337 Mon Sep 17 00:00:00 2001 From: Rotem Refael <48656234+rotemamsa@users.noreply.github.com> Date: Mon, 18 Oct 2021 14:15:45 +0300 Subject: [PATCH 4/6] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 63d26e8b..0d611779 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,6 @@ Kubescape scans K8s clusters, YAML files, and HELM charts, and detect misconfigu Kubescape integrates natively with other DevOps tools, including Jenkins, CircleCI and Github workflows. -Use Kubescape to test clusters or scan single YAML files and integrate it to your processes. From 6f9d6b4af3ba40de2585125ff41fe319d14e27cc Mon Sep 17 00:00:00 2001 From: Rotem Refael <48656234+rotemamsa@users.noreply.github.com> Date: Mon, 18 Oct 2021 14:16:29 +0300 Subject: [PATCH 5/6] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0d611779..6b0da00c 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ regulatory, customized company policies and DevSecOps best practices, such as th Kubescape scans K8s clusters, YAML files, and HELM charts, and detect misconfigurations and software vulnerabilities at early stages of the CI/CD pipeline and provides a risk score instantly and risk trends over time. Kubescape integrates natively with other DevOps tools, including Jenkins, CircleCI and Github workflows. - +
From 63367f4f31fae302f41d3fc328feb97592f66eeb Mon Sep 17 00:00:00 2001 From: moshep Date: Mon, 18 Oct 2021 14:29:44 +0300 Subject: [PATCH 6/6] support yamls from repo --- policyhandler/repositoryscanner.go | 155 +++++++++++++++++++++++++++++ policyhandler/urlloader.go | 11 +- 2 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 policyhandler/repositoryscanner.go diff --git a/policyhandler/repositoryscanner.go b/policyhandler/repositoryscanner.go new file mode 100644 index 00000000..a568e8c5 --- /dev/null +++ b/policyhandler/repositoryscanner.go @@ -0,0 +1,155 @@ +package policyhandler + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/armosec/kubescape/cautils/getter" +) + +type IRepository interface { + setBranch(string) error + setTree() error + getYamlFromTree() []string +} + +type innerTree struct { + Path string `json:"path"` +} +type tree struct { + InnerTrees []innerTree `json:"tree"` +} + +type GitHubRepository struct { + host string + name string // / + branch string + tree tree +} +type githubDefaultBranchAPI struct { + DefaultBranch string `json:"default_branch"` +} + +func NewGitHubRepository(rep string) *GitHubRepository { + return &GitHubRepository{ + host: "github", + name: rep, + } +} + +func ScanRepository(command string, branchOptional string) ([]string, error) { + repo, err := getRepository(command) + if err != nil { + return nil, err + } + + err = repo.setBranch(branchOptional) + if err != nil { + return nil, err + } + + err = repo.setTree() + if err != nil { + return nil, err + } + + // get all paths that are of the yaml type, and build them into a valid url + return repo.getYamlFromTree(), nil +} + +func getHostAndRepoName(url string) (string, string, error) { + splitUrl := strings.Split(url, "/") + + if len(splitUrl) != 5 { + return "", "", fmt.Errorf("failed to pars url: %s", url) + } + + hostUrl := splitUrl[2] // github.com, gitlab.com, etc. + repository := splitUrl[3] + "/" + strings.Split(splitUrl[4], ".")[0] // user/reposetory + + return hostUrl, repository, nil +} + +func getRepository(url string) (IRepository, error) { + hostUrl, repoName, err := getHostAndRepoName(url) + if err != nil { + return nil, err + } + + var repo IRepository + switch repoHost := strings.Split(hostUrl, ".")[0]; repoHost { + case "github": + repo = NewGitHubRepository(repoName) + default: + return nil, fmt.Errorf("unknown repository host: %s", repoHost) + } + + // Returns the host-url, and the part of the user and repository from the url + return repo, nil +} + +func (g *GitHubRepository) setBranch(branchOptional string) error { + // Checks whether the repository type is a master or another type. + // By default it is "master", unless the branchOptional came with a value + if branchOptional == "" { + + body, err := getter.HttpGetter(&http.Client{}, g.defaultBranchAPI()) + if err != nil { + return err + } + + var data githubDefaultBranchAPI + err = json.Unmarshal([]byte(body), &data) + if err != nil { + return err + } + g.branch = data.DefaultBranch + } else { + g.branch = branchOptional + } + return nil +} + +func (g *GitHubRepository) defaultBranchAPI() string { + return fmt.Sprintf("https://api.github.com/repos/%s", g.name) +} + +func (g *GitHubRepository) setTree() error { + body, err := getter.HttpGetter(&http.Client{}, g.treeAPI()) + if err != nil { + return err + } + + // press all tree to json + var tree tree + err = json.Unmarshal([]byte(body), &tree) + if err != nil { + return fmt.Errorf("failed to unmarshal response body from '%s', reason: %s", g.treeAPI(), err.Error()) + // fmt.Printf("failed to unmarshal response body from '%s', reason: %s", urlCommand, err.Error()) + // return nil + } + g.tree = tree + + return nil +} + +func (g *GitHubRepository) treeAPI() string { + return fmt.Sprintf("https://api.github.com/repos/%s/git/trees/%s?recursive=1", g.name, g.branch) +} + +// return a list of yaml for a given repository tree +func (g *GitHubRepository) getYamlFromTree() []string { + var urls []string + for _, path := range g.tree.InnerTrees { + if strings.HasSuffix(path.Path, ".yaml") { + urls = append(urls, fmt.Sprintf("%s/%s", g.rowYamlUrl(), path.Path)) + } + } + return urls +} + +func (g *GitHubRepository) rowYamlUrl() string { + return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s", g.name, g.branch) +} diff --git a/policyhandler/urlloader.go b/policyhandler/urlloader.go index 151aeaa5..faa8f634 100644 --- a/policyhandler/urlloader.go +++ b/policyhandler/urlloader.go @@ -28,9 +28,18 @@ func listUrls(patterns []string) []string { urls := []string{} for i := range patterns { if strings.HasPrefix(patterns[i], "http") { - urls = append(urls, patterns[i]) + if !isYaml(patterns[i]) || !isJson(patterns[i]) { // if url of repo + if yamls, err := ScanRepository(patterns[i], ""); err == nil { // TODO - support branch + urls = append(urls, yamls...) + } else { + fmt.Print(err) // TODO - handle errors + } + } else { // url of single file + urls = append(urls, patterns[i]) + } } } + return urls }