From 8e950e0f5448f2eddb1a2e6ab1ea0f463d575250 Mon Sep 17 00:00:00 2001 From: Calvin Figuereo-Supraner Date: Sat, 9 Oct 2021 21:25:01 -0700 Subject: [PATCH 1/3] Add missing return --- cautils/apis/backendconnectormethods.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cautils/apis/backendconnectormethods.go b/cautils/apis/backendconnectormethods.go index 26c195c6..bc05456e 100644 --- a/cautils/apis/backendconnectormethods.go +++ b/cautils/apis/backendconnectormethods.go @@ -21,7 +21,7 @@ func MakeBackendConnector(client *http.Client, baseURL string, loginDetails *Cus func ValidateBEConnectorMakerInput(client *http.Client, baseURL string, loginDetails *CustomerLoginDetails) error { if client == nil { - fmt.Errorf("You must provide an initialized httpclient") + return fmt.Errorf("You must provide an initialized httpclient") } if len(baseURL) == 0 { return fmt.Errorf("you must provide a valid backend url") From 9c65aadcc70223d616e7249c6de726043c8fa7e3 Mon Sep 17 00:00:00 2001 From: Rotem Refael <48656234+rotemamsa@users.noreply.github.com> Date: Mon, 18 Oct 2021 10:26:59 +0300 Subject: [PATCH 2/3] Delete backendconnectormethods.go --- cautils/apis/backendconnectormethods.go | 128 ------------------------ 1 file changed, 128 deletions(-) delete mode 100644 cautils/apis/backendconnectormethods.go diff --git a/cautils/apis/backendconnectormethods.go b/cautils/apis/backendconnectormethods.go deleted file mode 100644 index bc05456e..00000000 --- a/cautils/apis/backendconnectormethods.go +++ /dev/null @@ -1,128 +0,0 @@ -package apis - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "strings" -) - -func MakeBackendConnector(client *http.Client, baseURL string, loginDetails *CustomerLoginDetails) (*BackendConnector, error) { - if err := ValidateBEConnectorMakerInput(client, baseURL, loginDetails); err != nil { - return nil, err - } - conn := &BackendConnector{BaseURL: baseURL, Credentials: loginDetails, HTTPClient: client} - err := conn.Login() - - return conn, err -} - -func ValidateBEConnectorMakerInput(client *http.Client, baseURL string, loginDetails *CustomerLoginDetails) error { - if client == nil { - return fmt.Errorf("You must provide an initialized httpclient") - } - if len(baseURL) == 0 { - return fmt.Errorf("you must provide a valid backend url") - } - - if loginDetails == nil || (len(loginDetails.Email) == 0 && len(loginDetails.Password) == 0) { - return fmt.Errorf("you must provide valid login details") - } - return nil - -} - -func (r *BackendConnector) Login() error { - if !r.IsExpired() { - return nil - } - - loginInfoBytes, err := json.Marshal(r.Credentials) - if err != nil { - return fmt.Errorf("unable to marshal credentials properly") - } - - beURL := fmt.Sprintf("%v/%v", r.BaseURL, "login") - - req, err := http.NewRequest("POST", beURL, bytes.NewReader(loginInfoBytes)) - if err != nil { - return err - } - - req.Header.Set("Referer", strings.Replace(beURL, "dashbe", "cpanel", 1)) - resp, err := r.HTTPClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("unable to read login response") - } - - loginS := &BELoginResponse{} - json.Unmarshal(body, &loginS) - - loginS.Cookies = resp.Cookies() - r.BELoginResponse = loginS - - return nil -} - -func (r *BackendConnector) IsExpired() bool { - return r.BELoginResponse == nil || r.BELoginResponse.ToLoginObject().IsExpired() -} - -func (r *BackendConnector) GetBaseURL() string { - return r.BaseURL -} -func (r *BackendConnector) GetLoginObj() *LoginObject { - return r.BELoginResponse.ToLoginObject() -} -func (r *BackendConnector) GetClient() *http.Client { - return r.HTTPClient -} - -func (r *BackendConnector) HTTPSend(httpverb string, - endpoint string, - payload []byte, - f HTTPReqFunc, - qryData interface{}) ([]byte, error) { - - beURL := fmt.Sprintf("%v/%v", r.GetBaseURL(), endpoint) - req, err := http.NewRequest(httpverb, beURL, bytes.NewReader(payload)) - if err != nil { - return nil, err - } - - if r.IsExpired() { - r.Login() - } - - loginobj := r.GetLoginObj() - req.Header.Set("Authorization", loginobj.Authorization) - f(req, qryData) - q := req.URL.Query() - q.Set("customerGUID", loginobj.GUID) - req.URL.RawQuery = q.Encode() - - for _, cookie := range loginobj.Cookies { - req.AddCookie(cookie) - } - resp, err := r.GetClient().Do(req) - if err != nil { - return nil, err - } - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - fmt.Printf("req:\n%v\nresp:%v\n", req, resp) - return nil, fmt.Errorf("Error #%v Due to: %v", resp.StatusCode, resp.Status) - } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - return body, nil -} From 63367f4f31fae302f41d3fc328feb97592f66eeb Mon Sep 17 00:00:00 2001 From: moshep Date: Mon, 18 Oct 2021 14:29:44 +0300 Subject: [PATCH 3/3] support yamls from repo --- policyhandler/repositoryscanner.go | 155 +++++++++++++++++++++++++++++ policyhandler/urlloader.go | 11 +- 2 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 policyhandler/repositoryscanner.go diff --git a/policyhandler/repositoryscanner.go b/policyhandler/repositoryscanner.go new file mode 100644 index 00000000..a568e8c5 --- /dev/null +++ b/policyhandler/repositoryscanner.go @@ -0,0 +1,155 @@ +package policyhandler + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/armosec/kubescape/cautils/getter" +) + +type IRepository interface { + setBranch(string) error + setTree() error + getYamlFromTree() []string +} + +type innerTree struct { + Path string `json:"path"` +} +type tree struct { + InnerTrees []innerTree `json:"tree"` +} + +type GitHubRepository struct { + host string + name string // / + branch string + tree tree +} +type githubDefaultBranchAPI struct { + DefaultBranch string `json:"default_branch"` +} + +func NewGitHubRepository(rep string) *GitHubRepository { + return &GitHubRepository{ + host: "github", + name: rep, + } +} + +func ScanRepository(command string, branchOptional string) ([]string, error) { + repo, err := getRepository(command) + if err != nil { + return nil, err + } + + err = repo.setBranch(branchOptional) + if err != nil { + return nil, err + } + + err = repo.setTree() + if err != nil { + return nil, err + } + + // get all paths that are of the yaml type, and build them into a valid url + return repo.getYamlFromTree(), nil +} + +func getHostAndRepoName(url string) (string, string, error) { + splitUrl := strings.Split(url, "/") + + if len(splitUrl) != 5 { + return "", "", fmt.Errorf("failed to pars url: %s", url) + } + + hostUrl := splitUrl[2] // github.com, gitlab.com, etc. + repository := splitUrl[3] + "/" + strings.Split(splitUrl[4], ".")[0] // user/reposetory + + return hostUrl, repository, nil +} + +func getRepository(url string) (IRepository, error) { + hostUrl, repoName, err := getHostAndRepoName(url) + if err != nil { + return nil, err + } + + var repo IRepository + switch repoHost := strings.Split(hostUrl, ".")[0]; repoHost { + case "github": + repo = NewGitHubRepository(repoName) + default: + return nil, fmt.Errorf("unknown repository host: %s", repoHost) + } + + // Returns the host-url, and the part of the user and repository from the url + return repo, nil +} + +func (g *GitHubRepository) setBranch(branchOptional string) error { + // Checks whether the repository type is a master or another type. + // By default it is "master", unless the branchOptional came with a value + if branchOptional == "" { + + body, err := getter.HttpGetter(&http.Client{}, g.defaultBranchAPI()) + if err != nil { + return err + } + + var data githubDefaultBranchAPI + err = json.Unmarshal([]byte(body), &data) + if err != nil { + return err + } + g.branch = data.DefaultBranch + } else { + g.branch = branchOptional + } + return nil +} + +func (g *GitHubRepository) defaultBranchAPI() string { + return fmt.Sprintf("https://api.github.com/repos/%s", g.name) +} + +func (g *GitHubRepository) setTree() error { + body, err := getter.HttpGetter(&http.Client{}, g.treeAPI()) + if err != nil { + return err + } + + // press all tree to json + var tree tree + err = json.Unmarshal([]byte(body), &tree) + if err != nil { + return fmt.Errorf("failed to unmarshal response body from '%s', reason: %s", g.treeAPI(), err.Error()) + // fmt.Printf("failed to unmarshal response body from '%s', reason: %s", urlCommand, err.Error()) + // return nil + } + g.tree = tree + + return nil +} + +func (g *GitHubRepository) treeAPI() string { + return fmt.Sprintf("https://api.github.com/repos/%s/git/trees/%s?recursive=1", g.name, g.branch) +} + +// return a list of yaml for a given repository tree +func (g *GitHubRepository) getYamlFromTree() []string { + var urls []string + for _, path := range g.tree.InnerTrees { + if strings.HasSuffix(path.Path, ".yaml") { + urls = append(urls, fmt.Sprintf("%s/%s", g.rowYamlUrl(), path.Path)) + } + } + return urls +} + +func (g *GitHubRepository) rowYamlUrl() string { + return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s", g.name, g.branch) +} diff --git a/policyhandler/urlloader.go b/policyhandler/urlloader.go index 151aeaa5..faa8f634 100644 --- a/policyhandler/urlloader.go +++ b/policyhandler/urlloader.go @@ -28,9 +28,18 @@ func listUrls(patterns []string) []string { urls := []string{} for i := range patterns { if strings.HasPrefix(patterns[i], "http") { - urls = append(urls, patterns[i]) + if !isYaml(patterns[i]) || !isJson(patterns[i]) { // if url of repo + if yamls, err := ScanRepository(patterns[i], ""); err == nil { // TODO - support branch + urls = append(urls, yamls...) + } else { + fmt.Print(err) // TODO - handle errors + } + } else { // url of single file + urls = append(urls, patterns[i]) + } } } + return urls }