From 5bf179810b2c530366883e5707b0b4bcbc406a8d Mon Sep 17 00:00:00 2001 From: Amir Malka Date: Sun, 17 Jul 2022 10:08:09 +0300 Subject: [PATCH] use git2go to get file commit information --- core/cautils/localgitrepository.go | 163 ++++++++++++++++++++---- core/cautils/localgitrepository_test.go | 6 + core/pkg/resourcehandler/filesloader.go | 69 +++++++--- go.mod | 1 + go.sum | 6 + 5 files changed, 203 insertions(+), 42 deletions(-) diff --git a/core/cautils/localgitrepository.go b/core/cautils/localgitrepository.go index 1f6463d9..245c60f0 100644 --- a/core/cautils/localgitrepository.go +++ b/core/cautils/localgitrepository.go @@ -4,26 +4,30 @@ import ( "fmt" "path" "strings" + "time" "github.com/armosec/go-git-url/apis" gitv5 "github.com/go-git/go-git/v5" configv5 "github.com/go-git/go-git/v5/config" plumbingv5 "github.com/go-git/go-git/v5/plumbing" + git2go "github.com/libgit2/git2go/v33" ) type LocalGitRepository struct { - repo *gitv5.Repository - head *plumbingv5.Reference - config *configv5.Config + goGitRepo *gitv5.Repository + git2GoRepo *git2go.Repository + head *plumbingv5.Reference + config *configv5.Config + fileToLastCommit map[string]*git2go.Commit } func NewLocalGitRepository(path string) (*LocalGitRepository, error) { - gitRepo, err := gitv5.PlainOpenWithOptions(path, &gitv5.PlainOpenOptions{DetectDotGit: true}) + goGitRepo, err := gitv5.PlainOpenWithOptions(path, &gitv5.PlainOpenOptions{DetectDotGit: true}) if err != nil { return nil, err } - head, err := gitRepo.Head() + head, err := goGitRepo.Head() if err != nil { return nil, err } @@ -32,15 +36,25 @@ func NewLocalGitRepository(path string) (*LocalGitRepository, error) { return nil, fmt.Errorf("current HEAD reference is not a branch") } - config, err := gitRepo.Config() + config, err := goGitRepo.Config() + if err != nil { + return nil, err + } + + if len(config.Remotes) == 0 { + return nil, fmt.Errorf("no remotes found") + } + + git2GoRepo, err := git2go.OpenRepository(path) if err != nil { return nil, err } return &LocalGitRepository{ - repo: gitRepo, - head: head, - config: config, + goGitRepo: goGitRepo, + head: head, + config: config, + git2GoRepo: git2GoRepo, }, nil } @@ -80,20 +94,7 @@ func (g *LocalGitRepository) GetName() (string, error) { // GetLastCommit get latest commit object func (g *LocalGitRepository) GetLastCommit() (*apis.Commit, error) { - return g.GetFileLastCommit("") -} - -// GetFileLastCommit get file latest commit object, if empty will return latest commit -func (g *LocalGitRepository) GetFileLastCommit(filePath string) (*apis.Commit, error) { - // By default, returns commit information from current HEAD - logOptions := &gitv5.LogOptions{} - - if filePath != "" { - logOptions.FileName = &filePath - logOptions.Order = gitv5.LogOrderCommitterTime // faster -> LogOrderDFSPost - } - - cIter, err := g.repo.Log(logOptions) + cIter, err := g.goGitRepo.Log(&gitv5.LogOptions{}) if err != nil { return nil, err } @@ -117,8 +118,122 @@ func (g *LocalGitRepository) GetFileLastCommit(filePath string) (*apis.Commit, e }, nil } +func (g *LocalGitRepository) getAllCommits() ([]*git2go.Commit, error) { + logItr, itrErr := g.git2GoRepo.Walk() + if itrErr != nil { + + return nil, itrErr + } + + pushErr := logItr.PushHead() + if pushErr != nil { + return nil, pushErr + } + + var allCommits []*git2go.Commit + err := logItr.Iterate(func(commit *git2go.Commit) bool { + if commit != nil { + allCommits = append(allCommits, commit) + return true + } + return false + }) + + if err != nil { + return nil, err + } + + if err != nil { + return nil, err + } + + return allCommits, nil +} + +func (g *LocalGitRepository) GetFileLastCommit(filePath string) (*apis.Commit, error) { + if g.fileToLastCommit == nil { + filePathToCommitTime := map[string]time.Time{} + filePathToCommit := map[string]*git2go.Commit{} + allCommits, _ := g.getAllCommits() + + // builds a map of all files to their last commit + for _, commit := range allCommits { + // Ignore merge commits (2+ parents) + if commit.ParentCount() <= 1 { + tree, err := commit.Tree() + if err != nil { + continue + } + + // ParentCount can be either 1 or 0 (initial commit) + // In case it's the initial commit, prevTree is nil + var prevTree *git2go.Tree + if commit.ParentCount() == 1 { + prevCommit := commit.Parent(0) + prevTree, err = prevCommit.Tree() + if err != nil { + continue + } + } + + diff, err := g.git2GoRepo.DiffTreeToTree(prevTree, tree, nil) + if err != nil { + continue + } + + numDeltas, err := diff.NumDeltas() + if err != nil { + continue + } + + for i := 0; i < numDeltas; i++ { + delta, err := diff.Delta(i) + if err != nil { + continue + } + + deltaFilePath := delta.NewFile.Path + commitTime := commit.Author().When + + // In case we have the commit information for the file which is not the latest - we override it + if currentCommitTime, exists := filePathToCommitTime[deltaFilePath]; exists { + if currentCommitTime.Before(commitTime) { + filePathToCommitTime[deltaFilePath] = commitTime + filePathToCommit[deltaFilePath] = commit + } + } else { + filePathToCommitTime[deltaFilePath] = commitTime + filePathToCommit[deltaFilePath] = commit + } + } + } + } + g.fileToLastCommit = filePathToCommit + } + + if relevantCommit, exists := g.fileToLastCommit[filePath]; exists { + return g.getCommit(relevantCommit), nil + } + + return nil, fmt.Errorf("failed to get commit information for file: %s", filePath) +} + +func (g *LocalGitRepository) getCommit(commit *git2go.Commit) *apis.Commit { + return &apis.Commit{ + SHA: commit.Id().String(), + Author: apis.Committer{ + Name: commit.Author().Name, + Email: commit.Author().Email, + Date: commit.Author().When, + }, + Message: commit.Message(), + Committer: apis.Committer{}, + Files: []apis.Files{}, + } +} + func (g *LocalGitRepository) GetRootDir() (string, error) { - wt, err := g.repo.Worktree() + wt, err := g.goGitRepo.Worktree() if err != nil { return "", fmt.Errorf("failed to get repo root") } diff --git a/core/cautils/localgitrepository_test.go b/core/cautils/localgitrepository_test.go index 1ce827d2..a822fc08 100644 --- a/core/cautils/localgitrepository_test.go +++ b/core/cautils/localgitrepository_test.go @@ -92,6 +92,12 @@ func (s *LocalGitRepositoryTestSuite) TestInvalidRepositoryPath() { } } +func (s *LocalGitRepositoryTestSuite) TestRepositoryWithoutRemotes() { + if _, err := NewLocalGitRepository("/Users/amirmalka/dev/gitwithoutremote"); s.Error(err) { + s.Equal("no remotes found", err.Error()) + } +} + func (s *LocalGitRepositoryTestSuite) TestGetBranchName() { if localRepo, err := NewLocalGitRepository(s.gitRepositoryPath); s.NoError(err) { s.Equal("master", localRepo.GetBranchName()) diff --git a/core/pkg/resourcehandler/filesloader.go b/core/pkg/resourcehandler/filesloader.go index 8af20ffe..0008476c 100644 --- a/core/pkg/resourcehandler/filesloader.go +++ b/core/pkg/resourcehandler/filesloader.go @@ -57,9 +57,9 @@ func (fileHandler *FileResourceHandler) GetResources(sessionObj *cautils.OPASess // Get repo root repoRoot := "" - giRepo, err := cautils.NewLocalGitRepository(path) + gitRepo, err := cautils.NewLocalGitRepository(path) if err == nil { - repoRoot, _ = giRepo.GetRootDir() + repoRoot, _ = gitRepo.GetRootDir() } // load resource from local file system @@ -75,19 +75,36 @@ func (fileHandler *FileResourceHandler) GetResources(sessionObj *cautils.OPASess if err == nil { source = relSource } + + var filetype string + if cautils.IsYaml(source) { + filetype = reporthandling.SourceTypeYaml + } else if cautils.IsJson(source) { + filetype = reporthandling.SourceTypeJson + } else { + continue + } + + var lastCommit reporthandling.LastCommit + commitInfo, _ := gitRepo.GetFileLastCommit(source) + if commitInfo != nil { + lastCommit = reporthandling.LastCommit{ + Hash: commitInfo.SHA, + Date: commitInfo.Author.Date, + CommitterName: commitInfo.Author.Name, + CommitterEmail: commitInfo.Author.Email, + Message: commitInfo.Message, + } + } + + workloadSource := reporthandling.Source{ + RelativePath: source, + FileType: filetype, + LastCommit: lastCommit, + } + for i := range ws { - var filetype string - if cautils.IsYaml(source) { - filetype = reporthandling.SourceTypeYaml - } else if cautils.IsJson(source) { - filetype = reporthandling.SourceTypeJson - } else { - continue - } - workloadIDToSource[ws[i].GetID()] = reporthandling.Source{ - RelativePath: source, - FileType: filetype, - } + workloadIDToSource[ws[i].GetID()] = workloadSource } } @@ -104,12 +121,28 @@ func (fileHandler *FileResourceHandler) GetResources(sessionObj *cautils.OPASess if err == nil { source = relSource } - for i := range ws { - workloadIDToSource[ws[i].GetID()] = reporthandling.Source{ - RelativePath: source, - FileType: reporthandling.SourceTypeHelmChart, + + var lastCommit reporthandling.LastCommit + commitInfo, _ := gitRepo.GetFileLastCommit(source) + if commitInfo != nil { + lastCommit = reporthandling.LastCommit{ + Hash: commitInfo.SHA, + Date: commitInfo.Author.Date, + CommitterName: commitInfo.Author.Name, + CommitterEmail: commitInfo.Author.Email, + Message: commitInfo.Message, } } + + workloadSource := reporthandling.Source{ + RelativePath: source, + FileType: reporthandling.SourceTypeHelmChart, + LastCommit: lastCommit, + } + + for i := range ws { + workloadIDToSource[ws[i].GetID()] = workloadSource + } } if len(helmSourceToWorkloads) > 0 { diff --git a/go.mod b/go.mod index 1c9d9655..7b629b3c 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/go-git/go-git/v5 v5.4.2 github.com/google/uuid v1.3.0 github.com/johnfercher/maroto v0.37.0 + github.com/libgit2/git2go/v33 v33.0.9 github.com/mattn/go-isatty v0.0.14 github.com/olekukonko/tablewriter v0.0.5 github.com/open-policy-agent/opa v0.41.0 diff --git a/go.sum b/go.sum index 7574832f..1ebeec2c 100644 --- a/go.sum +++ b/go.sum @@ -717,6 +717,8 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -862,6 +864,8 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/libgit2/git2go/v33 v33.0.9 h1:4ch2DJed6IhJO28BEohkUoGvxLsRzUjxljoNFJ6/O78= +github.com/libgit2/git2go/v33 v33.0.9/go.mod h1:KdpqkU+6+++4oHna/MIOgx4GCQ92IPCdpVRMRI80J+4= github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -1356,6 +1360,7 @@ golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= @@ -1608,6 +1613,7 @@ golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=