diff --git a/.github/workflows/00-pr-scanner.yaml b/.github/workflows/00-pr-scanner.yaml index 6d627239..076db592 100644 --- a/.github/workflows/00-pr-scanner.yaml +++ b/.github/workflows/00-pr-scanner.yaml @@ -35,7 +35,7 @@ jobs: CGO_ENABLED: 1 GO111MODULE: "" GO_VERSION: "1.20" - RELEASE: "" + RELEASE: "latest" CLIENT: test ARCH_MATRIX: '[ "" ]' OS_MATRIX: '[ "ubuntu-20.04" ]' diff --git a/core/cautils/git_native_disabled.go b/core/cautils/git_native_disabled.go index c8c58403..5727b914 100644 --- a/core/cautils/git_native_disabled.go +++ b/core/cautils/git_native_disabled.go @@ -3,20 +3,87 @@ package cautils import ( - "errors" + "fmt" + "path/filepath" + "time" "github.com/kubescape/go-git-url/apis" + "github.com/kubescape/go-logger" + "github.com/kubescape/go-logger/helpers" + "github.com/matthyx/go-gitlog" ) -var ErrFatalNotSupportedByBuild = errors.New(`git scan not supported by this build. Build with tag "gitenabled" to enable the git scan feature`) - type gitRepository struct { + gitLogDisabled bool + gitRepo gitlog.GitLog + fileToLastCommit map[string]*gitlog.Commit } func newGitRepository(root string) (*gitRepository, error) { - return &gitRepository{}, ErrWarnNotSupportedByBuild + gitRepo := gitlog.New(&gitlog.Config{Path: root}) + + return &gitRepository{ + gitRepo: gitRepo, + }, nil } func (g *gitRepository) GetFileLastCommit(filePath string) (*apis.Commit, error) { - return nil, ErrFatalNotSupportedByBuild + if len(g.fileToLastCommit) == 0 && !g.gitLogDisabled { + g.buildCommitMap() + } + + if relevantCommit, exists := g.fileToLastCommit[filepath.ToSlash(filePath)]; exists { + return g.getCommit(relevantCommit), nil + } + + return nil, fmt.Errorf("failed to get commit information for file: %s", filePath) +} + +func (g *gitRepository) buildCommitMap() { + filePathToCommitTime := map[string]time.Time{} + filePathToCommit := map[string]*gitlog.Commit{} + allCommits, err := g.gitRepo.Log(nil, &gitlog.Params{IgnoreMerges: true}) + if err != nil { + logger.L().Warning("git not found in PATH: git metadata (author and hash) will not be available", helpers.Error(err)) + g.gitLogDisabled = true + return + } + + // builds a map of all files to their last commit + for _, commit := range allCommits { + for _, file := range commit.Files { + commitTime := commit.Author.Date + + // In case we have the commit information for the file which is not the latest - we override it + if currentCommitTime, exists := filePathToCommitTime[file]; exists { + if currentCommitTime.Before(commitTime) { + filePathToCommitTime[file] = commitTime + filePathToCommit[file] = commit + } + } else { + filePathToCommitTime[file] = commitTime + filePathToCommit[file] = commit + } + } + } + + g.fileToLastCommit = filePathToCommit +} + +func (g *gitRepository) getCommit(commit *gitlog.Commit) *apis.Commit { + return &apis.Commit{ + SHA: commit.Hash.Long, + Author: apis.Committer{ + Name: commit.Author.Name, + Email: commit.Author.Email, + Date: commit.Author.Date, + }, + Message: commit.Subject + "\n" + commit.Body, + Committer: apis.Committer{ + Name: commit.Committer.Name, + Email: commit.Committer.Email, + Date: commit.Committer.Date, + }, + Files: []apis.Files{}, + } } diff --git a/core/cautils/git_native_disabled_test.go b/core/cautils/git_native_disabled_test.go deleted file mode 100644 index ccc23346..00000000 --- a/core/cautils/git_native_disabled_test.go +++ /dev/null @@ -1,11 +0,0 @@ -//go:build !gitenabled - -package cautils - -func (s *LocalGitRepositoryTestSuite) TestGetLastCommit() { - s.T().Log("warn: skipped testing native git functionality [GetLastCommit]") -} - -func (s *LocalGitRepositoryTestSuite) TestGetFileLastCommit() { - s.T().Log("warn: skipped testing native git functionality [GetFileLastCommit]") -} diff --git a/core/cautils/git_native_enabled.go b/core/cautils/git_native_enabled.go index 09cdaf71..10523359 100644 --- a/core/cautils/git_native_enabled.go +++ b/core/cautils/git_native_enabled.go @@ -1,4 +1,5 @@ //go:build gitenabled + package cautils import ( @@ -27,64 +28,7 @@ func newGitRepository(root string) (*gitRepository, error) { func (g *gitRepository) GetFileLastCommit(filePath string) (*apis.Commit, error) { if len(g.fileToLastCommit) == 0 { - filePathToCommitTime := map[string]time.Time{} - filePathToCommit := map[string]*git2go.Commit{} - allCommits, _ := g.getAllCommits() - - // builds a map of all files to their last commit - for _, commit := range allCommits { - // Ignore merge commits (2+ parents) - if commit.ParentCount() <= 1 { - tree, err := commit.Tree() - if err != nil { - continue - } - - // ParentCount can be either 1 or 0 (initial commit) - // In case it's the initial commit, prevTree is nil - var prevTree *git2go.Tree - if commit.ParentCount() == 1 { - prevCommit := commit.Parent(0) - prevTree, err = prevCommit.Tree() - if err != nil { - continue - } - } - - diff, err := g.git2GoRepo.DiffTreeToTree(prevTree, tree, nil) - if err != nil { - continue - } - - numDeltas, err := diff.NumDeltas() - if err != nil { - continue - } - - for i := 0; i < numDeltas; i++ { - delta, err := diff.Delta(i) - if err != nil { - continue - } - - deltaFilePath := delta.NewFile.Path - commitTime := commit.Author().When - - // In case we have the commit information for the file which is not the latest - we override it - if currentCommitTime, exists := filePathToCommitTime[deltaFilePath]; exists { - if currentCommitTime.Before(commitTime) { - filePathToCommitTime[deltaFilePath] = commitTime - filePathToCommit[deltaFilePath] = commit - } - } else { - filePathToCommitTime[deltaFilePath] = commitTime - filePathToCommit[deltaFilePath] = commit - } - } - } - } - - g.fileToLastCommit = filePathToCommit + g.buildCommitMap() } if relevantCommit, exists := g.fileToLastCommit[filePath]; exists { @@ -94,6 +38,67 @@ func (g *gitRepository) GetFileLastCommit(filePath string) (*apis.Commit, error) return nil, fmt.Errorf("failed to get commit information for file: %s", filePath) } +func (g *gitRepository) buildCommitMap() { + filePathToCommitTime := map[string]time.Time{} + filePathToCommit := map[string]*git2go.Commit{} + allCommits, _ := g.getAllCommits() + + // builds a map of all files to their last commit + for _, commit := range allCommits { + // Ignore merge commits (2+ parents) + if commit.ParentCount() <= 1 { + tree, err := commit.Tree() + if err != nil { + continue + } + + // ParentCount can be either 1 or 0 (initial commit) + // In case it's the initial commit, prevTree is nil + var prevTree *git2go.Tree + if commit.ParentCount() == 1 { + prevCommit := commit.Parent(0) + prevTree, err = prevCommit.Tree() + if err != nil { + continue + } + } + + diff, err := g.git2GoRepo.DiffTreeToTree(prevTree, tree, nil) + if err != nil { + continue + } + + numDeltas, err := diff.NumDeltas() + if err != nil { + continue + } + + for i := 0; i < numDeltas; i++ { + delta, err := diff.Delta(i) + if err != nil { + continue + } + + deltaFilePath := delta.NewFile.Path + commitTime := commit.Author().When + + // In case we have the commit information for the file which is not the latest - we override it + if currentCommitTime, exists := filePathToCommitTime[deltaFilePath]; exists { + if currentCommitTime.Before(commitTime) { + filePathToCommitTime[deltaFilePath] = commitTime + filePathToCommit[deltaFilePath] = commit + } + } else { + filePathToCommitTime[deltaFilePath] = commitTime + filePathToCommit[deltaFilePath] = commit + } + } + } + } + + g.fileToLastCommit = filePathToCommit +} + func (g *gitRepository) getAllCommits() ([]*git2go.Commit, error) { logItr, itrErr := g.git2GoRepo.Walk() if itrErr != nil { diff --git a/core/cautils/git_native_enabled_test.go b/core/cautils/git_native_enabled_test.go index 09ae5fbb..ffaa572d 100644 --- a/core/cautils/git_native_enabled_test.go +++ b/core/cautils/git_native_enabled_test.go @@ -1,13 +1,18 @@ -//go:build gitenabled package cautils +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + func (s *LocalGitRepositoryTestSuite) TestGetLastCommit() { if localRepo, err := NewLocalGitRepository(s.gitRepositoryPaths["localrepo"]); s.NoError(err) { if commit, err := localRepo.GetLastCommit(); s.NoError(err) { s.Equal("7e09312b8017695fadcd606882e3779f10a5c832", commit.SHA) s.Equal("Amir Malka", commit.Author.Name) s.Equal("amirm@armosec.io", commit.Author.Email) - s.Equal("2022-05-22 19:11:57 +0300 +0300", commit.Author.Date.String()) + s.Equal(int64(1653235917), commit.Author.Date.Unix()) s.Equal("added file B\n", commit.Message) } } @@ -21,7 +26,7 @@ func (s *LocalGitRepositoryTestSuite) TestGetFileLastCommit() { s.Equal("9fae4be19624297947d2b605cefbff516628612d", commit.SHA) s.Equal("Amir Malka", commit.Author.Name) s.Equal("amirm@armosec.io", commit.Author.Email) - s.Equal("2022-05-22 18:55:48 +0300 +0300", commit.Author.Date.String()) + s.Equal(int64(1653234948), commit.Author.Date.Unix()) s.Equal("added file A\n", commit.Message) } @@ -35,10 +40,19 @@ func (s *LocalGitRepositoryTestSuite) TestGetFileLastCommit() { s.Equal("7e09312b8017695fadcd606882e3779f10a5c832", commit.SHA) s.Equal("Amir Malka", commit.Author.Name) s.Equal("amirm@armosec.io", commit.Author.Email) - s.Equal("2022-05-22 19:11:57 +0300 +0300", commit.Author.Date.String()) + s.Equal(int64(1653235917), commit.Author.Date.Unix()) s.Equal("added file B\n", commit.Message) } } }) } + +func BenchmarkBuildCommitMap(b *testing.B) { + localRepo, err := NewLocalGitRepository("testdata/temp/localrepo") + assert.NoError(b, err) + for i := 0; i < b.N; i++ { + localRepo.buildCommitMap() + } + b.ReportAllocs() +} diff --git a/core/pkg/resourcehandler/filesloader.go b/core/pkg/resourcehandler/filesloader.go index ecc237f5..a27e2ac7 100644 --- a/core/pkg/resourcehandler/filesloader.go +++ b/core/pkg/resourcehandler/filesloader.go @@ -219,7 +219,7 @@ func getResourcesFromPath(ctx context.Context, path string) (map[string]reportha if gitRepo != nil { commitInfo, err := gitRepo.GetFileLastCommit(source) if err != nil && !warnIssued { - logger.L().Ctx(ctx).Warning("Git scan skipped", helpers.Error(err)) + logger.L().Debug("Git scan skipped", helpers.Error(err)) warnIssued = true // croak only once } diff --git a/go.mod b/go.mod index c34f76fe..96554b04 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/kubescape/rbac-utils v0.0.21-0.20230806101615-07e36f555520 github.com/kubescape/regolibrary v1.0.291-rc.0 github.com/libgit2/git2go/v33 v33.0.9 + github.com/matthyx/go-gitlog v0.0.0-20231005131906-9ffabe3c5bcd github.com/mattn/go-isatty v0.0.19 github.com/mikefarah/yq/v4 v4.29.1 github.com/olekukonko/tablewriter v0.0.6-0.20230417144759-edd1a71a5576 diff --git a/go.sum b/go.sum index d413560d..69c503e9 100644 --- a/go.sum +++ b/go.sum @@ -1330,6 +1330,8 @@ github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsI github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= +github.com/matthyx/go-gitlog v0.0.0-20231005131906-9ffabe3c5bcd h1:Fs7gIqboEWWiJ2OdaWxK0MHGL+8XskZRZdVjJhUnSqE= +github.com/matthyx/go-gitlog v0.0.0-20231005131906-9ffabe3c5bcd/go.mod h1:kKOGX+aQer8+30ROQinaA9vO810BGhSVHz88IpbPt60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= diff --git a/httphandler/go.mod b/httphandler/go.mod index 8bfe84f6..bfb04d4d 100644 --- a/httphandler/go.mod +++ b/httphandler/go.mod @@ -258,6 +258,7 @@ require ( github.com/libgit2/git2go/v33 v33.0.9 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/matthyx/go-gitlog v0.0.0-20231005131906-9ffabe3c5bcd // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect diff --git a/httphandler/go.sum b/httphandler/go.sum index e243fc2c..99f20a14 100644 --- a/httphandler/go.sum +++ b/httphandler/go.sum @@ -1336,6 +1336,8 @@ github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsI github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= +github.com/matthyx/go-gitlog v0.0.0-20231005131906-9ffabe3c5bcd h1:Fs7gIqboEWWiJ2OdaWxK0MHGL+8XskZRZdVjJhUnSqE= +github.com/matthyx/go-gitlog v0.0.0-20231005131906-9ffabe3c5bcd/go.mod h1:kKOGX+aQer8+30ROQinaA9vO810BGhSVHz88IpbPt60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=