use go-gitlog as an alternative to git2go (#1393)

* use go-gitlog as an alternative to git2go

Signed-off-by: Matthias Bertschy <matthias.bertschy@gmail.com>

* set RELEASE to something to avoid failing binary-build step

Signed-off-by: Matthias Bertschy <matthias.bertschy@gmail.com>

---------

Signed-off-by: Matthias Bertschy <matthias.bertschy@gmail.com>

.github/workflows/00-pr-scanner.yaml

@@ -35,7 +35,7 @@ jobs:
       CGO_ENABLED: 1
       GO111MODULE: ""
       GO_VERSION: "1.20"
-      RELEASE: ""
+      RELEASE: "latest"
       CLIENT: test
       ARCH_MATRIX: '[ "" ]'
       OS_MATRIX: '[ "ubuntu-20.04" ]'

core/cautils/git_native_disabled.go

@@ -3,20 +3,87 @@
 package cautils
 
 import (
-	"errors"
+	"fmt"
+	"path/filepath"
+	"time"
 
 	"github.com/kubescape/go-git-url/apis"
+	"github.com/kubescape/go-logger"
+	"github.com/kubescape/go-logger/helpers"
+	"github.com/matthyx/go-gitlog"
 )
 
-var ErrFatalNotSupportedByBuild = errors.New(`git scan not supported by this build. Build with tag "gitenabled" to enable the git scan feature`)
-
 type gitRepository struct {
+	gitLogDisabled   bool
+	gitRepo          gitlog.GitLog
+	fileToLastCommit map[string]*gitlog.Commit
 }
 
 func newGitRepository(root string) (*gitRepository, error) {
-	return &gitRepository{}, ErrWarnNotSupportedByBuild
+	gitRepo := gitlog.New(&gitlog.Config{Path: root})
+
+	return &gitRepository{
+		gitRepo: gitRepo,
+	}, nil
 }
 
 func (g *gitRepository) GetFileLastCommit(filePath string) (*apis.Commit, error) {
-	return nil, ErrFatalNotSupportedByBuild
+	if len(g.fileToLastCommit) == 0 && !g.gitLogDisabled {
+		g.buildCommitMap()
+	}
+
+	if relevantCommit, exists := g.fileToLastCommit[filepath.ToSlash(filePath)]; exists {
+		return g.getCommit(relevantCommit), nil
+	}
+
+	return nil, fmt.Errorf("failed to get commit information for file: %s", filePath)
+}
+
+func (g *gitRepository) buildCommitMap() {
+	filePathToCommitTime := map[string]time.Time{}
+	filePathToCommit := map[string]*gitlog.Commit{}
+	allCommits, err := g.gitRepo.Log(nil, &gitlog.Params{IgnoreMerges: true})
+	if err != nil {
+		logger.L().Warning("git not found in PATH: git metadata (author and hash) will not be available", helpers.Error(err))
+		g.gitLogDisabled = true
+		return
+	}
+
+	// builds a map of all files to their last commit
+	for _, commit := range allCommits {
+		for _, file := range commit.Files {
+			commitTime := commit.Author.Date
+
+			// In case we have the commit information for the file which is not the latest - we override it
+			if currentCommitTime, exists := filePathToCommitTime[file]; exists {
+				if currentCommitTime.Before(commitTime) {
+					filePathToCommitTime[file] = commitTime
+					filePathToCommit[file] = commit
+				}
+			} else {
+				filePathToCommitTime[file] = commitTime
+				filePathToCommit[file] = commit
+			}
+		}
+	}
+
+	g.fileToLastCommit = filePathToCommit
+}
+
+func (g *gitRepository) getCommit(commit *gitlog.Commit) *apis.Commit {
+	return &apis.Commit{
+		SHA: commit.Hash.Long,
+		Author: apis.Committer{
+			Name:  commit.Author.Name,
+			Email: commit.Author.Email,
+			Date:  commit.Author.Date,
+		},
+		Message: commit.Subject + "\n" + commit.Body,
+		Committer: apis.Committer{
+			Name:  commit.Committer.Name,
+			Email: commit.Committer.Email,
+			Date:  commit.Committer.Date,
+		},
+		Files: []apis.Files{},
+	}
 }

core/cautils/git_native_disabled_test.go

@@ -1,11 +0,0 @@
-//go:build !gitenabled
-
-package cautils
-
-func (s *LocalGitRepositoryTestSuite) TestGetLastCommit() {
-	s.T().Log("warn: skipped testing native git functionality [GetLastCommit]")
-}
-
-func (s *LocalGitRepositoryTestSuite) TestGetFileLastCommit() {
-	s.T().Log("warn: skipped testing native git functionality [GetFileLastCommit]")
-}

core/cautils/git_native_enabled.go

@@ -1,4 +1,5 @@
 //go:build gitenabled
+
 package cautils
 
 import (
@@ -27,64 +28,7 @@ func newGitRepository(root string) (*gitRepository, error) {
 
 func (g *gitRepository) GetFileLastCommit(filePath string) (*apis.Commit, error) {
 	if len(g.fileToLastCommit) == 0 {
-		filePathToCommitTime := map[string]time.Time{}
-		filePathToCommit := map[string]*git2go.Commit{}
-		allCommits, _ := g.getAllCommits()
-
-		// builds a map of all files to their last commit
-		for _, commit := range allCommits {
-			// Ignore merge commits (2+ parents)
-			if commit.ParentCount() <= 1 {
-				tree, err := commit.Tree()
-				if err != nil {
-					continue
-				}
-
-				// ParentCount can be either 1 or 0 (initial commit)
-				// In case it's the initial commit, prevTree is nil
-				var prevTree *git2go.Tree
-				if commit.ParentCount() == 1 {
-					prevCommit := commit.Parent(0)
-					prevTree, err = prevCommit.Tree()
-					if err != nil {
-						continue
-					}
-				}
-
-				diff, err := g.git2GoRepo.DiffTreeToTree(prevTree, tree, nil)
-				if err != nil {
-					continue
-				}
-
-				numDeltas, err := diff.NumDeltas()
-				if err != nil {
-					continue
-				}
-
-				for i := 0; i < numDeltas; i++ {
-					delta, err := diff.Delta(i)
-					if err != nil {
-						continue
-					}
-
-					deltaFilePath := delta.NewFile.Path
-					commitTime := commit.Author().When
-
-					// In case we have the commit information for the file which is not the latest - we override it
-					if currentCommitTime, exists := filePathToCommitTime[deltaFilePath]; exists {
-						if currentCommitTime.Before(commitTime) {
-							filePathToCommitTime[deltaFilePath] = commitTime
-							filePathToCommit[deltaFilePath] = commit
-						}
-					} else {
-						filePathToCommitTime[deltaFilePath] = commitTime
-						filePathToCommit[deltaFilePath] = commit
-					}
-				}
-			}
-		}
-
-		g.fileToLastCommit = filePathToCommit
+		g.buildCommitMap()
 	}
 
 	if relevantCommit, exists := g.fileToLastCommit[filePath]; exists {
@@ -94,6 +38,67 @@ func (g *gitRepository) GetFileLastCommit(filePath string) (*apis.Commit, error)
 	return nil, fmt.Errorf("failed to get commit information for file: %s", filePath)
 }
 
+func (g *gitRepository) buildCommitMap() {
+	filePathToCommitTime := map[string]time.Time{}
+	filePathToCommit := map[string]*git2go.Commit{}
+	allCommits, _ := g.getAllCommits()
+
+	// builds a map of all files to their last commit
+	for _, commit := range allCommits {
+		// Ignore merge commits (2+ parents)
+		if commit.ParentCount() <= 1 {
+			tree, err := commit.Tree()
+			if err != nil {
+				continue
+			}
+
+			// ParentCount can be either 1 or 0 (initial commit)
+			// In case it's the initial commit, prevTree is nil
+			var prevTree *git2go.Tree
+			if commit.ParentCount() == 1 {
+				prevCommit := commit.Parent(0)
+				prevTree, err = prevCommit.Tree()
+				if err != nil {
+					continue
+				}
+			}
+
+			diff, err := g.git2GoRepo.DiffTreeToTree(prevTree, tree, nil)
+			if err != nil {
+				continue
+			}
+
+			numDeltas, err := diff.NumDeltas()
+			if err != nil {
+				continue
+			}
+
+			for i := 0; i < numDeltas; i++ {
+				delta, err := diff.Delta(i)
+				if err != nil {
+					continue
+				}
+
+				deltaFilePath := delta.NewFile.Path
+				commitTime := commit.Author().When
+
+				// In case we have the commit information for the file which is not the latest - we override it
+				if currentCommitTime, exists := filePathToCommitTime[deltaFilePath]; exists {
+					if currentCommitTime.Before(commitTime) {
+						filePathToCommitTime[deltaFilePath] = commitTime
+						filePathToCommit[deltaFilePath] = commit
+					}
+				} else {
+					filePathToCommitTime[deltaFilePath] = commitTime
+					filePathToCommit[deltaFilePath] = commit
+				}
+			}
+		}
+	}
+
+	g.fileToLastCommit = filePathToCommit
+}
+
 func (g *gitRepository) getAllCommits() ([]*git2go.Commit, error) {
 	logItr, itrErr := g.git2GoRepo.Walk()
 	if itrErr != nil {

core/cautils/git_native_enabled_test.go

@@ -1,13 +1,18 @@
-//go:build gitenabled
 package cautils
 
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
 func (s *LocalGitRepositoryTestSuite) TestGetLastCommit() {
 	if localRepo, err := NewLocalGitRepository(s.gitRepositoryPaths["localrepo"]); s.NoError(err) {
 		if commit, err := localRepo.GetLastCommit(); s.NoError(err) {
 			s.Equal("7e09312b8017695fadcd606882e3779f10a5c832", commit.SHA)
 			s.Equal("Amir Malka", commit.Author.Name)
 			s.Equal("amirm@armosec.io", commit.Author.Email)
-			s.Equal("2022-05-22 19:11:57 +0300 +0300", commit.Author.Date.String())
+			s.Equal(int64(1653235917), commit.Author.Date.Unix())
 			s.Equal("added file B\n", commit.Message)
 		}
 	}
@@ -21,7 +26,7 @@ func (s *LocalGitRepositoryTestSuite) TestGetFileLastCommit() {
 				s.Equal("9fae4be19624297947d2b605cefbff516628612d", commit.SHA)
 				s.Equal("Amir Malka", commit.Author.Name)
 				s.Equal("amirm@armosec.io", commit.Author.Email)
-				s.Equal("2022-05-22 18:55:48 +0300 +0300", commit.Author.Date.String())
+				s.Equal(int64(1653234948), commit.Author.Date.Unix())
 				s.Equal("added file A\n", commit.Message)
 			}
 
@@ -35,10 +40,19 @@ func (s *LocalGitRepositoryTestSuite) TestGetFileLastCommit() {
 				s.Equal("7e09312b8017695fadcd606882e3779f10a5c832", commit.SHA)
 				s.Equal("Amir Malka", commit.Author.Name)
 				s.Equal("amirm@armosec.io", commit.Author.Email)
-				s.Equal("2022-05-22 19:11:57 +0300 +0300", commit.Author.Date.String())
+				s.Equal(int64(1653235917), commit.Author.Date.Unix())
 				s.Equal("added file B\n", commit.Message)
 			}
 
 		}
 	})
 }
+
+func BenchmarkBuildCommitMap(b *testing.B) {
+	localRepo, err := NewLocalGitRepository("testdata/temp/localrepo")
+	assert.NoError(b, err)
+	for i := 0; i < b.N; i++ {
+		localRepo.buildCommitMap()
+	}
+	b.ReportAllocs()
+}

core/pkg/resourcehandler/filesloader.go

@@ -219,7 +219,7 @@ func getResourcesFromPath(ctx context.Context, path string) (map[string]reportha
 		if gitRepo != nil {
 			commitInfo, err := gitRepo.GetFileLastCommit(source)
 			if err != nil && !warnIssued {
-				logger.L().Ctx(ctx).Warning("Git scan skipped", helpers.Error(err))
+				logger.L().Debug("Git scan skipped", helpers.Error(err))
 				warnIssued = true // croak only once
 			}
 

go.mod

@@ -28,6 +28,7 @@ require (
 	github.com/kubescape/rbac-utils v0.0.21-0.20230806101615-07e36f555520
 	github.com/kubescape/regolibrary v1.0.291-rc.0
 	github.com/libgit2/git2go/v33 v33.0.9
+	github.com/matthyx/go-gitlog v0.0.0-20231005131906-9ffabe3c5bcd
 	github.com/mattn/go-isatty v0.0.19
 	github.com/mikefarah/yq/v4 v4.29.1
 	github.com/olekukonko/tablewriter v0.0.6-0.20230417144759-edd1a71a5576

go.sum

@@ -1330,6 +1330,8 @@ github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsI
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
 github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
 github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
+github.com/matthyx/go-gitlog v0.0.0-20231005131906-9ffabe3c5bcd h1:Fs7gIqboEWWiJ2OdaWxK0MHGL+8XskZRZdVjJhUnSqE=
+github.com/matthyx/go-gitlog v0.0.0-20231005131906-9ffabe3c5bcd/go.mod h1:kKOGX+aQer8+30ROQinaA9vO810BGhSVHz88IpbPt60=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=

httphandler/go.mod

@@ -258,6 +258,7 @@ require (
 	github.com/libgit2/git2go/v33 v33.0.9 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/matthyx/go-gitlog v0.0.0-20231005131906-9ffabe3c5bcd // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect

httphandler/go.sum

@@ -1336,6 +1336,8 @@ github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsI
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
 github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
 github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
+github.com/matthyx/go-gitlog v0.0.0-20231005131906-9ffabe3c5bcd h1:Fs7gIqboEWWiJ2OdaWxK0MHGL+8XskZRZdVjJhUnSqE=
+github.com/matthyx/go-gitlog v0.0.0-20231005131906-9ffabe3c5bcd/go.mod h1:kKOGX+aQer8+30ROQinaA9vO810BGhSVHz88IpbPt60=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=