From d336f4484c0d8b5c99be16de20dc147e30c1fa14 Mon Sep 17 00:00:00 2001 From: Frederic BIDON Date: Tue, 13 Dec 2022 11:18:00 +0100 Subject: [PATCH] build(git): added build tag control over native git functionality * fixes #964 * adapted build and ci to use build tag * fixup error messages * report git scan skipped warning & version * fixed CI on windows: powershell parsing args... * fixup leftover comment * fixup typo in test message * resolved merge conflicts on unit tests * fix: added gitenabled tag to Makefile target Signed-off-by: Frederic BIDON --- .github/workflows/release.yaml | 2 +- .github/workflows/test.yaml | 4 +- Makefile | 2 +- build.py | 12 +- cmd/version/git_native_disabled.go | 7 ++ cmd/version/git_native_enabled.go | 7 ++ cmd/version/version.go | 6 +- core/cautils/git_native_disabled.go | 22 ++++ core/cautils/git_native_disabled_test.go | 11 ++ core/cautils/git_native_enabled.go | 141 +++++++++++++++++++++++ core/cautils/git_native_enabled_test.go | 44 +++++++ core/cautils/localgitrepository.go | 135 ++-------------------- core/cautils/localgitrepository_test.go | 42 ------- core/pkg/resourcehandler/filesloader.go | 8 +- 14 files changed, 265 insertions(+), 178 deletions(-) create mode 100644 cmd/version/git_native_disabled.go create mode 100644 cmd/version/git_native_enabled.go create mode 100644 core/cautils/git_native_disabled.go create mode 100644 core/cautils/git_native_disabled_test.go create mode 100644 core/cautils/git_native_enabled.go create mode 100644 core/cautils/git_native_enabled_test.go diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ee3cc92b..76d1e74a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -38,4 +38,4 @@ jobs: release_name: ${{ inputs.release_name }} draft: ${{ inputs.draft }} prerelease: false - \ No newline at end of file + diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9578c4fb..9129adea 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -73,10 +73,10 @@ jobs: if: matrix.os != 'windows-latest' - name: Test core pkg - run: go test -tags=static -v ./... + run: go test "-tags=static,gitenabled" -v ./... - name: Test httphandler pkg - run: cd httphandler && go test -tags=static -v ./... + run: cd httphandler && go test "-tags=static,gitenabled" -v ./... - name: Build env: diff --git a/Makefile b/Makefile index f45b5953..4801e37d 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ libgit2: cd git2go; make install-static # go build tags -TAGS = "static" +TAGS = "gitenabled,static" build: go build -v -tags=$(TAGS) . diff --git a/build.py b/build.py index 9ddd46bc..f2f0ade5 100644 --- a/build.py +++ b/build.py @@ -40,7 +40,7 @@ def main(): client_var = "github.com/kubescape/kubescape/v2/core/cautils.Client" client_name = os.getenv("CLIENT") - + # Create build directory build_dir = get_build_dir() @@ -56,15 +56,15 @@ def main(): ldflags += " -X {}={}".format(build_url, release_version) if client_name: ldflags += " -X {}={}".format(client_var, client_name) - - build_command = ["go", "build", "-buildmode=pie", "-tags=static", "-o", ks_file, "-ldflags" ,ldflags] + + build_command = ["go", "build", "-buildmode=pie", "-tags=static,gitenabled", "-o", ks_file, "-ldflags" ,ldflags] print("Building kubescape and saving here: {}".format(ks_file)) print("Build command: {}".format(" ".join(build_command))) status = subprocess.call(build_command) check_status(status, "Failed to build kubescape") - + sha256 = hashlib.sha256() with open(ks_file, "rb") as kube: sha256.update(kube.read()) @@ -74,7 +74,7 @@ def main(): kube_sha.write(sha256.hexdigest()) print("Build Done") - - + + if __name__ == "__main__": main() diff --git a/cmd/version/git_native_disabled.go b/cmd/version/git_native_disabled.go new file mode 100644 index 00000000..c0c9bd29 --- /dev/null +++ b/cmd/version/git_native_disabled.go @@ -0,0 +1,7 @@ +//go:build !gitenabled + +package version + +func isGitEnabled() bool { + return false +} diff --git a/cmd/version/git_native_enabled.go b/cmd/version/git_native_enabled.go new file mode 100644 index 00000000..518404f9 --- /dev/null +++ b/cmd/version/git_native_enabled.go @@ -0,0 +1,7 @@ +//go:build gitenabled + +package version + +func isGitEnabled() bool { + return true +} diff --git a/cmd/version/version.go b/cmd/version/version.go index e5cc8b13..e19f1372 100644 --- a/cmd/version/version.go +++ b/cmd/version/version.go @@ -16,7 +16,11 @@ func GetVersionCmd() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { v := cautils.NewIVersionCheckHandler() v.CheckLatestVersion(cautils.NewVersionCheckRequest(cautils.BuildNumber, "", "", "version")) - fmt.Fprintln(os.Stdout, "Your current version is: "+cautils.BuildNumber) + fmt.Fprintf(os.Stdout, + "Your current version is: %s [git enabled in build: %t]\n", + cautils.BuildNumber, + isGitEnabled(), + ) return nil }, } diff --git a/core/cautils/git_native_disabled.go b/core/cautils/git_native_disabled.go new file mode 100644 index 00000000..c8c58403 --- /dev/null +++ b/core/cautils/git_native_disabled.go @@ -0,0 +1,22 @@ +//go:build !gitenabled + +package cautils + +import ( + "errors" + + "github.com/kubescape/go-git-url/apis" +) + +var ErrFatalNotSupportedByBuild = errors.New(`git scan not supported by this build. Build with tag "gitenabled" to enable the git scan feature`) + +type gitRepository struct { +} + +func newGitRepository(root string) (*gitRepository, error) { + return &gitRepository{}, ErrWarnNotSupportedByBuild +} + +func (g *gitRepository) GetFileLastCommit(filePath string) (*apis.Commit, error) { + return nil, ErrFatalNotSupportedByBuild +} diff --git a/core/cautils/git_native_disabled_test.go b/core/cautils/git_native_disabled_test.go new file mode 100644 index 00000000..ccc23346 --- /dev/null +++ b/core/cautils/git_native_disabled_test.go @@ -0,0 +1,11 @@ +//go:build !gitenabled + +package cautils + +func (s *LocalGitRepositoryTestSuite) TestGetLastCommit() { + s.T().Log("warn: skipped testing native git functionality [GetLastCommit]") +} + +func (s *LocalGitRepositoryTestSuite) TestGetFileLastCommit() { + s.T().Log("warn: skipped testing native git functionality [GetFileLastCommit]") +} diff --git a/core/cautils/git_native_enabled.go b/core/cautils/git_native_enabled.go new file mode 100644 index 00000000..09cdaf71 --- /dev/null +++ b/core/cautils/git_native_enabled.go @@ -0,0 +1,141 @@ +//go:build gitenabled +package cautils + +import ( + "fmt" + "time" + + "github.com/kubescape/go-git-url/apis" + git2go "github.com/libgit2/git2go/v33" +) + +type gitRepository struct { + git2GoRepo *git2go.Repository + fileToLastCommit map[string]*git2go.Commit +} + +func newGitRepository(root string) (*gitRepository, error) { + git2GoRepo, err := git2go.OpenRepository(root) + if err != nil { + return nil, err + } + + return &gitRepository{ + git2GoRepo: git2GoRepo, + }, nil +} + +func (g *gitRepository) GetFileLastCommit(filePath string) (*apis.Commit, error) { + if len(g.fileToLastCommit) == 0 { + filePathToCommitTime := map[string]time.Time{} + filePathToCommit := map[string]*git2go.Commit{} + allCommits, _ := g.getAllCommits() + + // builds a map of all files to their last commit + for _, commit := range allCommits { + // Ignore merge commits (2+ parents) + if commit.ParentCount() <= 1 { + tree, err := commit.Tree() + if err != nil { + continue + } + + // ParentCount can be either 1 or 0 (initial commit) + // In case it's the initial commit, prevTree is nil + var prevTree *git2go.Tree + if commit.ParentCount() == 1 { + prevCommit := commit.Parent(0) + prevTree, err = prevCommit.Tree() + if err != nil { + continue + } + } + + diff, err := g.git2GoRepo.DiffTreeToTree(prevTree, tree, nil) + if err != nil { + continue + } + + numDeltas, err := diff.NumDeltas() + if err != nil { + continue + } + + for i := 0; i < numDeltas; i++ { + delta, err := diff.Delta(i) + if err != nil { + continue + } + + deltaFilePath := delta.NewFile.Path + commitTime := commit.Author().When + + // In case we have the commit information for the file which is not the latest - we override it + if currentCommitTime, exists := filePathToCommitTime[deltaFilePath]; exists { + if currentCommitTime.Before(commitTime) { + filePathToCommitTime[deltaFilePath] = commitTime + filePathToCommit[deltaFilePath] = commit + } + } else { + filePathToCommitTime[deltaFilePath] = commitTime + filePathToCommit[deltaFilePath] = commit + } + } + } + } + + g.fileToLastCommit = filePathToCommit + } + + if relevantCommit, exists := g.fileToLastCommit[filePath]; exists { + return g.getCommit(relevantCommit), nil + } + + return nil, fmt.Errorf("failed to get commit information for file: %s", filePath) +} + +func (g *gitRepository) getAllCommits() ([]*git2go.Commit, error) { + logItr, itrErr := g.git2GoRepo.Walk() + if itrErr != nil { + + return nil, itrErr + } + + pushErr := logItr.PushHead() + if pushErr != nil { + return nil, pushErr + } + + var allCommits []*git2go.Commit + err := logItr.Iterate(func(commit *git2go.Commit) bool { + if commit != nil { + allCommits = append(allCommits, commit) + return true + } + return false + }) + + if err != nil { + return nil, err + } + + if err != nil { + return nil, err + } + + return allCommits, nil +} + +func (g *gitRepository) getCommit(commit *git2go.Commit) *apis.Commit { + return &apis.Commit{ + SHA: commit.Id().String(), + Author: apis.Committer{ + Name: commit.Author().Name, + Email: commit.Author().Email, + Date: commit.Author().When, + }, + Message: commit.Message(), + Committer: apis.Committer{}, + Files: []apis.Files{}, + } +} diff --git a/core/cautils/git_native_enabled_test.go b/core/cautils/git_native_enabled_test.go new file mode 100644 index 00000000..09ae5fbb --- /dev/null +++ b/core/cautils/git_native_enabled_test.go @@ -0,0 +1,44 @@ +//go:build gitenabled +package cautils + +func (s *LocalGitRepositoryTestSuite) TestGetLastCommit() { + if localRepo, err := NewLocalGitRepository(s.gitRepositoryPaths["localrepo"]); s.NoError(err) { + if commit, err := localRepo.GetLastCommit(); s.NoError(err) { + s.Equal("7e09312b8017695fadcd606882e3779f10a5c832", commit.SHA) + s.Equal("Amir Malka", commit.Author.Name) + s.Equal("amirm@armosec.io", commit.Author.Email) + s.Equal("2022-05-22 19:11:57 +0300 +0300", commit.Author.Date.String()) + s.Equal("added file B\n", commit.Message) + } + } +} + +func (s *LocalGitRepositoryTestSuite) TestGetFileLastCommit() { + s.Run("fileA", func() { + if localRepo, err := NewLocalGitRepository(s.gitRepositoryPaths["localrepo"]); s.NoError(err) { + + if commit, err := localRepo.GetFileLastCommit("fileA"); s.NoError(err) { + s.Equal("9fae4be19624297947d2b605cefbff516628612d", commit.SHA) + s.Equal("Amir Malka", commit.Author.Name) + s.Equal("amirm@armosec.io", commit.Author.Email) + s.Equal("2022-05-22 18:55:48 +0300 +0300", commit.Author.Date.String()) + s.Equal("added file A\n", commit.Message) + } + + } + }) + + s.Run("fileB", func() { + if localRepo, err := NewLocalGitRepository(s.gitRepositoryPaths["localrepo"]); s.NoError(err) { + + if commit, err := localRepo.GetFileLastCommit("dirA/fileB"); s.NoError(err) { + s.Equal("7e09312b8017695fadcd606882e3779f10a5c832", commit.SHA) + s.Equal("Amir Malka", commit.Author.Name) + s.Equal("amirm@armosec.io", commit.Author.Email) + s.Equal("2022-05-22 19:11:57 +0300 +0300", commit.Author.Date.String()) + s.Equal("added file B\n", commit.Message) + } + + } + }) +} diff --git a/core/cautils/localgitrepository.go b/core/cautils/localgitrepository.go index 643fc0b6..adf9b96d 100644 --- a/core/cautils/localgitrepository.go +++ b/core/cautils/localgitrepository.go @@ -1,26 +1,26 @@ package cautils import ( + "errors" "fmt" "path" "strings" - "time" gitv5 "github.com/go-git/go-git/v5" configv5 "github.com/go-git/go-git/v5/config" plumbingv5 "github.com/go-git/go-git/v5/plumbing" "github.com/kubescape/go-git-url/apis" - git2go "github.com/libgit2/git2go/v33" ) type LocalGitRepository struct { - goGitRepo *gitv5.Repository - git2GoRepo *git2go.Repository - head *plumbingv5.Reference - config *configv5.Config - fileToLastCommit map[string]*git2go.Commit + *gitRepository + goGitRepo *gitv5.Repository + head *plumbingv5.Reference + config *configv5.Config } +var ErrWarnNotSupportedByBuild = errors.New(`git commits retrieval not supported by this build. Build with tag "gitenabled" to enable the full git scan feature`) + func NewLocalGitRepository(path string) (*LocalGitRepository, error) { goGitRepo, err := gitv5.PlainOpenWithOptions(path, &gitv5.PlainOpenOptions{DetectDotGit: true}) if err != nil { @@ -52,11 +52,12 @@ func NewLocalGitRepository(path string) (*LocalGitRepository, error) { } if repoRoot, err := l.GetRootDir(); err == nil { - git2GoRepo, err := git2go.OpenRepository(repoRoot) - if err != nil { + gitRepository, err := newGitRepository(repoRoot) + if err != nil && !errors.Is(err, ErrWarnNotSupportedByBuild) { return l, err } - l.git2GoRepo = git2GoRepo + + l.gitRepository = gitRepository } return l, nil @@ -125,120 +126,6 @@ func (g *LocalGitRepository) GetLastCommit() (*apis.Commit, error) { }, nil } -func (g *LocalGitRepository) getAllCommits() ([]*git2go.Commit, error) { - logItr, itrErr := g.git2GoRepo.Walk() - if itrErr != nil { - - return nil, itrErr - } - - pushErr := logItr.PushHead() - if pushErr != nil { - return nil, pushErr - } - - var allCommits []*git2go.Commit - err := logItr.Iterate(func(commit *git2go.Commit) bool { - if commit != nil { - allCommits = append(allCommits, commit) - return true - } - return false - }) - - if err != nil { - return nil, err - } - - if err != nil { - return nil, err - } - - return allCommits, nil -} - -func (g *LocalGitRepository) GetFileLastCommit(filePath string) (*apis.Commit, error) { - if len(g.fileToLastCommit) == 0 { - filePathToCommitTime := map[string]time.Time{} - filePathToCommit := map[string]*git2go.Commit{} - allCommits, _ := g.getAllCommits() - - // builds a map of all files to their last commit - for _, commit := range allCommits { - // Ignore merge commits (2+ parents) - if commit.ParentCount() <= 1 { - tree, err := commit.Tree() - if err != nil { - continue - } - - // ParentCount can be either 1 or 0 (initial commit) - // In case it's the initial commit, prevTree is nil - var prevTree *git2go.Tree - if commit.ParentCount() == 1 { - prevCommit := commit.Parent(0) - prevTree, err = prevCommit.Tree() - if err != nil { - continue - } - } - - diff, err := g.git2GoRepo.DiffTreeToTree(prevTree, tree, nil) - if err != nil { - continue - } - - numDeltas, err := diff.NumDeltas() - if err != nil { - continue - } - - for i := 0; i < numDeltas; i++ { - delta, err := diff.Delta(i) - if err != nil { - continue - } - - deltaFilePath := delta.NewFile.Path - commitTime := commit.Author().When - - // In case we have the commit information for the file which is not the latest - we override it - if currentCommitTime, exists := filePathToCommitTime[deltaFilePath]; exists { - if currentCommitTime.Before(commitTime) { - filePathToCommitTime[deltaFilePath] = commitTime - filePathToCommit[deltaFilePath] = commit - } - } else { - filePathToCommitTime[deltaFilePath] = commitTime - filePathToCommit[deltaFilePath] = commit - } - } - } - } - g.fileToLastCommit = filePathToCommit - } - - if relevantCommit, exists := g.fileToLastCommit[filePath]; exists { - return g.getCommit(relevantCommit), nil - } - - return nil, fmt.Errorf("failed to get commit information for file: %s", filePath) -} - -func (g *LocalGitRepository) getCommit(commit *git2go.Commit) *apis.Commit { - return &apis.Commit{ - SHA: commit.Id().String(), - Author: apis.Committer{ - Name: commit.Author().Name, - Email: commit.Author().Email, - Date: commit.Author().When, - }, - Message: commit.Message(), - Committer: apis.Committer{}, - Files: []apis.Files{}, - } -} - func (g *LocalGitRepository) GetRootDir() (string, error) { wt, err := g.goGitRepo.Worktree() if err != nil { diff --git a/core/cautils/localgitrepository_test.go b/core/cautils/localgitrepository_test.go index 4d149f66..59465418 100644 --- a/core/cautils/localgitrepository_test.go +++ b/core/cautils/localgitrepository_test.go @@ -134,48 +134,6 @@ func (s *LocalGitRepositoryTestSuite) TestGetOriginUrl() { } } -func (s *LocalGitRepositoryTestSuite) TestGetLastCommit() { - if localRepo, err := NewLocalGitRepository(s.gitRepositoryPaths["localrepo"]); s.NoError(err) { - if commit, err := localRepo.GetLastCommit(); s.NoError(err) { - s.Equal("7e09312b8017695fadcd606882e3779f10a5c832", commit.SHA) - s.Equal("Amir Malka", commit.Author.Name) - s.Equal("amirm@armosec.io", commit.Author.Email) - s.Equal("2022-05-22 19:11:57 +0300 +0300", commit.Author.Date.String()) - s.Equal("added file B\n", commit.Message) - } - } -} - -func (s *LocalGitRepositoryTestSuite) TestGetFileLastCommit() { - s.Run("fileA", func() { - if localRepo, err := NewLocalGitRepository(s.gitRepositoryPaths["localrepo"]); s.NoError(err) { - - if commit, err := localRepo.GetFileLastCommit("fileA"); s.NoError(err) { - s.Equal("9fae4be19624297947d2b605cefbff516628612d", commit.SHA) - s.Equal("Amir Malka", commit.Author.Name) - s.Equal("amirm@armosec.io", commit.Author.Email) - s.Equal("2022-05-22 18:55:48 +0300 +0300", commit.Author.Date.String()) - s.Equal("added file A\n", commit.Message) - } - - } - }) - - s.Run("fileB", func() { - if localRepo, err := NewLocalGitRepository(s.gitRepositoryPaths["localrepo"]); s.NoError(err) { - - if commit, err := localRepo.GetFileLastCommit("dirA/fileB"); s.NoError(err) { - s.Equal("7e09312b8017695fadcd606882e3779f10a5c832", commit.SHA) - s.Equal("Amir Malka", commit.Author.Name) - s.Equal("amirm@armosec.io", commit.Author.Email) - s.Equal("2022-05-22 19:11:57 +0300 +0300", commit.Author.Date.String()) - s.Equal("added file B\n", commit.Message) - } - - } - }) -} - func TestGetRemoteUrl(t *testing.T) { testCases := []struct { Name string diff --git a/core/pkg/resourcehandler/filesloader.go b/core/pkg/resourcehandler/filesloader.go index 411b86ff..e59765ff 100644 --- a/core/pkg/resourcehandler/filesloader.go +++ b/core/pkg/resourcehandler/filesloader.go @@ -112,6 +112,7 @@ func getResourcesFromPath(path string) (map[string]reporthandling.Source, []work sourceToWorkloads := cautils.LoadResourcesFromFiles(path, repoRoot) // update workloads and workloadIDToSource + var warnIssued bool for source, ws := range sourceToWorkloads { workloads = append(workloads, ws...) @@ -131,7 +132,12 @@ func getResourcesFromPath(path string) (map[string]reporthandling.Source, []work var lastCommit reporthandling.LastCommit if gitRepo != nil { - commitInfo, _ := gitRepo.GetFileLastCommit(source) + commitInfo, err := gitRepo.GetFileLastCommit(source) + if err != nil && !warnIssued { + logger.L().Warning("git scan skipped", helpers.Error(err)) + warnIssued = true // croak only once + } + if commitInfo != nil { lastCommit = reporthandling.LastCommit{ Hash: commitInfo.SHA,